Skip to content

Commit

Permalink
[YAML] Add support for non-printable characters
Browse files Browse the repository at this point in the history 
LLVM IR function names which disable mangling start with '\01'
(https://www.llvm.org/docs/LangRef.html#identifiers).

When an identifier like "\01@abc@" gets dumped to MIR, it is quoted, but
only with single quotes.

http://www.yaml.org/spec/1.2/spec.html#id2770814:

"The allowed character range explicitly excludes the C0 control block
allowed), the surrogate block #xD800-#xDFFF, #xFFFE, and #xFFFF."

http://www.yaml.org/spec/1.2/spec.html#id2776092:

"All non-printable characters must be escaped.
[...]
Note that escape sequences are only interpreted in double-quoted scalars."

This patch adds support for printing escaped non-printable characters
between double quotes if needed.

Should also fix PR31743.

Differential Revision: https://reviews.llvm.org/D41290

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@320996 91177308-0d34-0410-b5e6-96231b3b80d8
  • Loading branch information
@francisvm
francisvm committed Dec 18, 2017
1 parent d6be214 commit 65ad22d
Show file tree
Hide file tree
Showing 17 changed files with 245 additions and 76 deletions.
2 changes: 1 addition & 1 deletion docs/YamlIO.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -466,7 +466,7 @@ looks like:
return StringRef();
}
// Determine if this scalar needs quotes.
static bool mustQuote(StringRef) { return true; }
static QuotingType mustQuote(StringRef) { return QuotingType::Single; }
};

Block Scalars
Expand Down
6 changes: 3 additions & 3 deletions include/llvm/CodeGen/MIRYamlMapping.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ template <> struct ScalarTraits<StringValue> {
return "";
}

static bool mustQuote(StringRef Scalar) { return needsQuotes(Scalar); }
static QuotingType mustQuote(StringRef S) { return needsQuotes(S); }
};

struct FlowStringValue : StringValue {
Expand All @@ -73,7 +73,7 @@ template <> struct ScalarTraits<FlowStringValue> {
return ScalarTraits<StringValue>::input(Scalar, Ctx, S);
}

static bool mustQuote(StringRef Scalar) { return needsQuotes(Scalar); }
static QuotingType mustQuote(StringRef S) { return needsQuotes(S); }
};

struct BlockStringValue {
Expand Down Expand Up @@ -120,7 +120,7 @@ template <> struct ScalarTraits<UnsignedValue> {
return ScalarTraits<unsigned>::input(Scalar, Ctx, Value.Value);
}

static bool mustQuote(StringRef Scalar) {
static QuotingType mustQuote(StringRef Scalar) {
return ScalarTraits<unsigned>::mustQuote(Scalar);
}
};
Expand Down
2 changes: 1 addition & 1 deletion include/llvm/ObjectYAML/CodeViewYAMLTypeHashing.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ ArrayRef<uint8_t> toDebugH(const DebugHSection &DebugH,
} // end namespace llvm

LLVM_YAML_DECLARE_MAPPING_TRAITS(CodeViewYAML::DebugHSection)
LLVM_YAML_DECLARE_SCALAR_TRAITS(CodeViewYAML::GlobalHash, false)
LLVM_YAML_DECLARE_SCALAR_TRAITS(CodeViewYAML::GlobalHash, QuotingType::None)
LLVM_YAML_IS_SEQUENCE_VECTOR(CodeViewYAML::GlobalHash)

#endif // LLVM_OBJECTYAML_CODEVIEWYAMLTYPES_H
2 changes: 1 addition & 1 deletion include/llvm/ObjectYAML/CodeViewYAMLTypes.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ ArrayRef<uint8_t> toDebugT(ArrayRef<LeafRecord>, BumpPtrAllocator &Alloc);

} // end namespace llvm

LLVM_YAML_DECLARE_SCALAR_TRAITS(codeview::GUID, true)
LLVM_YAML_DECLARE_SCALAR_TRAITS(codeview::GUID, QuotingType::Single)

LLVM_YAML_DECLARE_MAPPING_TRAITS(CodeViewYAML::LeafRecord)
LLVM_YAML_DECLARE_MAPPING_TRAITS(CodeViewYAML::MemberRecord)
Expand Down
4 changes: 2 additions & 2 deletions include/llvm/ObjectYAML/MachOYAML.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -261,7 +261,7 @@ using char_16 = char[16];
template <> struct ScalarTraits<char_16> {
static void output(const char_16 &Val, void *, raw_ostream &Out);
static StringRef input(StringRef Scalar, void *, char_16 &Val);
static bool mustQuote(StringRef S);
static QuotingType mustQuote(StringRef S);
};

// This trait is used for UUIDs. It reads and writes them matching otool's
Expand All @@ -271,7 +271,7 @@ using uuid_t = raw_ostream::uuid_t;
template <> struct ScalarTraits<uuid_t> {
static void output(const uuid_t &Val, void *, raw_ostream &Out);
static StringRef input(StringRef Scalar, void *, uuid_t &Val);
static bool mustQuote(StringRef S);
static QuotingType mustQuote(StringRef S);
};

// Load Command struct mapping traits
Expand Down
2 changes: 1 addition & 1 deletion include/llvm/ObjectYAML/YAML.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ inline bool operator==(const BinaryRef &LHS, const BinaryRef &RHS) {
template <> struct ScalarTraits<BinaryRef> {
static void output(const BinaryRef &, void *, raw_ostream &);
static StringRef input(StringRef, void *, BinaryRef &);
static bool mustQuote(StringRef S) { return needsQuotes(S); }
static QuotingType mustQuote(StringRef S) { return needsQuotes(S); }
};

} // end namespace yaml
Expand Down
124 changes: 84 additions & 40 deletions include/llvm/Support/YAMLTraits.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@

#include "llvm/ADT/Optional.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Twine.h"
Expand Down Expand Up @@ -117,6 +118,11 @@ struct ScalarBitSetTraits {
// static void bitset(IO &io, T &value);
};

/// Describe which type of quotes should be used when quoting is necessary.
/// Some non-printable characters need to be double-quoted, while some others
/// are fine with simple-quoting, and some don't need any quoting.
enum class QuotingType { None, Single, Double };

/// This class should be specialized by type that requires custom conversion
/// to/from a yaml scalar. For example:
///
Expand All @@ -131,7 +137,7 @@ struct ScalarBitSetTraits {
/// // return empty string on success, or error string
/// return StringRef();
/// }
/// static bool mustQuote(StringRef) { return true; }
/// static QuotingType mustQuote(StringRef) { return QuotingType::Single; }
/// };
template<typename T>
struct ScalarTraits {
Expand All @@ -145,7 +151,7 @@ struct ScalarTraits {
//static StringRef input(StringRef scalar, void *ctxt, T &value);
//
// Function to determine if the value should be quoted.
//static bool mustQuote(StringRef);
//static QuotingType mustQuote(StringRef);
};

/// This class should be specialized by type that requires custom conversion
Expand Down Expand Up @@ -270,7 +276,7 @@ struct has_ScalarTraits
{
using Signature_input = StringRef (*)(StringRef, void*, T&);
using Signature_output = void (*)(const T&, void*, raw_ostream&);
using Signature_mustQuote = bool (*)(StringRef);
using Signature_mustQuote = QuotingType (*)(StringRef);

template <typename U>
static char test(SameType<Signature_input, &U::input> *,
Expand Down Expand Up @@ -495,28 +501,66 @@ inline bool isBool(StringRef S) {
S.equals("false") || S.equals("False") || S.equals("FALSE");
}

inline bool needsQuotes(StringRef S) {
// 5.1. Character Set
// The allowed character range explicitly excludes the C0 control block #x0-#x1F
// (except for TAB #x9, LF #xA, and CR #xD which are allowed), DEL #x7F, the C1
// control block #x80-#x9F (except for NEL #x85 which is allowed), the surrogate
// block #xD800-#xDFFF, #xFFFE, and #xFFFF.
inline QuotingType needsQuotes(StringRef S) {
if (S.empty())
return true;
return QuotingType::Single;
if (isspace(S.front()) || isspace(S.back()))
return true;
return QuotingType::Single;
if (S.front() == ',')
return true;

static const char ScalarSafeChars[] =
"abcdefghijklmnopqrstuvwxyz"
"ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-/^., \t";
if (S.find_first_not_of(ScalarSafeChars) != StringRef::npos)
return true;

return QuotingType::Single;
if (isNull(S))
return true;
return QuotingType::Single;
if (isBool(S))
return true;
return QuotingType::Single;
if (isNumeric(S))
return true;
return QuotingType::Single;

QuotingType MaxQuotingNeeded = QuotingType::None;
for (unsigned char C : S) {
// Alphanum is safe.
if (isAlnum(C))
continue;

switch (C) {
// Safe scalar characters.
case '_':
case '-':
case '/':
case '^':
case '.':
case ',':
case ' ':
// TAB (0x9), LF (0xA), CR (0xD) and NEL (0x85) are allowed.
case 0x9:
case 0xA:
case 0xD:
case 0x85:
continue;
// DEL (0x7F) are excluded from the allowed character range.
case 0x7F:
return QuotingType::Double;
default: {
// C0 control block (0x0 - 0x1F) is excluded from the allowed character
// range.
if (C <= 0x1F)
return QuotingType::Double;
// C1 control block (0x80 - 0x9F) is excluded from the allowed character
// range.
if (C >= 0x80 && C <= 0x9F)
return QuotingType::Double;

// The character is not safe, at least simple quoting needed.
MaxQuotingNeeded = QuotingType::Single;
}
}
}

return false;
return MaxQuotingNeeded;
}

template <typename T, typename Context>
Expand Down Expand Up @@ -581,7 +625,7 @@ class IO {
virtual bool bitSetMatch(const char*, bool) = 0;
virtual void endBitSetScalar() = 0;

virtual void scalarString(StringRef &, bool) = 0;
virtual void scalarString(StringRef &, QuotingType) = 0;
virtual void blockScalarString(StringRef &) = 0;

virtual void setError(const Twine &) = 0;
Expand Down Expand Up @@ -911,91 +955,91 @@ template<>
struct ScalarTraits<bool> {
static void output(const bool &, void* , raw_ostream &);
static StringRef input(StringRef, void *, bool &);
static bool mustQuote(StringRef) { return false; }
static QuotingType mustQuote(StringRef) { return QuotingType::None; }
};

template<>
struct ScalarTraits<StringRef> {
static void output(const StringRef &, void *, raw_ostream &);
static StringRef input(StringRef, void *, StringRef &);
static bool mustQuote(StringRef S) { return needsQuotes(S); }
static QuotingType mustQuote(StringRef S) { return needsQuotes(S); }
};

template<>
struct ScalarTraits<std::string> {
static void output(const std::string &, void *, raw_ostream &);
static StringRef input(StringRef, void *, std::string &);
static bool mustQuote(StringRef S) { return needsQuotes(S); }
static QuotingType mustQuote(StringRef S) { return needsQuotes(S); }
};

template<>
struct ScalarTraits<uint8_t> {
static void output(const uint8_t &, void *, raw_ostream &);
static StringRef input(StringRef, void *, uint8_t &);
static bool mustQuote(StringRef) { return false; }
static QuotingType mustQuote(StringRef) { return QuotingType::None; }
};

template<>
struct ScalarTraits<uint16_t> {
static void output(const uint16_t &, void *, raw_ostream &);
static StringRef input(StringRef, void *, uint16_t &);
static bool mustQuote(StringRef) { return false; }
static QuotingType mustQuote(StringRef) { return QuotingType::None; }
};

template<>
struct ScalarTraits<uint32_t> {
static void output(const uint32_t &, void *, raw_ostream &);
static StringRef input(StringRef, void *, uint32_t &);
static bool mustQuote(StringRef) { return false; }
static QuotingType mustQuote(StringRef) { return QuotingType::None; }
};

template<>
struct ScalarTraits<uint64_t> {
static void output(const uint64_t &, void *, raw_ostream &);
static StringRef input(StringRef, void *, uint64_t &);
static bool mustQuote(StringRef) { return false; }
static QuotingType mustQuote(StringRef) { return QuotingType::None; }
};

template<>
struct ScalarTraits<int8_t> {
static void output(const int8_t &, void *, raw_ostream &);
static StringRef input(StringRef, void *, int8_t &);
static bool mustQuote(StringRef) { return false; }
static QuotingType mustQuote(StringRef) { return QuotingType::None; }
};

template<>
struct ScalarTraits<int16_t> {
static void output(const int16_t &, void *, raw_ostream &);
static StringRef input(StringRef, void *, int16_t &);
static bool mustQuote(StringRef) { return false; }
static QuotingType mustQuote(StringRef) { return QuotingType::None; }
};

template<>
struct ScalarTraits<int32_t> {
static void output(const int32_t &, void *, raw_ostream &);
static StringRef input(StringRef, void *, int32_t &);
static bool mustQuote(StringRef) { return false; }
static QuotingType mustQuote(StringRef) { return QuotingType::None; }
};

template<>
struct ScalarTraits<int64_t> {
static void output(const int64_t &, void *, raw_ostream &);
static StringRef input(StringRef, void *, int64_t &);
static bool mustQuote(StringRef) { return false; }
static QuotingType mustQuote(StringRef) { return QuotingType::None; }
};

template<>
struct ScalarTraits<float> {
static void output(const float &, void *, raw_ostream &);
static StringRef input(StringRef, void *, float &);
static bool mustQuote(StringRef) { return false; }
static QuotingType mustQuote(StringRef) { return QuotingType::None; }
};

template<>
struct ScalarTraits<double> {
static void output(const double &, void *, raw_ostream &);
static StringRef input(StringRef, void *, double &);
static bool mustQuote(StringRef) { return false; }
static QuotingType mustQuote(StringRef) { return QuotingType::None; }
};

// For endian types, we just use the existing ScalarTraits for the underlying
Expand All @@ -1019,7 +1063,7 @@ struct ScalarTraits<support::detail::packed_endian_specific_integral<
return R;
}

static bool mustQuote(StringRef Str) {
static QuotingType mustQuote(StringRef Str) {
return ScalarTraits<value_type>::mustQuote(Str);
}
};
Expand Down Expand Up @@ -1148,7 +1192,7 @@ class Input : public IO {
bool beginBitSetScalar(bool &) override;
bool bitSetMatch(const char *, bool ) override;
void endBitSetScalar() override;
void scalarString(StringRef &, bool) override;
void scalarString(StringRef &, QuotingType) override;
void blockScalarString(StringRef &) override;
void setError(const Twine &message) override;
bool canElideEmptySequence() override;
Expand Down Expand Up @@ -1293,7 +1337,7 @@ class Output : public IO {
bool beginBitSetScalar(bool &) override;
bool bitSetMatch(const char *, bool ) override;
void endBitSetScalar() override;
void scalarString(StringRef &, bool) override;
void scalarString(StringRef &, QuotingType) override;
void blockScalarString(StringRef &) override;
void setError(const Twine &message) override;
bool canElideEmptySequence() override;
Expand Down Expand Up @@ -1371,28 +1415,28 @@ template<>
struct ScalarTraits<Hex8> {
static void output(const Hex8 &, void *, raw_ostream &);
static StringRef input(StringRef, void *, Hex8 &);
static bool mustQuote(StringRef) { return false; }
static QuotingType mustQuote(StringRef) { return QuotingType::None; }
};

template<>
struct ScalarTraits<Hex16> {
static void output(const Hex16 &, void *, raw_ostream &);
static StringRef input(StringRef, void *, Hex16 &);
static bool mustQuote(StringRef) { return false; }
static QuotingType mustQuote(StringRef) { return QuotingType::None; }
};

template<>
struct ScalarTraits<Hex32> {
static void output(const Hex32 &, void *, raw_ostream &);
static StringRef input(StringRef, void *, Hex32 &);
static bool mustQuote(StringRef) { return false; }
static QuotingType mustQuote(StringRef) { return QuotingType::None; }
};

template<>
struct ScalarTraits<Hex64> {
static void output(const Hex64 &, void *, raw_ostream &);
static StringRef input(StringRef, void *, Hex64 &);
static bool mustQuote(StringRef) { return false; }
static QuotingType mustQuote(StringRef) { return QuotingType::None; }
};

// Define non-member operator>> so that Input can stream in a document list.
Expand Down Expand Up @@ -1681,7 +1725,7 @@ template <typename T> struct StdMapStringCustomMappingTraitsImpl {
template <> struct ScalarTraits<Type> { \
static void output(const Type &Value, void *ctx, raw_ostream &Out); \
static StringRef input(StringRef Scalar, void *ctxt, Type &Value); \
static bool mustQuote(StringRef) { return MustQuote; } \
static QuotingType mustQuote(StringRef) { return MustQuote; } \
}; \
} \
}
Expand Down
Loading

0 comments on commit 65ad22d

Please sign in to comment.