forked from swiftlang/swift
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Preserve whitespace and comments during lexing as Trivia
Store leading a trailing "trivia" around a token, such as whitespace, comments, doc comments, and escaping backticks. These are syntactically important for preserving formatting when printing ASTs but don't semantically affect the program. Tokens take all trailing trivia up to, but not including, the next newline. This is important to maintain checks that statements without semicolon separators start on a new line, among other things. Trivia are now data attached to the ends of tokens, not tokens themselves. Create a new Syntax sublibrary for upcoming immutable, persistent, thread-safe ASTs, which will contain only the syntactic information about source structure, as well as for generating new source code, and structural editing. Proactively move swift::Token into there. Since this patch is getting a bit large, a token fuzzer which checks for round-trip equivlence with the workflow: fuzzer => token stream => file1 => Lexer => token stream => file 2 => diff(file1, file2) Will arrive in a subsequent commit. This patch does not change the grammar.
- Loading branch information
Showing
56 changed files
with
1,817 additions
and
1,049 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,94 @@ | ||
| //===--- String.h - String storage ------------------------------*- C++ -*-===// | ||
| // | ||
| // This source file is part of the Swift.org open source project | ||
| // | ||
| // Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors | ||
| // Licensed under Apache License v2.0 with Runtime Library Exception | ||
| // | ||
| // See http://swift.org/LICENSE.txt for license information | ||
| // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors | ||
| // | ||
| //===----------------------------------------------------------------------===// | ||
| // | ||
| // This file defines the 'String' storage wrapper, which can hold its own | ||
| // unique copy of a string, or merely hold a reference to some point in a | ||
| // source buffer, which is assumed to live at least as long as a value of | ||
| // this type. | ||
| // | ||
| //===----------------------------------------------------------------------===// | ||
| #ifndef SWIFT_BASIC_STRING_H | ||
| #define SWIFT_BASIC_STRING_H | ||
|
|
||
| #include "llvm/ADT/IntrusiveRefCntPtr.h" | ||
| #include "llvm/ADT/StringRef.h" | ||
|
|
||
| using llvm::StringRef; | ||
|
|
||
| namespace swift { | ||
|
|
||
| class String { | ||
| const char *Data; | ||
| size_t Length; | ||
| bool Managed; | ||
|
|
||
|
|
||
| static const char *copyBuffer(const String &Other) { | ||
| auto Buffer = (char *)malloc(Other.str().size()); | ||
| memcpy(Buffer, Other.str().data(), Other.str().size()); | ||
| return Buffer; | ||
| } | ||
|
|
||
| public: | ||
| String() : Data(nullptr), Length(0), Managed(false) {} | ||
|
|
||
| String(const char *Data, size_t Length, bool Managed) | ||
| : Data(Data), Length(Length), Managed(Managed) {} | ||
|
|
||
| String(StringRef Str, bool IsManaged = false) | ||
| : String(Str.data(), Str.size(), IsManaged) {} | ||
|
|
||
| String(const String &Other) | ||
| : Data(Other.Managed ? copyBuffer(Other) : Other.Data), Length(Other.Length), | ||
| Managed(Other.Managed) {} | ||
|
|
||
| static String createManaged(const char *Str, size_t Length) { | ||
| auto Buffer = malloc(Length); | ||
| memcpy(Buffer, Str, Length); | ||
| return String { reinterpret_cast<const char *>(Buffer), Length, | ||
| /* Managed */ true }; | ||
| } | ||
|
|
||
| static String createManaged(StringRef Str) { | ||
| return createManaged(Str.data(), Str.size()); | ||
| } | ||
|
|
||
| static String createUnmanaged(StringRef Str) { | ||
| return String { Str, /* Managed */ false }; | ||
| } | ||
|
|
||
| size_t size() const { | ||
| return Length; | ||
| } | ||
|
|
||
| bool empty() const { | ||
| return Length == 0; | ||
| } | ||
|
|
||
| StringRef str() const { | ||
| return StringRef { Data, Length }; | ||
| } | ||
|
|
||
| bool operator==(const String &Right) const { | ||
| return str() == Right.str(); | ||
| } | ||
|
|
||
| ~String() { | ||
| if (Managed) | ||
| free(reinterpret_cast<void *>(const_cast<char *>(Data))); | ||
| } | ||
| }; | ||
|
|
||
| } // end namespace swift | ||
|
|
||
| #endif // SWIFT_BASIC_STRING_H | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.