forked from j-mie6/parsley
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* Added an generalised form of SoftKeyword to handle both unicode, basic, and not required * Ruled out forward source compatibility issue * Fixed some style issues * Improved test coverage * Removed unused code * Coverage off for NotRequired
- Loading branch information
Showing
12 changed files
with
204 additions
and
73 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
33 changes: 33 additions & 0 deletions
33
...ared/src/main/scala/parsley/internal/deepembedding/singletons/token/SymbolEmbedding.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,33 @@ | ||
| /* SPDX-FileCopyrightText: © 2023 Parsley Contributors <https://github.com/j-mie6/Parsley/graphs/contributors> | ||
| * SPDX-License-Identifier: BSD-3-Clause | ||
| */ | ||
| package parsley.internal.deepembedding.singletons.token | ||
|
|
||
| import parsley.token.errors.LabelConfig | ||
| import parsley.token.predicate.CharPredicate | ||
|
|
||
| import parsley.internal.deepembedding.singletons.Singleton | ||
| import parsley.internal.machine.instructions | ||
|
|
||
| private [parsley] final class SoftKeyword(private [SoftKeyword] val specific: String, letter: CharPredicate, val caseSensitive: Boolean, | ||
| expected: LabelConfig, expectedEnd: String) extends Singleton[Unit] { | ||
| // $COVERAGE-OFF$ | ||
| override def pretty: String = s"softKeyword($specific)" | ||
| // $COVERAGE-ON$ | ||
| override def instr: instructions.Instr = new instructions.token.SoftKeyword(specific, letter, caseSensitive, expected, expectedEnd) | ||
| } | ||
|
|
||
| /* | ||
| private [parsley] final class MaxOp(private [MaxOp] val operator: String, ops: Set[String]) extends Singleton[Unit] { | ||
| // $COVERAGE-OFF$ | ||
| override def pretty: String = s"maxOp($operator)" | ||
| // $COVERAGE-ON$ | ||
| override def instr: instructions.Instr = new instructions.TokenMaxOp(operator, ops) | ||
| } | ||
| */ | ||
|
|
||
| // $COVERAGE-OFF$ | ||
| private [deepembedding] object SoftKeyword { | ||
| def unapply(self: SoftKeyword): Some[String] = Some(self.specific) | ||
| } | ||
| // $COVERAGE-ON$ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
26 changes: 26 additions & 0 deletions
26
...ley/shared/src/main/scala/parsley/internal/machine/instructions/token/CharPredicate.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| /* SPDX-FileCopyrightText: © 2023 Parsley Contributors <https://github.com/j-mie6/Parsley/graphs/contributors> | ||
| * SPDX-License-Identifier: BSD-3-Clause | ||
| */ | ||
| package parsley.internal.machine.instructions.token | ||
|
|
||
| import parsley.internal.machine.Context | ||
|
|
||
| private [parsley] sealed abstract class CharPredicate { | ||
| //def pop(ctx: Context): Boolean | ||
| def peek(ctx: Context): Boolean | ||
| } | ||
| private [parsley] class Basic(f: Char => Boolean) extends CharPredicate { | ||
| def peek(ctx: Context): Boolean = ctx.moreInput && f(ctx.peekChar) | ||
| } | ||
| private [parsley] class Unicode(f: Int => Boolean) extends CharPredicate { | ||
| def peek(ctx: Context): Boolean = { | ||
| lazy val hc = ctx.peekChar(0) | ||
| lazy val l = ctx.peekChar(1) | ||
| ctx.moreInput(2) && hc.isHighSurrogate && Character.isSurrogatePair(hc, l) && f(Character.toCodePoint(hc, l)) || ctx.moreInput && f(hc.toInt) | ||
| } | ||
| } | ||
| private [parsley] object NotRequired extends CharPredicate { | ||
| // $COVERAGE-OFF$ | ||
| def peek(ctx: Context): Boolean = false | ||
| // $COVERAGE-ON$ | ||
| } |
78 changes: 78 additions & 0 deletions
78
parsley/shared/src/main/scala/parsley/internal/machine/instructions/token/SymbolInstrs.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,78 @@ | ||
| /* SPDX-FileCopyrightText: © 2023 Parsley Contributors <https://github.com/j-mie6/Parsley/graphs/contributors> | ||
| * SPDX-License-Identifier: BSD-3-Clause | ||
| */ | ||
| package parsley.internal.machine.instructions.token | ||
|
|
||
| import parsley.token.errors.LabelConfig | ||
| import parsley.token.predicate | ||
|
|
||
| import parsley.internal.errors.ExpectDesc | ||
| import parsley.internal.machine.Context | ||
| import parsley.internal.machine.XAssert._ | ||
| import parsley.internal.machine.instructions.Instr | ||
|
|
||
| private [internal] final class SoftKeyword( | ||
| specific: String, letter: CharPredicate, caseSensitive: Boolean, expected: Option[ExpectDesc], expectedEnd: Option[ExpectDesc]) extends Instr { | ||
| def this(specific: String, letter: predicate.CharPredicate, caseSensitive: Boolean, expected: LabelConfig, expectedEnd: String) = { | ||
| this(if (caseSensitive) specific else specific.toLowerCase, | ||
| letter.asInternalPredicate, | ||
| caseSensitive, | ||
| expected.asExpectDesc, Some(new ExpectDesc(expectedEnd))) | ||
| } | ||
|
|
||
| private [this] final val strsz = specific.length | ||
| private [this] final val numCodePoints = specific.codePointCount(0, strsz) | ||
|
|
||
| final override def apply(ctx: Context): Unit = { | ||
| ensureRegularInstruction(ctx) | ||
| if (ctx.moreInput(strsz)) { | ||
| ctx.saveState() | ||
| readSpecific(ctx, 0) | ||
| } | ||
| else ctx.expectedFail(expected, numCodePoints) | ||
| } | ||
|
|
||
| private def postprocess(ctx: Context): Unit = { | ||
| if (letter.peek(ctx)) { | ||
| ctx.expectedFail(expectedEnd, unexpectedWidth = 1) //This should only report a single token | ||
| ctx.restoreState() | ||
| } | ||
| else { | ||
| ctx.states = ctx.states.tail | ||
| ctx.pushAndContinue(()) | ||
| } | ||
| } | ||
|
|
||
| val readCharCaseHandledBMP = { | ||
| if (caseSensitive) (ctx: Context) => ctx.peekChar | ||
| else (ctx: Context) => ctx.peekChar.toLower | ||
| } | ||
|
|
||
| val readCharCaseHandledSupplementary = { | ||
| if (caseSensitive) (ctx: Context) => Character.toCodePoint(ctx.peekChar(0), ctx.peekChar(1)) | ||
| else (ctx: Context) => Character.toLowerCase(Character.toCodePoint(ctx.peekChar(0), ctx.peekChar(1))) | ||
| } | ||
|
|
||
| final private def readSpecific(ctx: Context, j: Int): Unit = { | ||
| if (j < strsz) { | ||
| val c = specific.codePointAt(j) | ||
| if (Character.isSupplementaryCodePoint(c) && ctx.moreInput(2) && readCharCaseHandledSupplementary(ctx) == c) { | ||
| ctx.fastConsumeSupplementaryChar() | ||
| readSpecific(ctx, j + 2) | ||
| } | ||
| else if (ctx.moreInput && readCharCaseHandledBMP(ctx) == c.toChar) { | ||
| ctx.consumeChar() | ||
| readSpecific(ctx, j + 1) | ||
| } | ||
| else { | ||
| ctx.restoreState() | ||
| ctx.expectedFail(expected, numCodePoints) | ||
| } | ||
| } | ||
| else postprocess(ctx) | ||
| } | ||
|
|
||
| // $COVERAGE-OFF$ | ||
| override def toString: String = s"SoftKeyword($specific)" | ||
| // $COVERAGE-ON$ | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.