diff --git a/editor/grammar.ne b/editor/grammar.ne index 0d02507..eab8689 100644 --- a/editor/grammar.ne +++ b/editor/grammar.ne @@ -3,6 +3,9 @@ const moo = require('moo') +const backslashes = s => s.replace(/\\["\\]/g, x => x[1]) + + let lexer = moo.compile([ {name: 'NL', match: '\n', lineBreaks: true }, {name: 'WS', match: /[ \t]+/}, @@ -119,7 +122,7 @@ var negateNumber = factory(function (a, _, n) { }) var string = factory(function decodeString(...d) { - return d[0].value + return backslashes(d[0].value) }, function encodeString(d) { if (typeof d === 'string') { return d diff --git a/test/grammar.test.js b/test/grammar.test.js index f303d43..f53724f 100644 --- a/test/grammar.test.js +++ b/test/grammar.test.js @@ -27,6 +27,54 @@ function parseBlock(source) { } +describe('tokenize', () => { + + const lexer = grammar.lexer + const source = ` + when flag clicked + say "Hello \\"world\\"!" + forever { + move 10 steps + }` + + // TODO move backslash-escapes into tokenizer? + + test('file', () => { + lexer.reset(source) + expect(Array.from(lexer).map(t => `${t.type} ${t.value}`)).toEqual([ + "NL \n", + "WS ", + "symbol when", + "WS ", + "symbol flag", + "WS ", + "symbol clicked", + "NL \n", + "WS ", + "symbol say", + "WS ", + 'string Hello \\"world\\"!', + "NL \n", + "WS ", + "symbol forever", + "WS ", + "{ {", + "NL \n", + "WS ", + "symbol move", + "WS ", + "number 10", + "WS ", + "symbol steps", + "NL \n", + "WS ", + "} }", + ]) + }) + +}) + + describe('parse', () => { test('line', () => { @@ -72,6 +120,9 @@ describe('parse', () => { expect(parseBlock('forever {\n }')).toEqual(['doForever', null]) }) + test('backslash-escapes', () => { + expect(parseBlock('say "Hello \\"world\\"!"')).toEqual(['say:', 'Hello "world"!']) + }) }) diff --git a/test/mode.test.js b/test/mode.test.js index 3eec30b..a161e31 100644 --- a/test/mode.test.js +++ b/test/mode.test.js @@ -210,5 +210,8 @@ describe('highlight', () => { MT('closed string', '[s-looks say] [string "foo "] ') + MT('backslash-escaped strings', + '[s-looks say] [string "potato \\"waffles\\"."] ') + })