Refactor text and textwithoutspace rules in grammars hedyorg#1632 (he…

…dyorg#2003)

grammars/level1.lark

@@ -40,7 +40,7 @@ comment: _HASH /([^\n]+)/
 _EOL: "\r"?"\n"
 
 text: /([^\n]+)/ -> text //anything can be parsed except for a newline
-textwithoutspaces: /([^\n *+-\/]+)/ -> text //anything can be parsed except for spaces (plus: a newline)
+textwithoutspaces: /([^\n ]+)/ -> text //anything can be parsed except for a new line and spaces
 
 %import common.SIGNED_INT -> INT
 %import common.NUMBER -> NUMBER

grammars/level12-Additions.lark

@@ -13,5 +13,6 @@ equality_check_equals: (var | text_in_quotes | NUMBER) _EQUALS (var | text_in_qu
 in_list_check: (var | text_in_quotes | NUMBER) _SPACE _IN _SPACE var
 error_invalid: "Supercalifragilisticexpialidocious" //invalid node should be deleted but this probably never matches anything :D
 
-textwithspaces: /(?:[^\n,' ]| (?!else))+/ -> text //anything can be parsed except for a newline and a comma for list separators, and now single quotes
-text_in_quotes: _SINGLE_QUOTE textwithspaces _SINGLE_QUOTE
+// all literal strings have to be quoted now, so arithmetic operators don't need to be excluded anymore
+textwithspaces: /(?:[^\n،,' ]| (?!else))+/ -> text
+text_in_quotes: _SINGLE_QUOTE textwithspaces _SINGLE_QUOTE

grammars/level2-Additions.lark

@@ -2,7 +2,7 @@
 command:+= assign | error_ask_dep_2 | error_echo_dep_2 | sleep -= echo > error_invalid
 print: _PRINT (_SPACE _print_argument)?
 _print_argument: (_SPACE | textwithoutspaces | punctuation)*
-ask: var _SPACE _IS _SPACE _ASK (_SPACE + (_SPACE | text_ask | punctuation)*)?
+ask: var _SPACE _IS _SPACE _ASK (_SPACE + (_SPACE | text | punctuation)*)?
 
 //level 1 deprecated commands, for now manually added for better errors
 error_ask_dep_2: _ASK (_SPACE error_text_dep_2)?
@@ -17,11 +17,10 @@ sleep: _SLEEP (_SPACE INT)?
 turtle: _FORWARD (_SPACE (INT | textwithoutspaces))? -> forward | _TURN (_SPACE (textwithoutspaces))? -> turn
 assign: var _SPACE _IS _SPACE text -> assign
 
-//in Level 2 we need to treat punctuation differently, since it can be used to separate arguments (variables)
-textwithoutspaces: /([^\r\n!?. ]+)/ -> text //anything can be parsed except for spaces (plus: a newline and a comma for list separators)
-text: /([^\r\n,!?،.]+)/ -> text //todo FH, feb 2022 this can go to level 3 cause in (the new) level 2 it is allowed to have commas, lists are now introduced in level 2
-
-text_ask: /([^\r\n!?.]+)/ -> text //as ask may contain commas (punctionation is still needed separately so they can be printed after vars)
+// punctuation (?!.) has to be treated differently since it can be used to separate arguments, e.g.
+// by excluding punctuation symbols from the rules below, we ensure a ? can be printed after a variable
+textwithoutspaces: /([^\n!?. ]+)/ -> text
+text: /([^\n!?.]+)/ -> text
 
 punctuation : PUNCTUATION -> punctuation
 PUNCTUATION: _EXCLAMATION_MARK | _QUESTION_MARK | _PERIOD //uppercase places tokens in tree

grammars/level3-Additions.lark

@@ -1,13 +1,18 @@
 command:+= assign_list | add | remove > error_invalid   
 _print_argument: (_SPACE | list_access | textwithoutspaces | punctuation)*
+ask: var _SPACE _IS _SPACE _ASK (_SPACE + (_SPACE | text_ask | punctuation)*)?
 
 assign: var _SPACE _IS _SPACE (list_access | text) -> assign
 assign_list: var _SPACE _IS _SPACE text_list (_COMMA text_list)+
-text_list: /([^\r\n,،]+)/ -> text // list elements may contain punctuation but not commas or course, these are separators
 
 list_access: var _SPACE _AT _SPACE (INT | random) -> list_access //todo: could be merged with list_access_var?
 turtle: _FORWARD (_SPACE (INT | list_access | textwithoutspaces))? -> forward | _TURN (_SPACE (list_access | textwithoutspaces))? -> turn
 
+// lists are introduced and list separators (comma and arabic comma) have to excluded from text.
+text: /([^\n!?.،,]+)/ -> text
+text_ask: /([^\n!?.]+)/ -> text // the ask command is an exception since it needs to include commas in its value
+text_list: /([^\n,،]+)/ -> text // list elements are another exception since they can contain punctuation but not list separators
+
 // FH, jan 22: not exactly sure why this works, while textwithoutspaces parses the whole line in add/remove
 // leaving this for now
 some_spaces : /([^\r\n!?. ]+)/ (_SPACE /([^\r\n!?. ]+)/)* -> text

grammars/level4-Additions.lark

@@ -1,4 +1,4 @@
-// redifining it entirely since it has many order-depending rules (e.g ask_no_quotes should be after ask and before assign)
+// redefining it entirely since it has many order-depending rules (e.g ask_no_quotes should be after ask and before assign)
 command:print | ask | turtle | assign_list | add | remove | sleep | error_ask_no_quotes| assign | error_print_no_quotes | comment | error_invalid
 
 // we need a separate rule for a var when used in a print argument
@@ -12,12 +12,10 @@ error_ask_no_quotes: var _SPACE _IS _SPACE _ASK _SPACE text  -> error_print_nq
 
 _print_argument: (_SPACE | list_access | quoted_text | var_access)*
 
-//anything can be parsed except for spaces, and a newlines and commas for list separators
-textwithoutspaces: /([^\r\n, *+-\/]+)/ -> text
-
-//anything can be parsed except for a newline and a comma for list separators
-//punctuation does not need to be treated differently anymore
-text: /([^\n,]+)/ -> text
+// literal strings must be single-quoted in ask and print commands so punctuation should not be treated differently anymore
+// anything can be parsed except for a newline, a space and a list separator
+textwithoutspaces: /([^\n،, ]+)/ -> text
+text: /([^\n،,]+)/ -> text
 
 quoted_text_no_escape: /'([^']*)'/ -> text //simply all between quotes should this be used at earlier levels?
 quoted_text: /'((?:[^\\']|\\.)*)'/ -> text //text can be between single quotes, but quotes may be escaped with \

grammars/level6-Additions.lark

@@ -25,4 +25,5 @@ assign_equals: var _EQUALS sum | var _EQUALS textwithoutspaces
 ?atom: INT | var_access | error_unsupported_number //TODO: means we cannot assign strings with spaces? would we want that?
 error_unsupported_number: /([-+]?(\d+[\.,]\d+))/ -> error_unsupported_number
 
-textwithoutspaces: /(?:[^\n, *+\-\/ei]|e(?!lse)|i(?!f))+/ -> text //new cause in level 6 calculation elements need to be escaped too
+// arithmetic operators are added, so they need to be escaped from text
+textwithoutspaces: /(?:[^\n،, *+\-\/ei]|e(?!lse)|i(?!f))+/ -> text