(PUP-2288) Make [ be LISTSTART if first in file or preceded by WS

The problem was that there was only one special case being handled
by the lexer to dis-ambiguate if an expression followed by [ meant
the start of an access operations (e.g. getting an entry out of an
array), or if it represented the start of a literal array.

The supported special case was NAME followed by non-WS followed by [.

This meant that cases like foo(3) [a,b,c] were always treated as
an access operation. 

The fix generalizes the dis-ambiguation so that all access operations
must be written without interleaved WS.

lib/puppet/pops/parser/lexer2.rb

@@ -323,7 +323,7 @@ def lex_token
       emit(TOKEN_COMMA, before)
 
     when '['
-      if ctx[:after] == :NAME && (before == 0 || scn.string[before-1,1] =~ /[[:blank:]\r\n]+/)
+      if (before == 0 || scn.string[before-1,1] =~ /[[:blank:]\r\n]+/)
         emit(TOKEN_LISTSTART, before)
       else
         emit(TOKEN_LBRACK, before)

spec/unit/pops/parser/lexer2_spec.rb

@@ -21,7 +21,7 @@ def epp_tokens_scanned_from(s)
   include EgrammarLexer2Spec
 
   {
-    :LBRACK => '[',
+    :LISTSTART => '[',
     :RBRACK => ']',
     :LBRACE => '{',
     :RBRACE => '}',
@@ -69,6 +69,10 @@ def epp_tokens_scanned_from(s)
     end
   end
 
+  it "should lex [ in position after non whitespace as LBRACK" do
+    tokens_scanned_from("a[").should match_tokens2(:NAME, :LBRACK)
+  end
+
   {
     "case"     => :CASE,
     "class"    => :CLASS,
@@ -221,7 +225,7 @@ def epp_tokens_scanned_from(s)
 
   it "differentiates between foo[x] and foo [x] (whitespace)" do
     tokens_scanned_from("$a[1]").should match_tokens2(:VARIABLE, :LBRACK, :NUMBER, :RBRACK)
-    tokens_scanned_from("$a [1]").should match_tokens2(:VARIABLE, :LBRACK, :NUMBER, :RBRACK)
+    tokens_scanned_from("$a [1]").should match_tokens2(:VARIABLE, :LISTSTART, :NUMBER, :RBRACK)
     tokens_scanned_from("a[1]").should match_tokens2(:NAME, :LBRACK, :NUMBER, :RBRACK)
     tokens_scanned_from("a [1]").should match_tokens2(:NAME, :LISTSTART, :NUMBER, :RBRACK)
     tokens_scanned_from(" if \n\r\t\nif if ").should match_tokens2(:IF, :IF, :IF)
@@ -249,15 +253,18 @@ def epp_tokens_scanned_from(s)
     "!~" => [:NOMATCH, "!~ /./"],
     ","  => [:COMMA, ", /./"],
     "("  => [:LPAREN, "( /./"],
-    "["  => [:LBRACK, "[ /./"],
+    "["  => [:LISTSTART, "[ /./"],
+    "["  => [[:NAME, :LBRACK], "a[ /./"],
+    "["  => [[:NAME, :LISTSTART], "a [ /./"],
     "{"  => [:LBRACE, "{ /./"],
     "+"  => [:PLUS, "+ /./"],
     "-"  => [:MINUS, "- /./"],
     "*"  => [:TIMES, "* /./"],
     ";"  => [:SEMIC, "; /./"],
   }.each do |token, entry|
     it "should lex regexp after '#{token}'" do
-      tokens_scanned_from(entry[1]).should match_tokens2(entry[0], :REGEX)
+      expected = [entry[0], :REGEX].flatten
+      tokens_scanned_from(entry[1]).should match_tokens2(*expected)
     end
   end