Merge branch 'master' of https://github.com/dabeaz/ply

doc/makedoc.py

@@ -25,7 +25,7 @@
 heading = re.compile(r"(_nn\d)", re.IGNORECASE)
 
 def getheadingname(m):
-    autogeneratedheading = True;
+    autogeneratedheading = True
     if m.group(1) != None:
         amatch = alink.match(m.group(1))
         if amatch:
@@ -65,8 +65,9 @@ def getheadingname(m):
 h4 = re.compile(r".*?<H4>(<a.*a>)*[\d\.\s]*(.*?)</H4>", re.IGNORECASE)
 h5 = re.compile(r".*?<H5>(<a.*a>)*[\d\.\s]*(.*?)</H5>", re.IGNORECASE)
 
-data = open(filename).read()            # Read data
-open(filename+".bak","w").write(data)   # Make backup
+# Make backup
+with open(filename) as src, open(filename+".bak","w") as dst:
+    dst.write(src.read())
 
 lines = data.splitlines()
 result = [ ] # This is the result of postprocessing the file
@@ -82,7 +83,7 @@ def getheadingname(m):
             skip = 1
         else:
             skip = 0
-        continue;
+        continue
     if skip:
         continue
 
@@ -186,9 +187,8 @@ def getheadingname(m):
 
 data = "\n".join(result)
 
-data = data.replace("@INDEX@",index) + "\n";
+data = data.replace("@INDEX@",index) + "\n"
 
 # Write the file back out
-open(filename,"w").write(data)
-
-
+with open(filename,"w") as f:
+    f.write(data)

doc/ply.html

@@ -353,7 +353,7 @@ <H3><a name="ply_nn4"></a>4.1 Lex Example</H3>
 
 The <tt>tok.type</tt> and <tt>tok.value</tt> attributes contain the
 type and value of the token itself. 
-<tt>tok.line</tt> and <tt>tok.lexpos</tt> contain information about
+<tt>tok.lineno</tt> and <tt>tok.lexpos</tt> contain information about
 the location of the token.  <tt>tok.lexpos</tt> is the index of the
 token relative to the start of the input text.
 
@@ -3463,7 +3463,7 @@ <H2><a name="ply_nn49"></a>10. Packaging Advice</H2>
 </p>
 
 <p>
-During operation, is is normal for PLY to produce diagnostic error
+During operation, it is normal for PLY to produce diagnostic error
 messages (usually printed to standard error).  These are generated
 entirely using the <tt>logging</tt> module.  If you want to redirect
 these messages or silence them, you can provide your own logging

example/BASIC/basic.py

@@ -15,7 +15,8 @@
 # If a runtime error occurs, we bail out and enter
 # interactive mode below
 if len(sys.argv) == 2:
-    data = open(sys.argv[1]).read()
+    with open(sys.argv[1]) as f:
+        data = f.read()
     prog = basparse.parse(data)
     if not prog:
         raise SystemExit

example/BASIC/basiclog.py

@@ -23,7 +23,8 @@
 # If a runtime error occurs, we bail out and enter
 # interactive mode below
 if len(sys.argv) == 2:
-    data = open(sys.argv[1]).read()
+    with open(sys.argv[1]) as f:
+        data = f.read()
     prog = basparse.parse(data, debug=log)
     if not prog:
         raise SystemExit

ply/cpp.py

@@ -436,7 +436,7 @@ def macro_prescan(self,macro):
     # representing the replacement macro tokens
     # ----------------------------------------------------------------------
 
-    def macro_expand_args(self,macro,args):
+    def macro_expand_args(self,macro,args,expanded):
         # Make a copy of the macro token sequence
         rep = [copy.copy(_x) for _x in macro.value]
 
@@ -460,16 +460,16 @@ def macro_expand_args(self,macro,args):
         # has been sorted in reverse order of patch location since replacements will cause the
         # size of the replacement sequence to expand from the patch point.
 
-        expanded = { }
+        expanded_args = { }
         for ptype, argnum, i in macro.patch:
             # Concatenation.   Argument is left unexpanded
             if ptype == 'c':
                 rep[i:i+1] = args[argnum]
             # Normal expansion.  Argument is macro expanded first
             elif ptype == 'e':
-                if argnum not in expanded:
-                    expanded[argnum] = self.expand_macros(args[argnum])
-                rep[i:i+1] = expanded[argnum]
+                if argnum not in expanded_args:
+                    expanded_args[argnum] = self.expand_macros(args[argnum],expanded)
+                rep[i:i+1] = expanded_args[argnum]
 
         # Get rid of removed comma if necessary
         if comma_patch:
@@ -530,7 +530,7 @@ def expand_macros(self,tokens,expanded=None):
                                         del args[len(m.arglist):]
 
                                 # Get macro replacement text
-                                rep = self.macro_expand_args(m,args)
+                                rep = self.macro_expand_args(m,args,expanded)
                                 rep = self.expand_macros(rep,expanded)
                                 for r in rep:
                                     r.lineno = t.lineno
@@ -777,7 +777,8 @@ def include(self,tokens):
         for p in path:
             iname = os.path.join(p,filename)
             try:
-                data = open(iname,"r").read()
+                with open(iname) as f:
+                    data = f.read()
                 dname = os.path.dirname(iname)
                 if dname:
                     self.temp_path.insert(0,dname)
@@ -903,8 +904,8 @@ def token(self):
 
     # Run a preprocessor
     import sys
-    f = open(sys.argv[1])
-    input = f.read()
+    with open(sys.argv[1]) as f:
+        input = f.read()
 
     p = Preprocessor(lexer)
     p.parse(input,sys.argv[1])

ply/lex.py

@@ -1057,9 +1057,8 @@ def runmain(lexer=None, data=None):
     if not data:
         try:
             filename = sys.argv[1]
-            f = open(filename)
-            data = f.read()
-            f.close()
+            with open(filename) as f:
+                data = f.read()
         except IndexError:
             sys.stdout.write('Reading from standard input (type EOF to end):\n')
             data = sys.stdin.read()

test/testcpp.py

@@ -45,6 +45,22 @@ def __test_preprocessing(self, in_, expected, time_limit = 1.0):
         else:
             self.assertMultiLineEqual(out, expected)
 
+    def test_infinite_argument_expansion(self):
+        # CPP does not drags set of currently expanded macros through macro
+        # arguments expansion. If there is a match between an argument value
+        # and name of an already expanded macro then CPP falls into infinite
+        # recursion.
+        self.__test_preprocessing("""\
+#define a(x) x
+#define b a(b)
+b
+"""         , """\
+
+
+b"""
+        )
+
+
     def test_concatenation(self):
         self.__test_preprocessing("""\
 #define a(x) x##_

test/testlex.py

@@ -479,8 +479,10 @@ def test_lex_optimize3(self):
 
         os.mkdir("lexdir")
         os.mkdir("lexdir/sub")
-        open("lexdir/__init__.py","w").write("")
-        open("lexdir/sub/__init__.py","w").write("")
+        with open("lexdir/__init__.py","w") as f:
+            f.write("")
+        with open("lexdir/sub/__init__.py","w") as f:
+            f.write("")
         run_import("lex_optimize3")
         result = sys.stdout.getvalue()
         self.assert_(check_expected(result,