from __future__ import with_statement addon for 'with', mostly writte…

…n by

Neal.

Include/code.h

@@ -46,6 +46,12 @@ typedef struct {
 #endif
 #define CO_FUTURE_DIVISION    	0x2000
 #define CO_FUTURE_ABSIMPORT	0x4000 /* absolute import by default */
+#define CO_FUTURE_WITH_STATEMENT  0x8000
+
+/* This should be defined if a future statement modifies the syntax.
+   For example, when a keyword is added.
+*/
+#define PY_PARSER_REQUIRES_FUTURE_KEYWORD
 
 #define CO_MAXBLOCKS 20 /* Max static block nesting within a function */
 

Include/compile.h

@@ -23,6 +23,7 @@ typedef struct {
 #define FUTURE_GENERATORS "generators"
 #define FUTURE_DIVISION "division"
 #define FUTURE_ABSIMPORT "absolute_import"
+#define FUTURE_WITH_STATEMENT "with_statement"
 
 struct _mod; /* Declare the existence of this type */
 PyAPI_FUNC(PyCodeObject *) PyAST_Compile(struct _mod *, const char *,

Include/modsupport.h

@@ -39,8 +39,8 @@ PyAPI_FUNC(int) PyModule_AddIntConstant(PyObject *, const char *, long);
 PyAPI_FUNC(int) PyModule_AddStringConstant(PyObject *, const char *, const char *);
 
 
-#define PYTHON_API_VERSION 1012
-#define PYTHON_API_STRING "1012"
+#define PYTHON_API_VERSION 1013
+#define PYTHON_API_STRING "1013"
 /* The API version is maintained (independently from the Python version)
    so we can detect mismatches between the interpreter and dynamically
    loaded modules.  These are diagnosed by an error message but
@@ -54,6 +54,8 @@ PyAPI_FUNC(int) PyModule_AddStringConstant(PyObject *, const char *, const char
    Please add a line or two to the top of this log for each API
    version change:
 
+   22-Feb-2006  GvR	1013	PEP 353 - long indices for sequence lengths
+
    19-Aug-2002  GvR	1012	Changes to string object struct for
    				interning changes, saving 3 bytes.
 

Include/parsetok.h

@@ -23,6 +23,8 @@ typedef struct {
 
 #define PyPARSE_DONT_IMPLY_DEDENT	0x0002
 
+#define PyPARSE_WITH_IS_KEYWORD		0x0003
+
 PyAPI_FUNC(node *) PyParser_ParseString(const char *, grammar *, int,
                                               perrdetail *);
 PyAPI_FUNC(node *) PyParser_ParseFile (FILE *, const char *, grammar *, int,

Include/pythonrun.h

@@ -7,7 +7,8 @@
 extern "C" {
 #endif
 
-#define PyCF_MASK (CO_FUTURE_DIVISION | CO_FUTURE_ABSIMPORT)
+#define PyCF_MASK (CO_FUTURE_DIVISION | CO_FUTURE_ABSIMPORT | \
+                   CO_FUTURE_WITH_STATEMENT)
 #define PyCF_MASK_OBSOLETE (CO_NESTED)
 #define PyCF_SOURCE_IS_UTF8  0x0100
 #define PyCF_DONT_IMPLY_DEDENT 0x0200

Lib/__future__.py

@@ -52,6 +52,7 @@
     "generators",
     "division",
     "absolute_import",
+    "with_statement",
 ]
 
 __all__ = ["all_feature_names"] + all_feature_names
@@ -64,6 +65,7 @@
 CO_GENERATOR_ALLOWED = 0        # generators (obsolete, was 0x1000)
 CO_FUTURE_DIVISION   = 0x2000   # division
 CO_FUTURE_ABSIMPORT  = 0x4000   # absolute_import
+CO_FUTURE_WITH_STATEMENT  = 0x8000   # with statement added in 2.5
 
 class _Feature:
     def __init__(self, optionalRelease, mandatoryRelease, compiler_flag):
@@ -108,3 +110,7 @@ def __repr__(self):
 absolute_import = _Feature((2, 5, 0, "alpha", 1),
                            (2, 7, 0, "alpha", 0),
                            CO_FUTURE_ABSIMPORT)
+
+with_statement = _Feature((2, 5, 0, "alpha", 2),
+                          (2, 6, 0, "alpha", 0),
+                          CO_FUTURE_WITH_STATEMENT)

Lib/compiler/future.py

@@ -15,7 +15,8 @@ def is_future(stmt):
 
 class FutureParser:
 
-    features = ("nested_scopes", "generators", "division")
+    features = ("nested_scopes", "generators", "division",
+                "absolute_import", "with_statement")
 
     def __init__(self):
         self.found = {} # set

Lib/test/test_with.py

@@ -2,6 +2,8 @@
 
 """Unit tests for the with statement specified in PEP 343."""
 
+from __future__ import with_statement
+
 __author__ = "Mike Bland"
 __email__ = "mbland at acm dot org"
 

Misc/NEWS

@@ -32,7 +32,8 @@ Core and builtins
   - dict.__getitem__ now looks for a __missing__ hook before raising
     KeyError.
 
-- PEP 343: with statement implemented.
+- PEP 343: with statement implemented. Needs 'from __future__ import
+  with_statement'. Use of 'with' as a variable will generate a warning.
 
 - Fix the encodings package codec search function to only search
   inside its own package. Fixes problem reported in patch #1433198.

Parser/parser.c

@@ -79,8 +79,8 @@ PyParser_New(grammar *g, int start)
 	if (ps == NULL)
 		return NULL;
 	ps->p_grammar = g;
-#if 0 /* future keyword */
-	ps->p_generators = 0;
+#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
+	ps->p_flags = 0;
 #endif
 	ps->p_tree = PyNode_New(start);
 	if (ps->p_tree == NULL) {
@@ -147,10 +147,10 @@ classify(parser_state *ps, int type, char *str)
 			if (l->lb_type == NAME && l->lb_str != NULL &&
 					l->lb_str[0] == s[0] &&
 					strcmp(l->lb_str, s) == 0) {
-#if 0 /* future keyword */
-				if (!ps->p_generators &&
-				    s[0] == 'y' &&
-				    strcmp(s, "yield") == 0)
+#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
+				if (!(ps->p_flags & CO_FUTURE_WITH_STATEMENT) &&
+				    s[0] == 'w' &&
+				    strcmp(s, "with") == 0)
 					break; /* not a keyword */
 #endif
 				D(printf("It's a keyword\n"));
@@ -174,24 +174,35 @@ classify(parser_state *ps, int type, char *str)
 	return -1;
 }
 
-#if 0 /* future keyword */
+#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
 static void
 future_hack(parser_state *ps)
 {
 	node *n = ps->p_stack.s_top->s_parent;
 	node *ch;
 	int i;
 
-	if (strcmp(STR(CHILD(n, 0)), "from") != 0)
+	/* from __future__ import ..., must have at least 4 children */
+	n = CHILD(n, 0);
+	if (NCH(n) < 4)
+		return;
+	ch = CHILD(n, 0);
+	if (STR(ch) == NULL || strcmp(STR(ch), "from") != 0)
 		return;
 	ch = CHILD(n, 1);
-	if (strcmp(STR(CHILD(ch, 0)), "__future__") != 0)
+	if (NCH(ch) == 1 && STR(CHILD(ch, 0)) &&
+	    strcmp(STR(CHILD(ch, 0)), "__future__") != 0)
 		return;
 	for (i = 3; i < NCH(n); i += 2) {
+		/* XXX: assume we don't have parentheses in import:
+		        from __future__ import (x, y, z)
+		*/
 		ch = CHILD(n, i);
+		if (NCH(ch) == 1)
+			ch = CHILD(ch, 0);
 		if (NCH(ch) >= 1 && TYPE(CHILD(ch, 0)) == NAME &&
-		    strcmp(STR(CHILD(ch, 0)), "generators") == 0) {
-			ps->p_generators = 1;
+		    strcmp(STR(CHILD(ch, 0)), "with_statement") == 0) {
+			ps->p_flags |= CO_FUTURE_WITH_STATEMENT;
 			break;
 		}
 	}
@@ -255,7 +266,7 @@ PyParser_AddToken(register parser_state *ps, register int type, char *str,
 						 "Direct pop.\n",
 						 d->d_name,
 						 ps->p_stack.s_top->s_state));
-#if 0 /* future keyword */
+#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
 					if (d->d_name[0] == 'i' &&
 					    strcmp(d->d_name,
 						   "import_stmt") == 0)
@@ -273,7 +284,7 @@ PyParser_AddToken(register parser_state *ps, register int type, char *str,
 		}
 
 		if (s->s_accept) {
-#if 0 /* future keyword */
+#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
 			if (d->d_name[0] == 'i' &&
 			    strcmp(d->d_name, "import_stmt") == 0)
 				future_hack(ps);

Parser/parser.h

@@ -25,8 +25,8 @@ typedef struct {
 	stack	 	p_stack;	/* Stack of parser states */
 	grammar		*p_grammar;	/* Grammar to use */
 	node		*p_tree;	/* Top of parse tree */
-#if 0 /* future keyword */
-	int		p_generators;	/* 1 if yield is a keyword */
+#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
+	unsigned long	p_flags;	/* see co_flags in Include/code.h */
 #endif
 } parser_state;
 

Parser/parsetok.c

@@ -92,28 +92,37 @@ PyParser_ParseFileFlags(FILE *fp, const char *filename, grammar *g, int start,
 /* Parse input coming from the given tokenizer structure.
    Return error code. */
 
-#if 0 /* future keyword */
-static char yield_msg[] =
-"%s:%d: Warning: 'yield' will become a reserved keyword in the future\n";
-#endif
+static char with_msg[] =
+"%s:%d: Warning: 'with' will become a reserved keyword in Python 2.6\n";
+
+static char as_msg[] =
+"%s:%d: Warning: 'as' will become a reserved keyword in Python 2.6\n";
+
+static void
+warn(const char *msg, const char *filename, int lineno)
+{
+	if (filename == NULL)
+		filename = "<string>";
+	PySys_WriteStderr(msg, filename, lineno);
+}
 
 static node *
 parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
 	 int flags)
 {
 	parser_state *ps;
 	node *n;
-	int started = 0;
+	int started = 0, handling_import = 0, handling_with = 0;
 
 	if ((ps = PyParser_New(g, start)) == NULL) {
 		fprintf(stderr, "no mem for new parser\n");
 		err_ret->error = E_NOMEM;
 		PyTokenizer_Free(tok);
 		return NULL;
 	}
-#if 0 /* future keyword */
-	if (flags & PyPARSE_YIELD_IS_KEYWORD)
-		ps->p_generators = 1;
+#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
+	if (flags & PyPARSE_WITH_IS_KEYWORD)
+		ps->p_flags |= CO_FUTURE_WITH_STATEMENT;
 #endif
 
 	for (;;) {
@@ -129,6 +138,7 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
 		}
 		if (type == ENDMARKER && started) {
 			type = NEWLINE; /* Add an extra newline */
+			handling_with = handling_import = 0;
 			started = 0;
 			/* Add the right number of dedent tokens,
 			   except if a certain flag is given --
@@ -153,14 +163,27 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
 			strncpy(str, a, len);
 		str[len] = '\0';
 
-#if 0 /* future keyword */
-		/* Warn about yield as NAME */
-		if (type == NAME && !ps->p_generators &&
-		    len == 5 && str[0] == 'y' && strcmp(str, "yield") == 0)
-			PySys_WriteStderr(yield_msg,
-					  err_ret->filename==NULL ?
-					  "<string>" : err_ret->filename,
-					  tok->lineno);
+#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
+		/* This is only necessary to support the "as" warning, but
+		   we don't want to warn about "as" in import statements. */
+		if (type == NAME &&
+		    len == 6 && str[0] == 'i' && strcmp(str, "import") == 0)
+			handling_import = 1;
+
+		/* Warn about with as NAME */
+		if (type == NAME &&
+		    !(ps->p_flags & CO_FUTURE_WITH_STATEMENT)) {
+		    if (len == 4 && str[0] == 'w' && strcmp(str, "with") == 0)
+			warn(with_msg, err_ret->filename, tok->lineno);
+		    else if (!(handling_import || handling_with) &&
+			     len == 2 &&
+			     str[0] == 'a' && strcmp(str, "as") == 0)
+			warn(as_msg, err_ret->filename, tok->lineno);
+		}
+		else if (type == NAME &&
+			 (ps->p_flags & CO_FUTURE_WITH_STATEMENT) &&
+			 len == 4 && str[0] == 'w' && strcmp(str, "with") == 0)
+			handling_with = 1;
 #endif
 
 		if ((err_ret->error =

Python/Python-ast.c

@@ -371,7 +371,7 @@ static PyTypeObject* make_type(char *type, PyTypeObject* base, char**fields, int
         }
         PyTuple_SET_ITEM(fnames, i, field);
     }
-    result = PyObject_CallFunction((PyObject*)&PyType_Type, "s(O){sOss}", 
+    result = PyObject_CallFunction((PyObject*)&PyType_Type, "s(O){sOss}",
                     type, base, "_fields", fnames, "__module__", "_ast");
     Py_DECREF(fnames);
     return (PyTypeObject*)result;
@@ -2956,7 +2956,7 @@ init_ast(void)
         if (PyDict_SetItemString(d, "AST", (PyObject*)AST_type) < 0) return;
         if (PyModule_AddIntConstant(m, "PyCF_ONLY_AST", PyCF_ONLY_AST) < 0)
                 return;
-        if (PyModule_AddStringConstant(m, "__version__", "42635") < 0)
+        if (PyModule_AddStringConstant(m, "__version__", "42649") < 0)
                 return;
         if(PyDict_SetItemString(d, "mod", (PyObject*)mod_type) < 0) return;
         if(PyDict_SetItemString(d, "Module", (PyObject*)Module_type) < 0)

Python/compile.c

@@ -4286,6 +4286,8 @@ compute_code_flags(struct compiler *c)
 		flags |= CO_GENERATOR;
         if (c->c_flags->cf_flags & CO_FUTURE_DIVISION)
                 flags |= CO_FUTURE_DIVISION;
+        if (c->c_flags->cf_flags & CO_FUTURE_WITH_STATEMENT)
+                flags |= CO_FUTURE_WITH_STATEMENT;
 	n = PyDict_Size(c->u->u_freevars);
 	if (n < 0)
 	    return -1;

Python/future.c

@@ -31,6 +31,8 @@ future_check_features(PyFutureFeatures *ff, stmt_ty s, const char *filename)
 			ff->ff_features |= CO_FUTURE_DIVISION;
 		} else if (strcmp(feature, FUTURE_ABSIMPORT) == 0) {
 			ff->ff_features |= CO_FUTURE_ABSIMPORT;
+		} else if (strcmp(feature, FUTURE_WITH_STATEMENT) == 0) {
+			ff->ff_features |= CO_FUTURE_WITH_STATEMENT;
 		} else if (strcmp(feature, "braces") == 0) {
 			PyErr_SetString(PyExc_SyntaxError,
 					"not a chance");

Python/pythonrun.c

@@ -690,8 +690,10 @@ PyRun_InteractiveLoopFlags(FILE *fp, const char *filename, PyCompilerFlags *flag
 
 /* compute parser flags based on compiler flags */
 #define PARSER_FLAGS(flags) \
-	(((flags) && (flags)->cf_flags & PyCF_DONT_IMPLY_DEDENT) ? \
-		PyPARSE_DONT_IMPLY_DEDENT : 0)
+	((flags) ? ((((flags)->cf_flags & PyCF_DONT_IMPLY_DEDENT) ? \
+		      PyPARSE_DONT_IMPLY_DEDENT : 0) \
+	            | ((flags)->cf_flags & CO_FUTURE_WITH_STATEMENT ? \
+	               PyPARSE_WITH_IS_KEYWORD : 0)) : 0)
 
 int
 PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags)