Python3.5: Coroutine code generation progress

* Cleanup function code generation, adding dedicated modules for generators
  and coroutines to easier oversee them.

* Harmonized naming of coroutine nodes and generator nodes.

* Treat coroutines like generators where they should be.

* Extended coroutines with the necessary interfaces.

* Clean up generator returns to be automatically added when the function
  is built.

* The generated code for coroutines is nearly good, but doesn't compile
  in the backend yet.

nuitka/build/include/nuitka/compiled_coroutine.hpp

@@ -23,8 +23,8 @@
 
 #if PYTHON_VERSION >= 350
 
-extern PyObject *Nuitka_Coroutine_New( yielder_func code, PyCellObject **closure, Py_ssize_t closure_given );
-extern PyObject *Nuitka_Coroutine_New( yielder_func code );
+extern PyObject *Nuitka_Coroutine_New( yielder_func code, PyObject *name, PyObject *qualname, PyCodeObject *code_object, PyCellObject **closure, Py_ssize_t closure_given );
+
 
 // The Nuitka_GeneratorObject is the storage associated with a compiled
 // generator object instance of which there can be many for each code.
@@ -71,6 +71,8 @@ typedef struct {
 
 extern PyTypeObject Nuitka_CoroutineWrapper_Type;
 
+PyObject *AWAIT_COROUTINE( PyObject *awaitable );
+
 #endif
 
 #endif

nuitka/build/static_src/CompiledCoroutineType.cpp

@@ -701,3 +701,36 @@ PyTypeObject Nuitka_CoroutineWrapper_Type = {
     0,                                          /* tp_new */
     PyObject_Del,                               /* tp_free */
 };
+
+extern PyObject *PyGen_Send( PyGenObject *gen, PyObject *arg );
+extern PyObject *const_str_plain_send;
+
+PyObject *AWAIT_COROUTINE( PyObject *awaitable )
+{
+    PyObject *coroutine = _PyCoro_GetAwaitableIter( awaitable );
+
+    if (unlikely( coroutine == NULL ))
+    {
+        return NULL;
+    }
+
+    PyObject *retval;
+
+    if ( PyGen_CheckExact( coroutine ) || PyCoro_CheckExact( coroutine ) )
+    {
+        retval = PyGen_Send( (PyGenObject *)coroutine, Py_None );
+    }
+    else if ( Py_TYPE( coroutine )->tp_iternext != NULL )
+    {
+        // TODO: That's probably not allowed anyway.
+        retval = Py_TYPE( coroutine )->tp_iternext( coroutine );
+    }
+    else
+    {
+        retval = PyObject_CallMethodObjArgs( coroutine, const_str_plain_send, Py_None, NULL );
+    }
+
+    Py_DECREF( coroutine );
+
+    return retval;
+}

nuitka/codegen/CodeGeneration.py

@@ -57,6 +57,7 @@
 from .ComparisonCodes import getComparisonExpressionCode
 from .ConditionalCodes import generateConditionCode, getConditionCheckTrueCode
 from .ConstantCodes import generateConstantReferenceCode, getConstantCode
+from .CoroutineCodes import generateAwaitCode, generateMakeCoroutineObjectCode
 from .DictCodes import (
     generateDictionaryCreationCode,
     generateDictOperationUpdateCode,
@@ -85,15 +86,16 @@
     getFrameRestoreExceptionCode
 )
 from .FunctionCodes import (
-    generateCoroutineCreationCode,
     generateFunctionCreationCode,
     generateFunctionDeclCode,
-    generateGeneratorEntryCode,
-    generateMakeGeneratorObjectCode,
     getDirectFunctionCallCode,
     getExportScopeCode,
     getFunctionCode,
-    getFunctionDirectDecl,
+    getFunctionDirectDecl
+)
+from .GeneratorCodes import (
+    generateGeneratorEntryCode,
+    generateMakeGeneratorObjectCode,
     getGeneratorObjectCode
 )
 from .GlobalsLocalsCodes import (
@@ -275,6 +277,11 @@ def generateFunctionBodyCode(function_body, context):
             parent   = context,
             function = function_body
         )
+    elif function_body.isExpressionCoroutineObjectBody():
+        function_context = Contexts.PythonCoroutineObjectContext(
+            parent   = context,
+            function = function_body
+        )
     elif function_body.needsCreation():
         function_context = Contexts.PythonFunctionCreatedContext(
             parent   = context,
@@ -307,6 +314,16 @@ def generateFunctionBodyCode(function_body, context):
             needs_exception_exit   = needs_exception_exit,
             needs_generator_return = function_body.needsGeneratorReturnExit()
         )
+    elif function_body.isExpressionCoroutineObjectBody():
+        function_code = getGeneratorObjectCode(
+            context                = function_context,
+            function_identifier    = function_identifier,
+            user_variables         = function_body.getUserLocalVariables(),
+            temp_variables         = function_body.getTempVariables(),
+            function_codes         = function_codes.codes,
+            needs_exception_exit   = needs_exception_exit,
+            needs_generator_return = function_body.needsGeneratorReturnExit()
+        )
     else:
         parameters = function_body.getParameters()
 
@@ -3095,6 +3112,7 @@ def generateBuiltinHashCode(to_name, expression, emit, context):
         "LIST_OPERATION_POP"        : generateListOperationPopCode,
         "MODULE_FILE_ATTRIBUTE_REF" : generateModuleFileAttributeCode,
         "MAKE_GENERATOR_OBJECT"     : generateMakeGeneratorObjectCode,
+        "MAKE_COROUTINE_OBJECT"     : generateMakeCoroutineObjectCode,
         "OPERATION_BINARY"          : generateOperationBinaryCode,
         "OPERATION_BINARY_INPLACE"  : generateOperationBinaryCode,
         "OPERATION_UNARY"           : generateOperationUnaryCode,
@@ -3107,6 +3125,6 @@ def generateBuiltinHashCode(to_name, expression, emit, context):
         "VARIABLE_REF"              : generateVariableReferenceCode,
         "YIELD"                     : generateYieldCode,
         "YIELD_FROM"                : generateYieldFromCode,
-        "COROUTINE_CREATION"        : generateCoroutineCreationCode
+        "AWAIT"                     : generateAwaitCode,
     }
 )

nuitka/codegen/Contexts.py

@@ -741,8 +741,7 @@ def isForCrossModuleUsage(self):
     def isForCreatedFunction(self):
         return False
 
-
-class PythonFunctionCoroutineContext(PythonFunctionContext):
+class PythonGeneratorObjectContext(PythonFunctionContext):
     def isForDirectCall(self):
         return False
 
@@ -753,15 +752,8 @@ def isForCreatedFunction(self):
         return False
 
 
-class PythonGeneratorObjectContext(PythonFunctionContext):
-    def isForDirectCall(self):
-        return False
-
-    def isForCrossModuleUsage(self):
-        return self.function.isCrossModuleUsed()
-
-    def isForCreatedFunction(self):
-        return False
+class PythonCoroutineObjectContext(PythonGeneratorObjectContext):
+    pass
 
 
 class PythonFunctionCreatedContext(PythonFunctionContext):

nuitka/codegen/CoroutineCodes.py

@@ -0,0 +1,109 @@
+#     Copyright 2015, Kay Hayen, mailto:[email protected]
+#
+#     Part of "Nuitka", an optimizing Python compiler that is compatible and
+#     integrates with CPython, but also works on its own.
+#
+#     Licensed under the Apache License, Version 2.0 (the "License");
+#     you may not use this file except in compliance with the License.
+#     You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#     Unless required by applicable law or agreed to in writing, software
+#     distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.
+#
+""" Code to generate and interact with compiled coroutine objects.
+
+"""
+
+from .Indentation import indented
+from .PythonAPICodes import generateCAPIObjectCode
+from .templates.CodeTemplatesCoroutines import (
+    template_make_coroutine_with_context_template,
+    template_make_coroutine_without_context_template
+)
+from .templates.CodeTemplatesFunction import (
+    template_function_closure_making
+)
+from .VariableCodes import getVariableCode
+
+
+def generateAwaitCode(to_name, expression, emit, context):
+    generateCAPIObjectCode(
+        to_name    = to_name,
+        capi       = "AWAIT_COROUTINE",
+        arg_desc   = (
+            ("await_arg", expression.getValue()),
+        ),
+        may_raise  = expression.mayRaiseException(BaseException),
+        source_ref = expression.getCompatibleSourceReference(),
+        emit       = emit,
+        context    = context
+    )
+
+
+def generateMakeCoroutineObjectCode(to_name, expression, emit, context):
+    coroutine_object_body = expression.getCoroutineRef().getFunctionBody()
+
+    closure_variables = coroutine_object_body.getClosureVariables()
+
+    code_identifier = context.getCodeObjectHandle(
+        code_object  = expression.getCodeObject(),
+        filename     = coroutine_object_body.getParentModule().getRunTimeFilename(),
+        line_number  = coroutine_object_body.getSourceReference().getLineNumber(),
+        is_optimized = True,
+        new_locals   = not coroutine_object_body.needsLocalsDict(),
+        has_closure  = len(closure_variables) > 0,
+        future_flags = coroutine_object_body.getSourceReference().getFutureSpec().asFlags()
+    )
+
+    if closure_variables:
+        # TODO: Copy duplication with generator codes, ought to be shared.
+        closure_copy = []
+
+        for count, variable in enumerate(closure_variables):
+            variable_code = getVariableCode(
+                context  = context,
+                variable = variable
+            )
+
+            # Generators might not use them, but they still need to be put there.
+            # TODO: But they don't have to be cells.
+            if not variable.isSharedTechnically():
+                variable_code = "PyCell_NEW0( %s )" % variable_code
+
+            closure_copy.append(
+                "closure[%d] = %s;" % (
+                    count,
+                    variable_code
+                )
+            )
+            closure_copy.append(
+                "Py_INCREF( closure[%d] );" % count
+            )
+
+        closure_making = template_function_closure_making % {
+            "closure_copy"  : indented(closure_copy),
+            "closure_count" : len(closure_variables)
+        }
+
+        emit(
+            template_make_coroutine_with_context_template % {
+                "closure_making"       : closure_making,
+                "coroutine_identifier" : coroutine_object_body.getCodeName(),
+                "to_name"              : to_name,
+                "code_identifier"      : code_identifier,
+                "closure_count"        : len(closure_variables)
+            }
+        )
+    else:
+        emit(
+            template_make_coroutine_without_context_template % {
+                "coroutine_identifier" : coroutine_object_body.getCodeName(),
+                "to_name"              : to_name,
+                "code_identifier"      : code_identifier,
+            }
+        )

nuitka/codegen/FrameCodes.py

@@ -57,7 +57,8 @@ def generateStatementsFrameCode(statement_sequence, emit, context):
     old_frame_handle = context.getFrameHandle()
 
     if guard_mode != "pass_through":
-        if provider.isExpressionGeneratorObjectBody():
+        if provider.isExpressionGeneratorObjectBody() or \
+           provider.isExpressionCoroutineObjectBody():
             context.setFrameHandle("generator->m_frame")
         elif provider.isExpressionFunctionBody():
             context.setFrameHandle("frame_function")
@@ -104,7 +105,7 @@ def generateStatementsFrameCode(statement_sequence, emit, context):
         frame_return_exit = None
 
     if guard_mode == "generator":
-        assert provider.isExpressionGeneratorObjectBody()
+        assert provider.isExpressionGeneratorObjectBody() or provider.isExpressionCoroutineObjectBody()
 
         # TODO: This case should care about "needs_preserve", as for
         # Python3 it is actually not a stub of empty code.