Merge pull request apache#7675 from markflyhigh/py3-tox-2

[BEAM-6527] Use Gradle to parallel Python tox tests

build.gradle

@@ -191,7 +191,10 @@ task goIntegrationTests() {
 }
 
 task pythonPreCommit() {
-  dependsOn ":beam-sdks-python:preCommit"
+  dependsOn ":beam-sdks-python:preCommitPy2"
+  dependsOn ":beam-sdks-python-test-suites-tox-py35:preCommitPy35"
+  dependsOn ":beam-sdks-python-test-suites-tox-py36:preCommitPy36"
+  dependsOn ":beam-sdks-python-test-suites-dataflow:preCommitIT"
 }
 
 task pythonPostCommit() {

buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy

@@ -1611,16 +1611,38 @@ class BeamModulePlugin implements Plugin<Project> {
         outputs.dirs(project.ext.envdir)
       }
 
+      def pythonSdkDeps = project.files(
+              project.fileTree(
+              dir: "${project.rootDir}",
+              include: ['model/**', 'sdks/python/**'],
+              // Exclude temporary directories used in build and test.
+              exclude: [
+                'sdks/python/build/**',
+                'sdks/python/dist/**',
+                'sdks/python/target/**',
+                'sdks/python/test-suites/**',
+              ])
+              )
+      def copiedSrcRoot = "${project.buildDir}/srcs"
+
       project.configurations { distConfig }
 
       project.task('sdist', dependsOn: 'setupVirtualenv') {
         doLast {
+          // Copy sdk sources to an isolated directory
+          project.copy {
+            from pythonSdkDeps
+            into copiedSrcRoot
+          }
+
+          // Build artifact
           project.exec {
             executable 'sh'
-            args '-c', ". ${project.ext.envdir}/bin/activate && cd ${pythonRootDir} && python setup.py sdist --keep-temp --formats zip,gztar --dist-dir ${project.buildDir}"
+            args '-c', ". ${project.ext.envdir}/bin/activate && cd ${copiedSrcRoot}/sdks/python && python setup.py sdist --formats zip,gztar --dist-dir ${project.buildDir}"
           }
           def collection = project.fileTree("${project.buildDir}"){ include '**/*.tar.gz' exclude '**/apache-beam.tar.gz'}
           println "sdist archive name: ${collection.singleFile}"
+
           // we need a fixed name for the artifact
           project.copy { from collection.singleFile; into "${project.buildDir}"; rename { 'apache-beam.tar.gz' } }
         }
@@ -1646,7 +1668,7 @@ class BeamModulePlugin implements Plugin<Project> {
           project.exec {
             executable 'sh'
             args '-c', "if [ -e ${activate} ]; then " +
-                    ". ${activate} && python ${pythonRootDir}/setup.py clean; " +
+                    ". ${activate} && cd ${pythonRootDir} && python setup.py clean; " +
                     "fi"
           }
           project.delete project.buildDir     // Gradle build directory
@@ -1673,6 +1695,21 @@ class BeamModulePlugin implements Plugin<Project> {
         }
         return argList.join(' ')
       }
+
+      project.ext.toxTask = { name, tox_env ->
+        project.tasks.create(name) {
+          dependsOn = ['sdist']
+          doLast {
+            def copiedPyRoot = "${copiedSrcRoot}/sdks/python"
+            project.exec {
+              executable 'sh'
+              args '-c', ". ${project.ext.envdir}/bin/activate && cd ${copiedPyRoot} && scripts/run_tox.sh $tox_env ${project.buildDir}/apache-beam.tar.gz"
+            }
+          }
+          inputs.files pythonSdkDeps
+          outputs.files project.fileTree(dir: "${pythonRootDir}/target/.tox/${tox_env}/log/")
+        }
+      }
     }
   }
 }

sdks/python/build.gradle

@@ -38,34 +38,8 @@ build.dependsOn buildPython
 
 
 /*************************************************************************************************/
-// Unit testing
-
-def pythonSdkDeps = files(
-    fileTree(dir: 'apache_beam', includes: ['**/*.py', '**/*.pyx', '**/*.pxd']),
-    fileTree(dir: 'apache_beam/testing/data'),
-    fileTree(dir: "${project.rootDir}/model"),
-    fileTree(dir: 'scripts'),
-    ".pylintrc",
-    "MANIFEST.in",
-    "gen_protos.py",
-    "setup.cfg",
-    "setup.py",
-    "test_config.py",
-    "tox.ini")
-
-def toxTask = {
-  name, tox_env -> tasks.create(name) {
-    dependsOn = ['setupVirtualenv']
-    doLast {
-      exec {
-        executable 'sh'
-        args '-c', ". ${project.ext.envdir}/bin/activate && ./scripts/run_tox.sh $tox_env"
-      }
-    }
-    inputs.files pythonSdkDeps
-    outputs.files fileTree(dir: "${project.rootDir}/sdks/python/target/.tox/${tox_env}/log/")
-  }
-}
+// Unit tests for Python 2
+// See Python 3 tests in test-suites/tox
 
 task lint {}
 check.dependsOn lint
@@ -76,54 +50,30 @@ lint.dependsOn lintPy27
 toxTask "lintPy27_3", "py27-lint3"
 lint.dependsOn lintPy27_3
 
-toxTask "lintPy35", "py35-lint"
-lint.dependsOn lintPy35
-
 toxTask "testPy2Gcp", "py27-gcp"
 test.dependsOn testPy2Gcp
 
-toxTask "testPy35Gcp", "py35-gcp"
-test.dependsOn testPy35Gcp
-
 toxTask "testPython2", "py27"
 test.dependsOn testPython2
 
-toxTask "testPython35", "py35"
-test.dependsOn testPython35
-
-toxTask "testPython36", "py36"
-test.dependsOn testPython36
-
 toxTask "testPy2Cython", "py27-cython"
 test.dependsOn testPy2Cython
 // Ensure that testPy2Cython runs exclusively to other tests. This line is not
 // actually required, since gradle doesn't do parallel execution within a
 // project.
 testPy2Cython.mustRunAfter testPython2, testPy2Gcp
 
-toxTask "testPy35Cython", "py35-cython"
-test.dependsOn testPy35Cython
-// Ensure that testPy35Cython runs exclusively to other tests. This line is not
-// actually required, since gradle doesn't do parallel execution within a
-// project.
-testPy35Cython.mustRunAfter testPython35, testPy35Gcp
-
 toxTask "docs", "docs"
 assemble.dependsOn docs
 
 toxTask "cover", "cover"
 
-task preCommit() {
+task preCommitPy2() {
   dependsOn "docs"
   dependsOn "testPy2Cython"
-  dependsOn "testPy35Cython"
   dependsOn "testPython2"
-  dependsOn "testPython35"
-  dependsOn "testPython36"
   dependsOn "testPy2Gcp"
-  dependsOn "testPy35Gcp"
   dependsOn "lint"
-  dependsOn ":beam-sdks-python-test-suites-dataflow:preCommitIT"
 }
 
 task portablePreCommit() {

sdks/python/scripts/generate_pydoc.sh

@@ -29,7 +29,7 @@ if [[ $PWD != *sdks/python* ]]; then
 fi
 
 # Go to the Apache Beam Python SDK root
-if [[ "*sdks/python" != $PWD ]]; then
+if [[ $PWD != *sdks/python ]]; then
   cd $(pwd | sed 's/sdks\/python.*/sdks\/python/')
 fi
 

sdks/python/scripts/run_integration_test.sh

@@ -163,7 +163,7 @@ if [[ -z $PIPELINE_OPTS ]]; then
   fi
 
   # Go to the Apache Beam Python SDK root
-  if [[ "*sdks/python" != $PWD ]]; then
+  if [[ $PWD != *sdks/python ]]; then
     cd $(pwd | sed 's/sdks\/python.*/sdks\/python/')
   fi
 

sdks/python/scripts/run_mini_py3lint.sh

@@ -29,7 +29,7 @@ if [[ $PWD != *sdks/python* ]]; then
 fi
 
 # Go to the Apache Beam Python SDK root
-if [[ "*sdks/python" != $PWD ]]; then
+if [[ $PWD != *sdks/python ]]; then
   cd $(pwd | sed 's/sdks\/python.*/sdks\/python/')
 fi
 

sdks/python/scripts/run_pylint.sh

@@ -30,7 +30,7 @@ if [[ $PWD != *sdks/python* ]]; then
 fi
 
 # Go to the Apache Beam Python SDK root
-if [[ "*sdks/python" != $PWD ]]; then
+if [[ $PWD != *sdks/python ]]; then
   cd $(pwd | sed 's/sdks\/python.*/sdks\/python/')
 fi
 

sdks/python/scripts/run_pylint_2to3.sh

@@ -29,7 +29,7 @@ if [[ $PWD != *sdks/python* ]]; then
 fi
 
 # Go to the Apache Beam Python SDK root
-if [[ "*sdks/python" != $PWD ]]; then
+if [[ $PWD != *sdks/python ]]; then
   cd $(pwd | sed 's/sdks\/python.*/sdks\/python/')
 fi
 

sdks/python/scripts/run_tox.sh

@@ -24,9 +24,10 @@
 
 ###########################################################################
 # Usage check.
-if [[ $# != 1 ]]; then
-  printf "Usage: \n$> ./scripts/run_tox.sh <tox_environment>"
+if [[ $# < 1 || $# > 2 ]]; then
+  printf "Usage: \n$> ./scripts/run_tox.sh <tox_environment> [<sdk_location>]"
   printf "\n\ttox_environment: [required] Tox environment to run the test in.\n"
+  printf "\n\tsdk_location: [optional] SDK tarball artifact location.\n"
   exit 1
 fi
 
@@ -37,11 +38,19 @@ if [[ $PWD != *sdks/python* ]]; then
 fi
 
 # Go to the Apache Beam Python SDK root
-if [[ "*sdks/python" != $PWD ]]; then
+if [[ $PWD != *sdks/python ]]; then
   cd $(pwd | sed 's/sdks\/python.*/sdks\/python/')
 fi
 
-tox -c tox.ini --recreate -e $1
+# Used in tox.ini to isolate toxworkdir of each environment.
+export ENV_NAME=.tox-$1
+
+if [[ ! -z $2 ]]; then
+  tox -c tox.ini --recreate -e $1 --installpkg $2
+else
+  tox -c tox.ini --recreate -e $1
+fi
+
 exit_code=$?
 # Retry once for the specific exit code 245.
 if [[ $exit_code == 245 ]]; then

sdks/python/scripts/run_tox_cleanup.sh

@@ -26,7 +26,7 @@ if [[ $PWD != *sdks/python* ]]; then
 fi
 
 # Go to the Apache Beam Python SDK root
-if [[ "*sdks/python" != $PWD ]]; then
+if [[ $PWD != *sdks/python ]]; then
   cd $(pwd | sed 's/sdks\/python.*/sdks\/python/')
 fi
 

sdks/python/test-suites/tox/py35/build.gradle

@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Unit tests for Python 3.5
+ */
+
+plugins { id 'org.apache.beam.module' }
+applyPythonNature()
+
+// Required to setup a Python 3 virtualenv.
+project.ext.python3 = true
+
+task lint {}
+check.dependsOn lint
+
+toxTask "lintPy35", "py35-lint"
+lint.dependsOn lintPy35
+
+toxTask "testPython35", "py35"
+test.dependsOn testPython35
+
+toxTask "testPy35Gcp", "py35-gcp"
+test.dependsOn testPy35Gcp
+
+toxTask "testPy35Cython", "py35-cython"
+test.dependsOn testPy35Cython
+// Ensure that testPy35Cython runs exclusively to other tests. This line is not
+// actually required, since gradle doesn't do parallel execution within a
+// project.
+testPy35Cython.mustRunAfter testPython35, testPy35Gcp
+
+task preCommitPy35() {
+    dependsOn "testPython35"
+    dependsOn "testPy35Gcp"
+    dependsOn "testPy35Cython"
+    dependsOn "lint"
+}

sdks/python/test-suites/tox/py36/build.gradle

@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Unit tests for Python 3.6
+ */
+
+plugins { id 'org.apache.beam.module' }
+applyPythonNature()
+
+// Required to setup a Python 3 virtualenv.
+project.ext.python3 = true
+
+toxTask "testPython36", "py36"
+test.dependsOn testPython36
+
+task preCommitPy36() {
+  dependsOn "testPython36"
+}

sdks/python/tox.ini

@@ -18,7 +18,7 @@
 [tox]
 # new environments will be excluded by default unless explicitly added to envlist.
 envlist = py27,py35,py36,py27-{gcp,cython,lint,lint3},py35-{gcp,cython,lint},docs
-toxworkdir = {toxinidir}/target/.tox
+toxworkdir = {toxinidir}/target/{env:ENV_NAME:.tox}
 
 [pycodestyle]
 # Disable all errors and warnings except for the ones related to blank lines.

settings.gradle

@@ -207,6 +207,10 @@ include "beam-sdks-python-test-suites-dataflow-py3"
 project(":beam-sdks-python-test-suites-dataflow-py3").dir = file("sdks/python/test-suites/dataflow/py3")
 include "beam-sdks-python-test-suites-direct-py3"
 project(":beam-sdks-python-test-suites-direct-py3").dir = file("sdks/python/test-suites/direct/py3")
+include "beam-sdks-python-test-suites-tox-py35"
+project(":beam-sdks-python-test-suites-tox-py35").dir = file("sdks/python/test-suites/tox/py35")
+include "beam-sdks-python-test-suites-tox-py36"
+project(":beam-sdks-python-test-suites-tox-py36").dir = file("sdks/python/test-suites/tox/py36")
 include "beam-sdks-python-load-tests"
 project(":beam-sdks-python-load-tests").dir = file("sdks/python/apache_beam/testing/load_tests")
 include "beam-vendor-grpc-1_13_1"