Moved testing of examples and testprojects to tests

Instead of invoking 'test examples/::' and 'test testprojects/::' in ci.sh created Python integration tests for them.

Moved compile_strategy_utils.py to testutils.

Moved test_binary_util, test_maven_layout and test_thrift_util to util folder.

Testing Done:
CI passes: https://travis-ci.org/fkorotkov/pants/builds/61192561

Reviewed at https://rbcommons.com/s/twitter/r/2158/

.travis.osx.yml

@@ -26,11 +26,10 @@ env:
   matrix:
     # No need to run the self-checks on OSX where vms are at a premium since these are file-level,
     # non-platform fragile lints.
-    # - CI_FLAGS="-clpnet 'Various pants self checks'"  # (fkmsr)
-    - CI_FLAGS="-fkmsrcjlpn 'Test examples and testprojects'"  # (et)
-    - CI_FLAGS="-fkmsrcnet 'Unit tests for pants and pants-plugins'"  # (jlp)
-    - CI_FLAGS="-fkmsrcjlpet 'Python contrib tests'"  # (n)
-    - CI_FLAGS="-fkmsrjlpnet 'Python integration tests for pants'"  # (c)
+    # - CI_FLAGS="-clpn 'Various pants self checks'"  # (fkmsr)
+    - CI_FLAGS="-fkmsrcn 'Unit tests for pants and pants-plugins'"  # (jlp)
+    - CI_FLAGS="-fkmsrcjlp 'Python contrib tests'"  # (n)
+    - CI_FLAGS="-fkmsrjlpn 'Python integration tests for pants'"  # (c)
 
 before_install:
   # We need a sane python and as of 3/5/2015 on OSX 10.9 the provided python 2.7.5 encounters

.travis.yml

@@ -32,16 +32,15 @@ env:
     # See: http://docs.travis-ci.com/user/encryption-keys/
     - secure: VvwbndU++a2/iNAjk9cd67ATiipDwqcKnxDR4/J2Ik3GH10wHEDUhJ1+MK4WLhedfaOakDOEmarZQS3GwtgvCHO3knpTJuJc8d/bCfZovYuSqdi//BEv4dS7hDt6tQeJfkbBjG0T4yNjPJ3W9R9KDWCy/vj2CUm90BGg2CmxUbg=
   matrix:
-    - CI_FLAGS="-cjlpnet 'Various pants self checks'"  # (fkmsr)
-    - CI_FLAGS="-fkmsrcjlpn 'Test examples and testprojects'"  # (et)
-    - CI_FLAGS="-fkmsrcnet 'Unit tests for pants and pants-plugins'"  # (jlp)
-    - CI_FLAGS="-fkmsrcjlpet 'Python contrib tests'"  # (n)
-    - CI_FLAGS="-fkmsrjlpnet -i 6:0 'Python integration tests for pants - shard 1'"  # (c)
-    - CI_FLAGS="-fkmsrjlpnet -i 6:1 'Python integration tests for pants - shard 2'"
-    - CI_FLAGS="-fkmsrjlpnet -i 6:2 'Python integration tests for pants - shard 3'"
-    - CI_FLAGS="-fkmsrjlpnet -i 6:3 'Python integration tests for pants - shard 4'"
-    - CI_FLAGS="-fkmsrjlpnet -i 6:4 'Python integration tests for pants - shard 5'"
-    - CI_FLAGS="-fkmsrjlpnet -i 6:5 'Python integration tests for pants - shard 6'"
+    - CI_FLAGS="-cjlpn 'Various pants self checks'"  # (fkmsr)
+    - CI_FLAGS="-fkmsrcn 'Unit tests for pants and pants-plugins'"  # (jlp)
+    - CI_FLAGS="-fkmsrcjlp 'Python contrib tests'"  # (n)
+    - CI_FLAGS="-fkmsrjlpn -i 6:0 'Python integration tests for pants - shard 1'"  # (c)
+    - CI_FLAGS="-fkmsrjlpn -i 6:1 'Python integration tests for pants - shard 2'"
+    - CI_FLAGS="-fkmsrjlpn -i 6:2 'Python integration tests for pants - shard 3'"
+    - CI_FLAGS="-fkmsrjlpn -i 6:3 'Python integration tests for pants - shard 4'"
+    - CI_FLAGS="-fkmsrjlpn -i 6:4 'Python integration tests for pants - shard 5'"
+    - CI_FLAGS="-fkmsrjlpn -i 6:5 'Python integration tests for pants - shard 6'"
 
 before_script: |
   ./build-support/bin/ci-sync.sh

build-support/bin/ci.sh

@@ -8,7 +8,7 @@ source build-support/common.sh
 function usage() {
   echo "Runs commons tests for local or hosted CI."
   echo
-  echo "Usage: $0 (-h|-fxbkmsrjlpncieat)"
+  echo "Usage: $0 (-h|-fxbkmsrjlpncia)"
   echo " -h           print out this help message"
   echo " -f           skip python code formatting checks"
   echo " -x           skip bootstrap clean-all (assume bootstrapping from a"
@@ -23,14 +23,12 @@ function usage() {
   echo " -l           skip internal backends python tests"
   echo " -p           skip core python tests"
   echo " -n           skip contrib python tests"
-  echo " -c           skip pants integration tests"
+  echo " -c           skip pants integration tests (includes examples and testprojects)"
   echo " -i TOTAL_SHARDS:SHARD_NUMBER"
   echo "              if running integration tests, divide them into"
   echo "              TOTAL_SHARDS shards and just run those in SHARD_NUMBER"
   echo "              to run only even tests: '-i 2:0', odd: '-i 2:1'"
-  echo " -e           skip example tests"
   echo " -a           skip android targets when running tests"
-  echo " -t           skip testprojects tests"
   if (( $# > 0 )); then
     die "$@"
   else
@@ -44,7 +42,7 @@ bootstrap_compile_args=(
   --fail-slow
 )
 
-while getopts "hfxbkmsrjlpnci:eat" opt; do
+while getopts "hfxbkmsrjlpnci:a" opt; do
   case ${opt} in
     h) usage ;;
     f) skip_pre_commit_checks="true" ;;
@@ -66,9 +64,7 @@ while getopts "hfxbkmsrjlpnci:eat" opt; do
       TOTAL_SHARDS=${OPTARG%%:*}
       SHARD_NUMBER=${OPTARG##*:}
       ;;
-    e) skip_examples="true" ;;
     a) skip_android="true" ;;
-    t) skip_testprojects="true" ;;
     *) usage "Invalid option: -${OPTARG}" ;;
   esac
 done
@@ -77,9 +73,9 @@ shift $((${OPTIND} - 1))
 # Android testing requires the SDK to be installed and configured in Pants.
 # Skip if ANDROID_HOME isn't configured in the environment
 if [[ -z "${ANDROID_HOME}"  || "${skip_android:-false}" == "true" ]] ; then
-  android_test_opts="--exclude-target-regexp=.*android.*"
+  export SKIP_ANDROID="true"
 else
-  android_test_opts=""
+  export SKIP_ANDROID="false"
 fi
 
 if [[ $# > 0 ]]; then
@@ -201,77 +197,6 @@ if [[ "${skip_contrib:-false}" == "false" ]]; then
   ) || die "Contrib python test failure"
 fi
 
-if [[ "${skip_testprojects:-false}" == "false" ]]; then
-
-  # TODO(Eric Ayers) find a better way to deal with tests that are known to fail.
-  # right now, just split them into two categories and ignore them.
-
-  # Targets that fail but shouldn't
-  known_failing_targets=(
-    # The following two targets lose out due to a resource collision, because `example_b` happens
-    # to be first in the context, and test.junit mixes all classpaths.
-    testprojects/maven_layout/resource_collision/example_b/src/test/java/org/pantsbuild/duplicateres/exampleb:exampleb
-    testprojects/maven_layout/resource_collision/example_c/src/test/java/org/pantsbuild/duplicateres/examplec:examplec
-  )
-
-  # Targets that are intended to fail
-  negative_test_targets=(
-    testprojects/src/antlr/pants/backend/python/test:antlr_failure
-    testprojects/src/java/org/pantsbuild/testproject/bundle:missing-files
-    testprojects/src/java/org/pantsbuild/testproject/cycle1
-    testprojects/src/java/org/pantsbuild/testproject/cycle2
-    testprojects/src/java/org/pantsbuild/testproject/missingdepswhitelist.*
-    testprojects/src/python/antlr:test_antlr_failure
-    testprojects/src/scala/org/pantsbuild/testproject/compilation_failure
-    testprojects/src/thrift/org/pantsbuild/thrift_linter:
-    testprojects/tests/java/org/pantsbuild/testproject/empty:
-    testprojects/tests/java/org/pantsbuild/testproject/dummies:failing_target
-    testprojects/tests/python/pants/dummies:failing_target
-  )
-
-  targets_to_exclude=(
-    ${known_failing_targets[@]}
-    ${negative_test_targets[@]}
-  )
-  exclude_opts="${targets_to_exclude[@]/#/--exclude-target-regexp=}"
-
-  run()
-  {
-    local strategy=$1
-    banner "Running tests in testprojects/ with the $strategy strategy"
-    (
-      ./pants.pex ${PANTS_ARGS[@]} test \
-        --compile-apt-strategy=$strategy \
-        --compile-java-strategy=$strategy \
-        --compile-scala-strategy=$strategy \
-        testprojects:: $android_test_opts $exclude_opts
-    ) || die "test failure in testprojects/ with the $strategy strategy"
-  }
-
-  run "global"
-  run "isolated"
-fi
-
-if [[ "${skip_examples:-false}" == "false" ]]; then
-  run()
-  {
-    local strategy=$1
-    banner "Running example tests with the $strategy strategy"
-    (
-      ./pants.pex ${PANTS_ARGS[@]} test \
-        --compile-apt-strategy=$strategy \
-        --compile-java-strategy=$strategy \
-        --compile-scala-strategy=$strategy \
-        examples:: \
-        $android_test_opts
-    ) || die "Examples tests failure with the $strategy strategy"
-  }
-
-  run "global"
-  run "isolated"
-fi
-
-
 if [[ "${skip_integration:-false}" == "false" ]]; then
   if [[ ! -z "${TOTAL_SHARDS}" ]]; then
     shard_desc=" [shard $((SHARD_NUMBER+1)) of ${TOTAL_SHARDS}]"

tests/python/pants_test/BUILD

@@ -120,6 +120,7 @@ target(
     'tests/python/pants_test/net',
     'tests/python/pants_test/option',
     'tests/python/pants_test/process',
+    'tests/python/pants_test/projects',
     'tests/python/pants_test/python',
     'tests/python/pants_test/reporting',
     'tests/python/pants_test/scm',

tests/python/pants_test/backend/jvm/tasks/jvm_compile/BUILD

@@ -40,14 +40,6 @@ python_tests(
   ]
 )
 
-python_library(
-  name = 'utils',
-  sources = ['utils.py'],
-  dependencies = [
-    '3rdparty/python:pytest',
-  ]
-)
-
 python_library(
   name='base_compile_integration_test',
   sources=['base_compile_integration_test.py'],

tests/python/pants_test/backend/jvm/tasks/jvm_compile/java/test_apt_compile_integration.py

@@ -6,7 +6,7 @@
                         unicode_literals, with_statement)
 
 from pants_test.backend.jvm.tasks.jvm_compile.base_compile_integration_test import BaseCompileIT
-from pants_test.backend.jvm.tasks.jvm_compile.utils import provide_compile_strategies
+from pants_test.testutils.compile_strategy_utils import provide_compile_strategies
 
 
 class AptCompileIntegrationTest(BaseCompileIT):

tests/python/pants_test/backend/jvm/tasks/jvm_compile/java/test_java_compile_integration.py

@@ -11,8 +11,8 @@
 from pants.fs.archive import TarArchiver
 from pants.util.contextutil import temporary_dir
 from pants.util.dirutil import safe_walk
-from pants_test.backend.jvm.tasks.jvm_compile.utils import provide_compile_strategies
 from pants_test.pants_run_integration_test import PantsRunIntegrationTest
+from pants_test.testutils.compile_strategy_utils import provide_compile_strategies
 
 
 class JavaCompileIntegrationTest(PantsRunIntegrationTest):

tests/python/pants_test/backend/jvm/tasks/jvm_compile/java/test_java_zinc_compile_integration.py

@@ -6,7 +6,7 @@
                         unicode_literals, with_statement)
 
 from pants_test.backend.jvm.tasks.jvm_compile.base_compile_integration_test import BaseCompileIT
-from pants_test.backend.jvm.tasks.jvm_compile.utils import provide_compile_strategies
+from pants_test.testutils.compile_strategy_utils import provide_compile_strategies
 
 
 class JvmExamplesCompileIntegrationTest(BaseCompileIT):

tests/python/pants_test/backend/jvm/tasks/jvm_compile/scala/test_scala_compile_integration.py

@@ -9,7 +9,7 @@
 from contextlib import contextmanager
 
 from pants_test.backend.jvm.tasks.jvm_compile.base_compile_integration_test import BaseCompileIT
-from pants_test.backend.jvm.tasks.jvm_compile.utils import provide_compile_strategies
+from pants_test.testutils.compile_strategy_utils import provide_compile_strategies
 
 
 class ScalaCompileIntegrationTest(BaseCompileIT):

tests/python/pants_test/projects/BUILD

@@ -0,0 +1,36 @@
+# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+target(
+  name='projects',
+  dependencies=[
+    ':examples_integration',
+    ':testprojects_integration',
+  ]
+)
+
+python_tests(
+  name='examples_integration',
+  sources=['test_examples_integration.py'],
+  dependencies=[
+    ':base',
+    'tests/python/pants_test/testutils',
+  ]
+)
+
+python_tests(
+  name='testprojects_integration',
+  sources=['test_testprojects_integration.py'],
+  dependencies=[
+    ':base',
+    'tests/python/pants_test/testutils',
+  ]
+)
+
+python_tests(
+  name='base',
+  sources=['base_project_integration_test.py'],
+  dependencies=[
+    'tests/python/pants_test:int-test',
+  ]
+)

tests/python/pants_test/projects/base_project_integration_test.py

@@ -0,0 +1,25 @@
+# coding=utf-8
+# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+import os
+
+from pants_test.pants_run_integration_test import PantsRunIntegrationTest
+
+
+class ProjectIntegrationTest(PantsRunIntegrationTest):
+  @staticmethod
+  def _android_flags():
+    exclude_android = os.environ.get('SKIP_ANDROID') == "true" or not os.environ.get('ANDROID_HOME')
+    return ['--exclude-target-regexp=.*android.*'] if exclude_android else []
+
+  def pants_test(self, strategy, command):
+    return self.run_pants([
+      'test'
+      '--compile-apt-strategy={}'.format(strategy),
+      '--compile-java-strategy={}'.format(strategy),
+      '--compile-scala-strategy={}'.format(strategy),
+    ] + command + self._android_flags())

tests/python/pants_test/projects/test_examples_integration.py

@@ -0,0 +1,16 @@
+# coding=utf-8
+# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+from pants_test.projects.base_project_integration_test import ProjectIntegrationTest
+from pants_test.testutils.compile_strategy_utils import provide_compile_strategies
+
+
+class ExamplesIntegrationTest(ProjectIntegrationTest):
+  @provide_compile_strategies
+  def tests_examples(self, strategy):
+    pants_run = self.pants_test(strategy, ['examples::'])
+    self.assert_success(pants_run)

tests/python/pants_test/projects/test_testprojects_integration.py

@@ -0,0 +1,44 @@
+# coding=utf-8
+# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+from pants_test.projects.base_project_integration_test import ProjectIntegrationTest
+from pants_test.testutils.compile_strategy_utils import provide_compile_strategies
+
+
+class TestProjectsIntegrationTest(ProjectIntegrationTest):
+  @provide_compile_strategies
+  def tests_testprojects(self, strategy):
+    # TODO(Eric Ayers) find a better way to deal with tests that are known to fail.
+    # right now, just split them into two categories and ignore them.
+
+    # Targets that fail but shouldn't
+    known_failing_targets = [
+      # The following two targets lose out due to a resource collision, because `example_b` happens
+      # to be first in the context, and test.junit mixes all classpaths.
+      'testprojects/maven_layout/resource_collision/example_b/src/test/java/org/pantsbuild/duplicateres/exampleb:exampleb',
+      'testprojects/maven_layout/resource_collision/example_c/src/test/java/org/pantsbuild/duplicateres/examplec:examplec',
+    ]
+
+    # Targets that are intended to fail
+    negative_test_targets = [
+      'testprojects/src/antlr/pants/backend/python/test:antlr_failure',
+      'testprojects/src/java/org/pantsbuild/testproject/bundle:missing-files',
+      'testprojects/src/java/org/pantsbuild/testproject/cycle1',
+      'testprojects/src/java/org/pantsbuild/testproject/cycle2',
+      'testprojects/src/java/org/pantsbuild/testproject/missingdepswhitelist.*',
+      'testprojects/src/python/antlr:test_antlr_failure',
+      'testprojects/src/scala/org/pantsbuild/testproject/compilation_failure',
+      'testprojects/src/thrift/org/pantsbuild/thrift_linter:',
+      'testprojects/tests/java/org/pantsbuild/testproject/empty:',
+      'testprojects/tests/java/org/pantsbuild/testproject/dummies:failing_target',
+      'testprojects/tests/python/pants/dummies:failing_target',
+    ]
+
+    targets_to_exclude = known_failing_targets + negative_test_targets
+    exclude_opts = map(lambda target: '--exclude-target-regexp={}'.format(target), targets_to_exclude)
+    pants_run = self.pants_test(strategy, ['testprojects::'] + exclude_opts)
+    self.assert_success(pants_run)

tests/python/pants_test/tasks/BUILD

@@ -346,7 +346,7 @@ python_tests(
     'src/python/pants/scm:scm',
     'src/python/pants/util:contextutil',
     'src/python/pants/util:dirutil',
-    'tests/python/pants_test/backend/jvm/tasks/jvm_compile:utils',
+    'tests/python/pants_test/testutils',
   ],
 )
 

tests/python/pants_test/tasks/test_jar_publish.py

@@ -22,8 +22,8 @@
 from pants.scm.scm import Scm
 from pants.util.contextutil import temporary_dir
 from pants.util.dirutil import safe_mkdir, safe_walk
-from pants_test.backend.jvm.tasks.jvm_compile.utils import set_compile_strategies
 from pants_test.tasks.task_test_base import TaskTestBase
+from pants_test.testutils.compile_strategy_utils import set_compile_strategies
 
 
 class JarPublishTest(TaskTestBase):

tests/python/pants_test/backend/jvm/tasks/jvm_compile/utils.py → tests/python/pants_test/testutils/compile_strategy_utils.py

@@ -29,7 +29,10 @@ def wrapped(self):
   return wrapped
 
 def provide_compile_strategies(testmethod):
-  """A decorator for test methods that provides the compilation strategy as a parameter."""
+  """ A decorator for test methods that provides the compilation strategy as a parameter.
+
+  Invokes the test multiple times, once for each built-in strategy in _STRATEGIES.
+  """
   return _wrap(testmethod, lambda self, testmethod, strategy: testmethod(self, strategy))
 
 def set_compile_strategies(testmethod):