Split up CI unit tests into two distinct shards (pantsbuild#7867)

### Problem
Now that we use the V2 test runner as of pantsbuild#7724, unit tests both take much longer (20 minutes -> 40-50 minutes) and have become very flaky (not exclusively thanks to V2).

Especially because the tests flake so much, it is frustrating to have to wait a whole 50 minutes to rerun the shard.

### Solution
We can't use automated sharding because V2 does not support that yet, but we can introduce our own manual shards. One shard runs the V2 tests, and the other runs all blacklisted tests, the contrib tests, and the `pants-plugin` tests.

### Result
Flakes will be slightly less painful, because when something flakes you will not have to run the entire 50 minutes of CI again, but just the subset for that specific shard.

.travis.yml

@@ -939,32 +939,57 @@ matrix:
     - <<: *cargo_audit
 
     - <<: *py27_linux_test_config
-      name: "Unit tests (Py2.7 PEX)"
+      name: "Unit tests - V2 test runner (Py2.7 PEX)"
       stage: *test
       env:
         - *py27_linux_test_config_env
-        - CACHE_NAME=linuxunittests.py27
+        - CACHE_NAME=unit_tests.v2.py27
+      script:
+        - travis_wait 50 ./build-support/bin/ci.py --python-tests-v2 --python-version 2.7
+
+    - <<: *py36_linux_test_config
+      name: "Unit tests - V2 test runner (Py3.6 PEX)"
+      env:
+        - *py36_linux_test_config_env
+        - CACHE_NAME=unit_tests.v2.py36
+      script:
+        - travis_wait 50 ./build-support/bin/ci.py --python-tests-v2
+
+    - <<: *py37_linux_test_config
+      name: "Unit tests - V2 test runner (Py3.7 PEX)"
+      env:
+        - *py37_linux_test_config_env
+        - CACHE_NAME=unit_tests.v2.py37
+      script:
+       - travis_wait 50 ./build-support/bin/ci.py --python-tests-v2 --python-version 3.7
+
+    - <<: *py27_linux_test_config
+      name: "Unit tests - V1 test runner (Py2.7 PEX)"
+      stage: *test
+      env:
+        - *py27_linux_test_config_env
+        - CACHE_NAME=unit_tests.v1.py27
       script:
         - ./build-support/bin/ci.py --plugin-tests --python-version 2.7
-        - travis_wait 60 ./build-support/bin/ci.py --python-tests --python-version 2.7
+        - ./build-support/bin/ci.py --python-tests-v1 --python-version 2.7
 
     - <<: *py36_linux_test_config
-      name: "Unit tests (Py3.6 PEX)"
+      name: "Unit tests - V1 test runner (Py3.6 PEX)"
       env:
         - *py36_linux_test_config_env
-        - CACHE_NAME=linuxunittests.py36
+        - CACHE_NAME=unit_tests.v1.py36
       script:
         - ./build-support/bin/ci.py --plugin-tests
-        - travis_wait 60 ./build-support/bin/ci.py --python-tests
+        - ./build-support/bin/ci.py --python-tests-v1
 
     - <<: *py37_linux_test_config
-      name: "Unit tests (Py3.7 PEX)"
+      name: "Unit tests - V1 test runner (Py3.7 PEX)"
       env:
         - *py37_linux_test_config_env
-        - CACHE_NAME=linuxunittests.py37
+        - CACHE_NAME=unit_tests.v1.py37
       script:
        - ./build-support/bin/ci.py --plugin-tests --python-version 3.7
-       - travis_wait 60 ./build-support/bin/ci.py --python-tests --python-version 3.7
+       - ./build-support/bin/ci.py --python-tests-v1 --python-version 3.7
 
     - <<: *py27_linux_build_wheels_ucs2
     - <<: *py27_linux_build_wheels_ucs4

build-support/bin/ci.py

@@ -35,8 +35,10 @@ def main() -> None:
     run_clippy()
   if args.cargo_audit:
     run_cargo_audit()
-  if args.python_tests:
-    run_python_tests()
+  if args.python_tests_v1:
+    run_python_tests_v1()
+  if args.python_tests_v2:
+    run_python_tests_v2()
   if args.rust_tests:
     run_rust_tests()
   if args.jvm_tests:
@@ -92,7 +94,15 @@ def create_parser() -> argparse.ArgumentParser:
   parser.add_argument(
     "--cargo-audit", action="store_true", help="Run Cargo audit of Rust dependencies."
   )
-  parser.add_argument("--python-tests", action="store_true", help="Run Python unit tests.")
+  # TODO(#7772): Simplify below to always use V2 and drop the blacklist.
+  parser.add_argument(
+    "--python-tests-v1", action="store_true",
+    help="Run Python unit tests with V1 test runner over the blacklist and contrib tests."
+  )
+  parser.add_argument(
+    "--python-tests-v2", action="store_true",
+    help="Run Python unit tests with V2 test runner."
+  )
   parser.add_argument("--rust-tests", action="store_true", help="Run Rust tests.")
   parser.add_argument("--jvm-tests", action="store_true", help="Run JVM tests.")
   parser.add_argument(
@@ -283,35 +293,10 @@ def run_cargo_audit() -> None:
       die("Cargo audit failure")
 
 
-def run_python_tests() -> None:
-  # TODO(#7772): Simplify below to always use V2 and drop the blacklist.
+def run_python_tests_v1() -> None:
   known_v2_failures_file = "build-support/unit_test_v2_blacklist.txt"
-  with open(known_v2_failures_file, "r") as f:
-    blacklisted_targets = {line.strip() for line in f.readlines()}
-
-  with travis_section("CoreTests", "Running Python unit tests"):
+  with travis_section("PythonTestsV1", "Running Python unit tests with V1 test runner"):
     check_pants_pex_exists()
-    try:
-      all_targets = subprocess.run([
-        "./pants.pex",
-        "--tag=-integration",
-        "--filter-type=python_tests",
-        "filter",
-        "src/python::",
-        "tests/python::",
-      ], stdout=subprocess.PIPE, encoding="utf-8", check=True).stdout.strip().split("\n")
-      v2_targets = set(all_targets) - blacklisted_targets
-      subprocess.run([
-        "./pants.pex",
-        "--no-v1",
-        "--v2",
-        "test.pytest"
-      ] + sorted(v2_targets) + PYTEST_PASSTHRU_ARGS, check=True)
-    except subprocess.CalledProcessError:
-      die("Core Python test failure (V2 test runner)")
-    else:
-      green("V2 unit tests passed.")
-
     try:
       subprocess.run([
         "./pants.pex",
@@ -338,6 +323,34 @@ def run_python_tests() -> None:
       green("Contrib unit tests passed.")
 
 
+def run_python_tests_v2() -> None:
+  known_v2_failures_file = "build-support/unit_test_v2_blacklist.txt"
+  with open(known_v2_failures_file, "r") as f:
+    blacklisted_targets = {line.strip() for line in f.readlines()}
+  with travis_section("PythonTestsV1", "Running Python unit tests with V2 test runner"):
+    check_pants_pex_exists()
+    try:
+      all_targets = subprocess.run([
+        "./pants.pex",
+        "--tag=-integration",
+        "--filter-type=python_tests",
+        "filter",
+        "src/python::",
+        "tests/python::",
+      ], stdout=subprocess.PIPE, encoding="utf-8", check=True).stdout.strip().split("\n")
+      v2_targets = set(all_targets) - blacklisted_targets
+      subprocess.run([
+        "./pants.pex",
+        "--no-v1",
+        "--v2",
+        "test.pytest"
+      ] + sorted(v2_targets) + PYTEST_PASSTHRU_ARGS, check=True)
+    except subprocess.CalledProcessError:
+      die("Python unit tests failure (V2 test runner)")
+    else:
+      green("V2 unit tests passed.")
+
+
 def run_rust_tests() -> None:
   command = [
     "build-support/bin/native/cargo",

build-support/travis/travis.yml.mustache

@@ -882,32 +882,57 @@ matrix:
     - <<: *cargo_audit
 
     - <<: *py27_linux_test_config
-      name: "Unit tests (Py2.7 PEX)"
+      name: "Unit tests - V2 test runner (Py2.7 PEX)"
       stage: *test
       env:
         - *py27_linux_test_config_env
-        - CACHE_NAME=linuxunittests.py27
+        - CACHE_NAME=unit_tests.v2.py27
+      script:
+        - travis_wait 50 ./build-support/bin/ci.py --python-tests-v2 --python-version 2.7
+
+    - <<: *py36_linux_test_config
+      name: "Unit tests - V2 test runner (Py3.6 PEX)"
+      env:
+        - *py36_linux_test_config_env
+        - CACHE_NAME=unit_tests.v2.py36
+      script:
+        - travis_wait 50 ./build-support/bin/ci.py --python-tests-v2
+
+    - <<: *py37_linux_test_config
+      name: "Unit tests - V2 test runner (Py3.7 PEX)"
+      env:
+        - *py37_linux_test_config_env
+        - CACHE_NAME=unit_tests.v2.py37
+      script:
+       - travis_wait 50 ./build-support/bin/ci.py --python-tests-v2 --python-version 3.7
+
+    - <<: *py27_linux_test_config
+      name: "Unit tests - V1 test runner (Py2.7 PEX)"
+      stage: *test
+      env:
+        - *py27_linux_test_config_env
+        - CACHE_NAME=unit_tests.v1.py27
       script:
         - ./build-support/bin/ci.py --plugin-tests --python-version 2.7
-        - travis_wait 60 ./build-support/bin/ci.py --python-tests --python-version 2.7
+        - ./build-support/bin/ci.py --python-tests-v1 --python-version 2.7
 
     - <<: *py36_linux_test_config
-      name: "Unit tests (Py3.6 PEX)"
+      name: "Unit tests - V1 test runner (Py3.6 PEX)"
       env:
         - *py36_linux_test_config_env
-        - CACHE_NAME=linuxunittests.py36
+        - CACHE_NAME=unit_tests.v1.py36
       script:
         - ./build-support/bin/ci.py --plugin-tests
-        - travis_wait 60 ./build-support/bin/ci.py --python-tests
+        - ./build-support/bin/ci.py --python-tests-v1
 
     - <<: *py37_linux_test_config
-      name: "Unit tests (Py3.7 PEX)"
+      name: "Unit tests - V1 test runner (Py3.7 PEX)"
       env:
         - *py37_linux_test_config_env
-        - CACHE_NAME=linuxunittests.py37
+        - CACHE_NAME=unit_tests.v1.py37
       script:
        - ./build-support/bin/ci.py --plugin-tests --python-version 3.7
-       - travis_wait 60 ./build-support/bin/ci.py --python-tests --python-version 3.7
+       - ./build-support/bin/ci.py --python-tests-v1 --python-version 3.7
 
     - <<: *py27_linux_build_wheels_ucs2
     - <<: *py27_linux_build_wheels_ucs4