Add AnnotatedParallelClassesAndMethodsTest* and AnnotatedParallelMeth…

…odsTest*

No bugs found, this just rounds out the testing of ConsoleRunnerImpl to make sure that
the @TestParallelMethods and @TestParallelClassesAndMethods annotations take precedence
over the -default-concurrency setting.

Testing Done:
CI green at https://travis-ci.org/pantsbuild/pants/builds/140545792

Bugs closed: 3605

Reviewed at https://rbcommons.com/s/twitter/r/4027/

src/java/org/pantsbuild/tools/junit/impl/SpecException.java

@@ -14,6 +14,6 @@ public SpecException(String spec, String message, Throwable t) {
   }
 
   private static String formatMessage(String spec, String message) {
-    return String.format("FATAL: Error parsing spec %s: %s",spec, message);
+    return String.format("FATAL: Error parsing spec '%s': %s",spec, message);
   }
 }

tests/java/org/pantsbuild/tools/junit/impl/ConsoleRunnerTest.java

@@ -10,6 +10,8 @@
 import org.hamcrest.CoreMatchers;
 import org.junit.Assert;
 import org.junit.Test;
+import org.pantsbuild.tools.junit.lib.AnnotatedParallelClassesAndMethodsTest1;
+import org.pantsbuild.tools.junit.lib.AnnotatedParallelMethodsTest1;
 import org.pantsbuild.tools.junit.lib.AnnotatedParallelTest1;
 import org.pantsbuild.tools.junit.lib.AnnotatedSerialTest1;
 import org.pantsbuild.tools.junit.lib.ConsoleRunnerTestBase;
@@ -190,33 +192,53 @@ public void testParallelAnnotation() throws Exception {
     assertEquals("aptest1 aptest2", TestRegistry.getCalledTests());
   }
 
+  @Test
+  public void testParallelMethodsAnnotation() throws Exception {
+    // @ParallelTestMethods only works reliably under the experimental runner
+    assumeThat(parameters.useExperimentalRunner, is(true));
+    AnnotatedParallelMethodsTest1.reset();
+    invokeConsoleRunner("AnnotatedParallelMethodsTest1 AnnotatedParallelMethodsTest2"
+        + " -parallel-threads 2");
+    assertEquals("apmtest11 apmtest12 apmtest21 apmtest22", TestRegistry.getCalledTests());
+  }
+
+  @Test
+  public void testParallelClassesAndMethodsAnnotation() throws Exception {
+    // @ParallelClassesAndMethods only works reliably under the experimental runner
+    assumeThat(parameters.useExperimentalRunner, is(true));
+    AnnotatedParallelClassesAndMethodsTest1.reset();
+    invokeConsoleRunner("AnnotatedParallelClassesAndMethodsTest1"
+        + " AnnotatedParallelClassesAndMethodsTest2 -parallel-threads 4");
+    assertEquals("apcamtest11 apcamtest12 apcamtest21 apcamtest22", TestRegistry.getCalledTests());
+  }
+
   @Test
   public void testSerialAnnotation() throws Exception {
     AnnotatedSerialTest1.reset();
-    invokeConsoleRunner("AnnotatedSerialTest1 AnnotatedSerialTest2 -parallel-threads 2");
+    invokeConsoleRunner("AnnotatedSerialTest1 AnnotatedSerialTest2");
     assertEquals("astest1 astest2", TestRegistry.getCalledTests());
   }
 
   /* LEGACY, remove after -default-parallel argument is removed */
   @Test
   public void testParallelDefaultParallel() throws Exception {
     ParallelTest1.reset();
-    invokeConsoleRunner("ParallelTest1 ParallelTest2 -parallel-threads 2 -default-parallel");
+    invokeConsoleRunner("ParallelTest1 ParallelTest2 -default-parallel -parallel-threads 2");
     assertEquals("ptest1 ptest2", TestRegistry.getCalledTests());
   }
 
   @Test
   public void testConcurrencyParallelClasses() throws Exception {
     assumeThat(parameters.defaultConcurrency, is(Concurrency.PARALLEL_CLASSES));
     ParallelTest1.reset();
-    invokeConsoleRunner("ParallelTest1 ParallelTest2 "
-        + "-parallel-threads 2 -default-concurrency PARALLEL_CLASSES");
+    invokeConsoleRunner("ParallelTest1 ParallelTest2 -parallel-threads 2"
+        + " -default-concurrency PARALLEL_CLASSES");
     assertEquals("ptest1 ptest2", TestRegistry.getCalledTests());
   }
 
   @Test
   public void testConcurrencyParallelMethods() throws Exception {
-    // These tests only work with the experimental runner
+    // -default-concurrency PARALLEL_METHODS tests only work reliably with the experimental runner
     assumeThat(parameters.useExperimentalRunner, is(true));
     assumeThat(parameters.defaultConcurrency, is(Concurrency.PARALLEL_METHODS));
     ParallelMethodsDefaultParallelTest1.reset();
@@ -254,7 +276,7 @@ public void testBurst() {
 
   @Test
   public void testConcurrencyParameterizedParallelMethods() {
-    // These tests only work with the experimental runner
+    // -default-concurrency PARALLEL_METHODS tests only work reliably with the experimental runner
     assumeThat(parameters.useExperimentalRunner, is(true));
     // Requires parallel methods
     assumeThat(parameters.defaultConcurrency, is(Concurrency.PARALLEL_METHODS));
@@ -265,7 +287,8 @@ public void testConcurrencyParameterizedParallelMethods() {
 
   @Test
   public void testConcurrencyParameterizedParallelClassesAndMethods() {
-    // These tests only work with the experimental runner
+    // -default-concurrency PARALLEL_CLASSES_AND_METHODS tests only work reliably with the
+    // experimental runner
     assumeThat(parameters.useExperimentalRunner, is(true));
     // Requires parallel methods
     assumeThat(parameters.defaultConcurrency, is(Concurrency.PARALLEL_CLASSES_AND_METHODS));
@@ -279,7 +302,7 @@ public void testConcurrencyParameterizedParallelClassesAndMethods() {
 
   @Test
   public void testConcurrencyBurstParallelMethods() {
-    // These tests only work with the experimental runner
+    // -default-concurrency PARALLEL_METHODS tests only work reliably with the experimental runner
     assumeThat(parameters.useExperimentalRunner, is(true));
     // Requires parallel methods
     assumeThat(parameters.defaultConcurrency, anyOf(is(Concurrency.PARALLEL_METHODS),
@@ -293,7 +316,8 @@ public void testConcurrencyBurstParallelMethods() {
 
   @Test
   public void testConcurrencyBurstParallelClassesAndMethods() {
-    // These tests only work with the experimental runner
+    // -default-concurrency PARALLEL_CLASSES_AND_METHODS tests only work reliably with the
+    // experimental runner
     assumeThat(parameters.useExperimentalRunner, is(true));
     // Requires parallel methods
     assumeThat(parameters.defaultConcurrency, is(Concurrency.PARALLEL_CLASSES_AND_METHODS));

tests/java/org/pantsbuild/tools/junit/lib/AnnotatedParallelClassesAndMethodsTest1.java

@@ -0,0 +1,53 @@
+// Copyright 2016 Pants project contributors (see CONTRIBUTORS.md).
+// Licensed under the Apache License, Version 2.0 (see LICENSE).
+package org.pantsbuild.tools.junit.lib;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import org.junit.Test;
+import org.pantsbuild.junit.annotations.TestParallelClassesAndMethods;
+
+import static org.junit.Assert.assertTrue;
+
+/**
+ * This test is intentionally under a java_library() BUILD target so it will not be run
+ * on its own. It is run by the ConsoleRunnerTest suite to test ConsoleRunnerImpl.
+ * <p>
+ * Exercises the TestParallelClassesAndMethods annotation.
+ * <p>
+ * For all methods in AnnotatedParallelMethodsTest1 and AnnotatedParallelMethodsTest2
+ * to succeed, both test classes  must be running at the same time with the flag:
+ * <pre>
+ *  -parallel-threads 2
+ * </pre>
+ * when running with just these two classes as specs.
+ * <p>
+ * Runs in on the order of 10 milliseconds locally, but it may take longer on a CI machine to spin
+ * up 2 threads, so it has a generous timeout set.
+ * </p>
+ */
+@TestParallelClassesAndMethods
+public class AnnotatedParallelClassesAndMethodsTest1 {
+  private static final int NUM_CONCURRENT_TESTS = 4;
+  private static final int WAIT_TIMEOUT_MS = 3000;
+  private static CountDownLatch latch = new CountDownLatch(NUM_CONCURRENT_TESTS);
+
+  public static void reset() {
+    latch = new CountDownLatch(NUM_CONCURRENT_TESTS);
+  }
+
+  @Test
+  public void apmcatest11() throws Exception {
+    awaitLatch("apcamtest11");
+  }
+
+  @Test
+  public void apmcatest12() throws Exception {
+    awaitLatch("apcamtest12");
+  }
+    static void awaitLatch(String methodName) throws Exception {
+      TestRegistry.registerTestCall(methodName);
+      latch.countDown();
+      assertTrue(latch.await(WAIT_TIMEOUT_MS, TimeUnit.MILLISECONDS));
+    }
+}

tests/java/org/pantsbuild/tools/junit/lib/AnnotatedParallelClassesAndMethodsTest2.java

@@ -0,0 +1,22 @@
+// Copyright 2016 Pants project contributors (see CONTRIBUTORS.md).
+// Licensed under the Apache License, Version 2.0 (see LICENSE).
+package org.pantsbuild.tools.junit.lib;
+
+import org.junit.Test;
+import org.pantsbuild.junit.annotations.TestParallelClassesAndMethods;
+
+/**
+ * See {@link AnnotatedParallelClassesAndMethodsTest1}
+ */
+@TestParallelClassesAndMethods
+public class AnnotatedParallelClassesAndMethodsTest2 {
+
+  @Test
+  public void apcamtest21() throws Exception {
+    AnnotatedParallelClassesAndMethodsTest1.awaitLatch("apcamtest21");
+  }
+  @Test
+  public void apcamtest22() throws Exception {
+    AnnotatedParallelClassesAndMethodsTest1.awaitLatch("apcamtest22");
+  }
+}

tests/java/org/pantsbuild/tools/junit/lib/AnnotatedParallelMethodsTest1.java

@@ -0,0 +1,71 @@
+// Copyright 2016 Pants project contributors (see CONTRIBUTORS.md).
+// Licensed under the Apache License, Version 2.0 (see LICENSE).
+package org.pantsbuild.tools.junit.lib;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.junit.Test;
+import org.pantsbuild.junit.annotations.TestParallelMethods;
+
+import static org.junit.Assert.assertTrue;
+
+/**
+ * This test is intentionally under a java_library() BUILD target so it will not be run
+ * on its own. It is run by the ConsoleRunnerTest suite to test ConsoleRunnerImpl.
+ * <p>
+ * Exercises the TestParallelMethods annotation.
+ * <p>
+ * For all methods in AnnotatedParallelMethodsTest1 and AnnotatedParallelMethodsTest2
+ * to succeed, both test classes  must be running at the same time with the flag:
+ * <pre>
+ *  -parallel-threads 2
+ * </pre>
+ * when running with just these two classes as specs.
+ * <p>
+ * Runs in on the order of 10 milliseconds locally, but it may take longer on a CI machine to spin
+ * up 2 threads, so it has a generous timeout set.
+ * </p>
+ */
+@TestParallelMethods
+public class AnnotatedParallelMethodsTest1 {
+  private static final int NUM_CONCURRENT_TESTS = 2;
+  private static final int WAIT_TIMEOUT_MS = 3000;
+  private static CountDownLatch latch = new CountDownLatch(NUM_CONCURRENT_TESTS);
+  private static final AtomicInteger numRunning = new AtomicInteger(0);
+
+  public static void reset() {
+    latch = new CountDownLatch(NUM_CONCURRENT_TESTS);
+    numRunning.set(0);
+  }
+
+  @Test
+  public void apmtest11() throws Exception {
+    awaitLatch("apmtest11");
+  }
+
+  @Test
+  public void apmtest12() throws Exception {
+    awaitLatch("apmtest12");
+  }
+
+  static void awaitLatch(String methodName) throws Exception {
+    // NB(zundel): this test currently doesn't ensure that both classes run all methods in
+    // parallel, but the classes run serially, it only ensures that at least two methods get
+    // started and that no more than 2 methods run at a time. A better test would show that
+    // methods are run in parallel in each class.
+
+    TestRegistry.registerTestCall(methodName);
+    latch.countDown();
+    // Make sure that we wait for at least 2 methods to get started to ensure there is some
+    // parallelism going on.
+    assertTrue(latch.await(WAIT_TIMEOUT_MS, TimeUnit.MILLISECONDS));
+    numRunning.incrementAndGet();
+    Thread.sleep(WAIT_TIMEOUT_MS);
+    // Make sure no more than 2 tests have been started concurrently
+    int running = numRunning.get();
+    assertTrue(String.format("Expected <= %d got %d", NUM_CONCURRENT_TESTS, running),
+        running <= NUM_CONCURRENT_TESTS);
+    numRunning.decrementAndGet();
+  }
+}

tests/java/org/pantsbuild/tools/junit/lib/AnnotatedParallelMethodsTest2.java

@@ -0,0 +1,23 @@
+// Copyright 2016 Pants project contributors (see CONTRIBUTORS.md).
+// Licensed under the Apache License, Version 2.0 (see LICENSE).
+package org.pantsbuild.tools.junit.lib;
+
+import org.junit.Test;
+import org.pantsbuild.junit.annotations.TestParallelMethods;
+
+/**
+ * See {@link AnnotatedParallelMethodsTest1}
+ */
+@TestParallelMethods
+public class AnnotatedParallelMethodsTest2 {
+
+  @Test
+  public void apmtest21() throws Exception {
+    AnnotatedParallelMethodsTest1.awaitLatch("apmtest21");
+  }
+
+  @Test
+  public void apmtest22() throws Exception {
+    AnnotatedParallelMethodsTest1.awaitLatch("apmtest22");
+  }
+}

tests/java/org/pantsbuild/tools/junit/lib/MockBurstParallelClassesAndMethodsTest1.java

@@ -14,7 +14,7 @@
 @RunWith(BurstJUnit4.class)
 public class MockBurstParallelClassesAndMethodsTest1 {
   private static final int NUM_CONCURRENT_TESTS = 5;
-  private static final int RETRY_TIMEOUT_MS = 1000;
+  private static final int RETRY_TIMEOUT_MS = 3000;
   private static CountDownLatch latch = new CountDownLatch(NUM_CONCURRENT_TESTS);
 
   private final ColorType colorType;

tests/java/org/pantsbuild/tools/junit/lib/MockParameterizedParallelClassesAndMethodsTest1.java

@@ -16,7 +16,7 @@
 @RunWith(Parameterized.class)
 public class MockParameterizedParallelClassesAndMethodsTest1 {
   private static final int NUM_CONCURRENT_TESTS = 5;
-  private static final int RETRY_TIMEOUT_MS = 1000;
+  private static final int RETRY_TIMEOUT_MS = 3000;
   private static CountDownLatch latch = new CountDownLatch(NUM_CONCURRENT_TESTS);
 
   private String parameter;