Benchmark part 3: main stuff

--
PiperOrigin-RevId: 146240366
MOS_MIGRATED_REVID=146240366

src/tools/benchmark/java/com/google/devtools/build/benchmark/BUILD

@@ -7,6 +7,22 @@ java_proto_library(
     src = "build_data.proto",
 )
 
+java_binary(
+    name = "benchmark",
+    srcs = glob(["*.java"]),
+    main_class = "com.google.devtools.build.benchmark.Main",
+    deps = [
+        ":build_data_proto",
+        "//src/main/java/com/google/devtools/build/lib:shell",
+        "//src/main/java/com/google/devtools/build/lib:vfs",
+        "//src/main/java/com/google/devtools/common/options",
+        "//src/tools/benchmark/java/com/google/devtools/build/benchmark/codegenerator:codegenerator_lib",
+        "//third_party:guava",
+        "//third_party/protobuf",
+        "//third_party/protobuf:protobuf-util",
+    ],
+)
+
 java_library(
     name = "benchmark_lib",
     testonly = 1,
@@ -19,6 +35,7 @@ java_library(
         "//src/tools/benchmark/java/com/google/devtools/build/benchmark/codegenerator:codegenerator_lib",
         "//third_party:guava",
         "//third_party/protobuf",
+        "//third_party/protobuf:protobuf-util",
     ],
 )
 

src/tools/benchmark/java/com/google/devtools/build/benchmark/BuildGroupRunner.java

@@ -0,0 +1,139 @@
+// Copyright 2017 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.benchmark;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.benchmark.codegenerator.JavaCodeGenerator;
+import com.google.devtools.build.lib.shell.CommandException;
+import java.io.IOException;
+import java.nio.file.Path;
+
+/** Class for running a build group with all build targets and getting performance results. */
+class BuildGroupRunner {
+
+  private static final String GENERATED_CODE_FOR_COPY_DIR = "GeneratedCodeForCopy";
+  private static final String GENERATED_CODE_DIR = "GeneratedCode";
+  private static final String BUILDER_DIR = "BuilderBazel";
+  private static final int REPEAT_TIMES = 3;
+
+  private final Path workspace;
+  private Builder builder = null;
+
+  BuildGroupRunner(Path workspace) {
+    this.workspace = workspace;
+  }
+
+  BuildGroupResult run(String from, String to) throws IOException, CommandException {
+    BuildCase buildCase = new BazelBuildCase();
+    ImmutableList<BuildTargetConfig> buildTargetConfigs = buildCase.getBuildTargetConfigs();
+    ImmutableList<BuildEnvConfig> buildEnvConfigs = buildCase.getBuildEnvConfigs();
+
+    // Prepare builder (Bazel)
+    prepareBuilder();
+    System.out.println("Done preparing builder.");
+
+    ImmutableList<String> codeVersions = buildCase.getCodeVersions(builder, from, to);
+
+    BuildGroupResult.Builder buildGroupResultBuilder =
+        getBuildGroupResultBuilder(buildTargetConfigs, buildEnvConfigs, codeVersions);
+
+    boolean lastIsIncremental = true;
+    for (int versionIndex = 0; versionIndex < codeVersions.size(); ++versionIndex) {
+      String version = codeVersions.get(versionIndex);
+      System.out.format("Benchmark for version %s started.\n", version);
+
+      // Get builder binary (build Bazel binary)
+      Path buildBinary = buildBinary = builder.getBuildBinary(version);
+
+      for (int t = 0; t < REPEAT_TIMES; ++t) {
+        for (int envIndex = 0; envIndex < buildEnvConfigs.size(); ++envIndex) {
+          BuildEnvConfig envConfig = buildEnvConfigs.get(envIndex);
+          System.out.println("Started config: " + envConfig.getDescription());
+
+          for (int targetIndex = 0; targetIndex < buildTargetConfigs.size(); ++targetIndex) {
+            BuildTargetConfig targetConfig = buildTargetConfigs.get(targetIndex);
+            System.out.println(targetConfig.getDescription());
+
+            // Prepare generated code for build
+            if (lastIsIncremental && !envConfig.getIncremental()) {
+              buildCase.prepareGeneratedCode(
+                  workspace.resolve(GENERATED_CODE_FOR_COPY_DIR),
+                  workspace.resolve(GENERATED_CODE_DIR));
+            }
+            if (!lastIsIncremental && envConfig.getIncremental()) {
+              JavaCodeGenerator.modifyExistingProject(
+                  workspace.resolve(GENERATED_CODE_DIR).toString(), true, true, true, true);
+            }
+            lastIsIncremental = envConfig.getIncremental();
+
+            // Builder's clean method
+            if (envConfig.getCleanBeforeBuild()) {
+              builder.clean();
+            }
+
+            // Run build
+            double elapsedTime =
+                builder.buildAndGetElapsedTime(
+                    buildBinary, builder.getCommandFromConfig(targetConfig, envConfig));
+            System.out.println(elapsedTime);
+
+            // Store result
+            buildGroupResultBuilder
+                .getBuildTargetResultsBuilder(targetIndex)
+                .getBuildEnvResultsBuilder(envIndex)
+                .getResultsBuilder(versionIndex)
+                .addResults(elapsedTime);
+          }
+        }
+      }
+    }
+
+    return buildGroupResultBuilder.build();
+  }
+
+  private static BuildGroupResult.Builder getBuildGroupResultBuilder(
+      ImmutableList<BuildTargetConfig> buildTargetConfigs,
+      ImmutableList<BuildEnvConfig> buildEnvConfigs,
+      ImmutableList<String> codeVersions) {
+    // Initialize a BuildGroupResult object to preserve array length
+    BuildGroupResult.Builder buildGroupResultBuilder = BuildGroupResult.newBuilder();
+    for (BuildTargetConfig targetConfig : buildTargetConfigs) {
+      BuildTargetResult.Builder targetBuilder =
+          BuildTargetResult.newBuilder().setBuildTargetConfig(targetConfig);
+      prepareBuildEnvConfigs(buildEnvConfigs, codeVersions, targetBuilder);
+      buildGroupResultBuilder.addBuildTargetResults(targetBuilder.build());
+    }
+    return buildGroupResultBuilder;
+  }
+
+  private static void prepareBuildEnvConfigs(
+      ImmutableList<BuildEnvConfig> buildEnvConfigs,
+      ImmutableList<String> codeVersions,
+      BuildTargetResult.Builder targetBuilder) {
+    for (BuildEnvConfig envConfig : buildEnvConfigs) {
+      BuildEnvResult.Builder envBuilder = BuildEnvResult.newBuilder().setConfig(envConfig);
+      for (String version : codeVersions) {
+        envBuilder.addResults(SingleBuildResult.newBuilder().setCodeVersion(version).build());
+      }
+      targetBuilder.addBuildEnvResults(envBuilder.build());
+    }
+  }
+
+  private void prepareBuilder() throws IOException, CommandException {
+    builder =
+        new BazelBuilder(workspace.resolve(GENERATED_CODE_DIR), workspace.resolve(BUILDER_DIR));
+    builder.prepare();
+  }
+}

src/tools/benchmark/java/com/google/devtools/build/benchmark/Main.java

@@ -0,0 +1,93 @@
+// Copyright 2017 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.benchmark;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.devtools.common.options.Options;
+import com.google.devtools.common.options.OptionsParsingException;
+import com.google.protobuf.InvalidProtocolBufferException;
+import com.google.protobuf.util.JsonFormat;
+import java.io.File;
+import java.io.IOException;
+import java.io.Writer;
+import java.nio.file.Files;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/** Main class for running benchmark. */
+public class Main {
+
+  private static final Logger logger = Logger.getLogger(Main.class.getName());
+
+  public static void main(String[] args) {
+
+    BenchmarkOptions opt = null;
+    try {
+      opt = parseArgs(args);
+    } catch (Exception e) {
+      if (!e.getMessage().isEmpty()) {
+        logger.log(Level.SEVERE, e.getMessage());
+      }
+      System.exit(1);
+    }
+
+    // Prepare paths
+    File workspace = new File(opt.workspace);
+    if (!workspace.exists() && !workspace.mkdirs()) {
+      logger.log(Level.SEVERE, "Failed to create workspace directory: " + opt.workspace);
+      System.exit(1);
+    }
+    File outputFile = new File(opt.output);
+    if (outputFile.exists()) {
+      logger.log(Level.SEVERE, "Output file already exists: " + opt.output);
+      System.exit(1);
+    }
+
+    BuildGroupRunner runner = new BuildGroupRunner(workspace.toPath());
+    BuildGroupResult result = null;
+    try {
+      result = runner.run(opt.from, opt.to);
+    } catch (Exception e) {
+      logger.log(Level.SEVERE, e.getMessage());
+      System.exit(1);
+    }
+
+    // Store data
+    try {
+      Writer writer = Files.newBufferedWriter(outputFile.toPath(), UTF_8);
+      JsonFormat.printer().appendTo(result, writer);
+      writer.flush();
+    } catch (InvalidProtocolBufferException e) {
+      logger.log(Level.SEVERE, "Invalid protobuf: " + e.getMessage());
+      System.exit(1);
+    } catch (IOException e) {
+      logger.log(Level.SEVERE, "Failed to write to output file: " + e.getMessage());
+      System.exit(1);
+    }
+  }
+
+  public static BenchmarkOptions parseArgs(String[] args) throws OptionsParsingException {
+    BenchmarkOptions opt = Options.parse(BenchmarkOptions.class, args).getOptions();
+
+    // Check options
+    if (opt.workspace.isEmpty() || opt.from.isEmpty() || opt.to.isEmpty() || opt.output.isEmpty()) {
+      System.err.println(Options.getUsage(BenchmarkOptions.class));
+      throw new IllegalArgumentException("Argument value should not be empty.");
+    }
+
+    return opt;
+  }
+}

src/tools/benchmark/javatests/com/google/devtools/build/benchmark/BUILD

@@ -1,5 +1,15 @@
 package(default_visibility = ["//src/tools/benchmark:__subpackages__"])
 
+java_test(
+    name = "MainTest",
+    srcs = ["MainTest.java"],
+    deps = [
+        "//src/main/java/com/google/devtools/common/options",
+        "//src/tools/benchmark/java/com/google/devtools/build/benchmark:benchmark_lib",
+        "//third_party:junit4",
+    ],
+)
+
 java_test(
     name = "BazelBuildCaseTest",
     srcs = ["BazelBuildCaseTest.java"],

src/tools/benchmark/javatests/com/google/devtools/build/benchmark/MainTest.java

@@ -0,0 +1,50 @@
+// Copyright 2017 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.benchmark;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import com.google.devtools.common.options.OptionsParsingException;
+import java.io.IOException;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Test for {@link Main}. */
+@RunWith(JUnit4.class)
+public class MainTest {
+
+  @Test
+  public void testParseArgsMissingArgs() throws OptionsParsingException, IOException {
+    try {
+      Main.parseArgs(new String[] {"--workspace=workspace", "--from=1"});
+      fail("Should throw IllegalArgumentException");
+    } catch (IllegalArgumentException e) {
+      assertEquals("Argument value should not be empty.", e.getMessage());
+    }
+  }
+
+  @Test
+  public void testParseArgsCorrect() throws OptionsParsingException, IOException {
+    BenchmarkOptions opt =
+        Main.parseArgs(
+            new String[] {"--output=output", "--workspace=workspace", "--from=1", "--to=3"});
+    assertEquals(opt.output, "output");
+    assertEquals(opt.workspace, "workspace");
+    assertEquals(opt.from, "1");
+    assertEquals(opt.to, "3");
+  }
+}