From eaa5281942eaeda294b7e36ab012f592c1f637c2 Mon Sep 17 00:00:00 2001 From: Yue Gan Date: Wed, 1 Feb 2017 15:20:15 +0000 Subject: [PATCH] Benchmark part 3: main stuff -- PiperOrigin-RevId: 146240366 MOS_MIGRATED_REVID=146240366 --- .../com/google/devtools/build/benchmark/BUILD | 17 +++ .../build/benchmark/BuildGroupRunner.java | 139 ++++++++++++++++++ .../google/devtools/build/benchmark/Main.java | 93 ++++++++++++ .../com/google/devtools/build/benchmark/BUILD | 10 ++ .../devtools/build/benchmark/MainTest.java | 50 +++++++ 5 files changed, 309 insertions(+) create mode 100644 src/tools/benchmark/java/com/google/devtools/build/benchmark/BuildGroupRunner.java create mode 100644 src/tools/benchmark/java/com/google/devtools/build/benchmark/Main.java create mode 100644 src/tools/benchmark/javatests/com/google/devtools/build/benchmark/MainTest.java diff --git a/src/tools/benchmark/java/com/google/devtools/build/benchmark/BUILD b/src/tools/benchmark/java/com/google/devtools/build/benchmark/BUILD index 8d433d9cd9204e..b2c4956837078d 100644 --- a/src/tools/benchmark/java/com/google/devtools/build/benchmark/BUILD +++ b/src/tools/benchmark/java/com/google/devtools/build/benchmark/BUILD @@ -7,6 +7,22 @@ java_proto_library( src = "build_data.proto", ) +java_binary( + name = "benchmark", + srcs = glob(["*.java"]), + main_class = "com.google.devtools.build.benchmark.Main", + deps = [ + ":build_data_proto", + "//src/main/java/com/google/devtools/build/lib:shell", + "//src/main/java/com/google/devtools/build/lib:vfs", + "//src/main/java/com/google/devtools/common/options", + "//src/tools/benchmark/java/com/google/devtools/build/benchmark/codegenerator:codegenerator_lib", + "//third_party:guava", + "//third_party/protobuf", + "//third_party/protobuf:protobuf-util", + ], +) + java_library( name = "benchmark_lib", testonly = 1, @@ -19,6 +35,7 @@ java_library( "//src/tools/benchmark/java/com/google/devtools/build/benchmark/codegenerator:codegenerator_lib", "//third_party:guava", "//third_party/protobuf", + "//third_party/protobuf:protobuf-util", ], ) diff --git a/src/tools/benchmark/java/com/google/devtools/build/benchmark/BuildGroupRunner.java b/src/tools/benchmark/java/com/google/devtools/build/benchmark/BuildGroupRunner.java new file mode 100644 index 00000000000000..48bd2d47d2f6f2 --- /dev/null +++ b/src/tools/benchmark/java/com/google/devtools/build/benchmark/BuildGroupRunner.java @@ -0,0 +1,139 @@ +// Copyright 2017 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.benchmark; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.benchmark.codegenerator.JavaCodeGenerator; +import com.google.devtools.build.lib.shell.CommandException; +import java.io.IOException; +import java.nio.file.Path; + +/** Class for running a build group with all build targets and getting performance results. */ +class BuildGroupRunner { + + private static final String GENERATED_CODE_FOR_COPY_DIR = "GeneratedCodeForCopy"; + private static final String GENERATED_CODE_DIR = "GeneratedCode"; + private static final String BUILDER_DIR = "BuilderBazel"; + private static final int REPEAT_TIMES = 3; + + private final Path workspace; + private Builder builder = null; + + BuildGroupRunner(Path workspace) { + this.workspace = workspace; + } + + BuildGroupResult run(String from, String to) throws IOException, CommandException { + BuildCase buildCase = new BazelBuildCase(); + ImmutableList buildTargetConfigs = buildCase.getBuildTargetConfigs(); + ImmutableList buildEnvConfigs = buildCase.getBuildEnvConfigs(); + + // Prepare builder (Bazel) + prepareBuilder(); + System.out.println("Done preparing builder."); + + ImmutableList codeVersions = buildCase.getCodeVersions(builder, from, to); + + BuildGroupResult.Builder buildGroupResultBuilder = + getBuildGroupResultBuilder(buildTargetConfigs, buildEnvConfigs, codeVersions); + + boolean lastIsIncremental = true; + for (int versionIndex = 0; versionIndex < codeVersions.size(); ++versionIndex) { + String version = codeVersions.get(versionIndex); + System.out.format("Benchmark for version %s started.\n", version); + + // Get builder binary (build Bazel binary) + Path buildBinary = buildBinary = builder.getBuildBinary(version); + + for (int t = 0; t < REPEAT_TIMES; ++t) { + for (int envIndex = 0; envIndex < buildEnvConfigs.size(); ++envIndex) { + BuildEnvConfig envConfig = buildEnvConfigs.get(envIndex); + System.out.println("Started config: " + envConfig.getDescription()); + + for (int targetIndex = 0; targetIndex < buildTargetConfigs.size(); ++targetIndex) { + BuildTargetConfig targetConfig = buildTargetConfigs.get(targetIndex); + System.out.println(targetConfig.getDescription()); + + // Prepare generated code for build + if (lastIsIncremental && !envConfig.getIncremental()) { + buildCase.prepareGeneratedCode( + workspace.resolve(GENERATED_CODE_FOR_COPY_DIR), + workspace.resolve(GENERATED_CODE_DIR)); + } + if (!lastIsIncremental && envConfig.getIncremental()) { + JavaCodeGenerator.modifyExistingProject( + workspace.resolve(GENERATED_CODE_DIR).toString(), true, true, true, true); + } + lastIsIncremental = envConfig.getIncremental(); + + // Builder's clean method + if (envConfig.getCleanBeforeBuild()) { + builder.clean(); + } + + // Run build + double elapsedTime = + builder.buildAndGetElapsedTime( + buildBinary, builder.getCommandFromConfig(targetConfig, envConfig)); + System.out.println(elapsedTime); + + // Store result + buildGroupResultBuilder + .getBuildTargetResultsBuilder(targetIndex) + .getBuildEnvResultsBuilder(envIndex) + .getResultsBuilder(versionIndex) + .addResults(elapsedTime); + } + } + } + } + + return buildGroupResultBuilder.build(); + } + + private static BuildGroupResult.Builder getBuildGroupResultBuilder( + ImmutableList buildTargetConfigs, + ImmutableList buildEnvConfigs, + ImmutableList codeVersions) { + // Initialize a BuildGroupResult object to preserve array length + BuildGroupResult.Builder buildGroupResultBuilder = BuildGroupResult.newBuilder(); + for (BuildTargetConfig targetConfig : buildTargetConfigs) { + BuildTargetResult.Builder targetBuilder = + BuildTargetResult.newBuilder().setBuildTargetConfig(targetConfig); + prepareBuildEnvConfigs(buildEnvConfigs, codeVersions, targetBuilder); + buildGroupResultBuilder.addBuildTargetResults(targetBuilder.build()); + } + return buildGroupResultBuilder; + } + + private static void prepareBuildEnvConfigs( + ImmutableList buildEnvConfigs, + ImmutableList codeVersions, + BuildTargetResult.Builder targetBuilder) { + for (BuildEnvConfig envConfig : buildEnvConfigs) { + BuildEnvResult.Builder envBuilder = BuildEnvResult.newBuilder().setConfig(envConfig); + for (String version : codeVersions) { + envBuilder.addResults(SingleBuildResult.newBuilder().setCodeVersion(version).build()); + } + targetBuilder.addBuildEnvResults(envBuilder.build()); + } + } + + private void prepareBuilder() throws IOException, CommandException { + builder = + new BazelBuilder(workspace.resolve(GENERATED_CODE_DIR), workspace.resolve(BUILDER_DIR)); + builder.prepare(); + } +} diff --git a/src/tools/benchmark/java/com/google/devtools/build/benchmark/Main.java b/src/tools/benchmark/java/com/google/devtools/build/benchmark/Main.java new file mode 100644 index 00000000000000..368d85a34b9dcb --- /dev/null +++ b/src/tools/benchmark/java/com/google/devtools/build/benchmark/Main.java @@ -0,0 +1,93 @@ +// Copyright 2017 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.benchmark; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.devtools.common.options.Options; +import com.google.devtools.common.options.OptionsParsingException; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.util.JsonFormat; +import java.io.File; +import java.io.IOException; +import java.io.Writer; +import java.nio.file.Files; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** Main class for running benchmark. */ +public class Main { + + private static final Logger logger = Logger.getLogger(Main.class.getName()); + + public static void main(String[] args) { + + BenchmarkOptions opt = null; + try { + opt = parseArgs(args); + } catch (Exception e) { + if (!e.getMessage().isEmpty()) { + logger.log(Level.SEVERE, e.getMessage()); + } + System.exit(1); + } + + // Prepare paths + File workspace = new File(opt.workspace); + if (!workspace.exists() && !workspace.mkdirs()) { + logger.log(Level.SEVERE, "Failed to create workspace directory: " + opt.workspace); + System.exit(1); + } + File outputFile = new File(opt.output); + if (outputFile.exists()) { + logger.log(Level.SEVERE, "Output file already exists: " + opt.output); + System.exit(1); + } + + BuildGroupRunner runner = new BuildGroupRunner(workspace.toPath()); + BuildGroupResult result = null; + try { + result = runner.run(opt.from, opt.to); + } catch (Exception e) { + logger.log(Level.SEVERE, e.getMessage()); + System.exit(1); + } + + // Store data + try { + Writer writer = Files.newBufferedWriter(outputFile.toPath(), UTF_8); + JsonFormat.printer().appendTo(result, writer); + writer.flush(); + } catch (InvalidProtocolBufferException e) { + logger.log(Level.SEVERE, "Invalid protobuf: " + e.getMessage()); + System.exit(1); + } catch (IOException e) { + logger.log(Level.SEVERE, "Failed to write to output file: " + e.getMessage()); + System.exit(1); + } + } + + public static BenchmarkOptions parseArgs(String[] args) throws OptionsParsingException { + BenchmarkOptions opt = Options.parse(BenchmarkOptions.class, args).getOptions(); + + // Check options + if (opt.workspace.isEmpty() || opt.from.isEmpty() || opt.to.isEmpty() || opt.output.isEmpty()) { + System.err.println(Options.getUsage(BenchmarkOptions.class)); + throw new IllegalArgumentException("Argument value should not be empty."); + } + + return opt; + } +} diff --git a/src/tools/benchmark/javatests/com/google/devtools/build/benchmark/BUILD b/src/tools/benchmark/javatests/com/google/devtools/build/benchmark/BUILD index f2bc4e85b47291..b70bebb4aa09f2 100644 --- a/src/tools/benchmark/javatests/com/google/devtools/build/benchmark/BUILD +++ b/src/tools/benchmark/javatests/com/google/devtools/build/benchmark/BUILD @@ -1,5 +1,15 @@ package(default_visibility = ["//src/tools/benchmark:__subpackages__"]) +java_test( + name = "MainTest", + srcs = ["MainTest.java"], + deps = [ + "//src/main/java/com/google/devtools/common/options", + "//src/tools/benchmark/java/com/google/devtools/build/benchmark:benchmark_lib", + "//third_party:junit4", + ], +) + java_test( name = "BazelBuildCaseTest", srcs = ["BazelBuildCaseTest.java"], diff --git a/src/tools/benchmark/javatests/com/google/devtools/build/benchmark/MainTest.java b/src/tools/benchmark/javatests/com/google/devtools/build/benchmark/MainTest.java new file mode 100644 index 00000000000000..015c2f63256136 --- /dev/null +++ b/src/tools/benchmark/javatests/com/google/devtools/build/benchmark/MainTest.java @@ -0,0 +1,50 @@ +// Copyright 2017 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.benchmark; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import com.google.devtools.common.options.OptionsParsingException; +import java.io.IOException; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Test for {@link Main}. */ +@RunWith(JUnit4.class) +public class MainTest { + + @Test + public void testParseArgsMissingArgs() throws OptionsParsingException, IOException { + try { + Main.parseArgs(new String[] {"--workspace=workspace", "--from=1"}); + fail("Should throw IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertEquals("Argument value should not be empty.", e.getMessage()); + } + } + + @Test + public void testParseArgsCorrect() throws OptionsParsingException, IOException { + BenchmarkOptions opt = + Main.parseArgs( + new String[] {"--output=output", "--workspace=workspace", "--from=1", "--to=3"}); + assertEquals(opt.output, "output"); + assertEquals(opt.workspace, "workspace"); + assertEquals(opt.from, "1"); + assertEquals(opt.to, "3"); + } +}