Skip to content

Commit

Permalink
Adds arguments --version_between --time_between --versions to the ben…
Browse files Browse the repository at this point in the history
…chmark as version filters.

--
PiperOrigin-RevId: 147141691
MOS_MIGRATED_REVID=147141691
  • Loading branch information
hermione521 authored and kchodorow committed Feb 10, 2017
1 parent 3e73d9b commit b3589a0
Show file tree
Hide file tree
Showing 14 changed files with 384 additions and 55 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ java_binary(
"//src/main/java/com/google/devtools/build/lib:vfs",
"//src/main/java/com/google/devtools/common/options",
"//src/tools/benchmark/java/com/google/devtools/build/benchmark/codegenerator:codegenerator_lib",
"//third_party:auto_value",
"//third_party:guava",
"//third_party/protobuf:protobuf-util",
"//third_party/protobuf:protobuf_java",
Expand All @@ -33,6 +34,7 @@ java_library(
"//src/main/java/com/google/devtools/build/lib:vfs",
"//src/main/java/com/google/devtools/common/options",
"//src/tools/benchmark/java/com/google/devtools/build/benchmark/codegenerator:codegenerator_lib",
"//third_party:auto_value",
"//third_party:guava",
"//third_party/protobuf:protobuf-util",
"//third_party/protobuf:protobuf_java",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -64,9 +64,15 @@ public ImmutableList<BuildTargetConfig> getBuildTargetConfigs() {
}

@Override
public ImmutableList<String> getCodeVersions(Builder builder, String from, String to)
public ImmutableList<String> getCodeVersions(Builder builder, BenchmarkOptions options)
throws IOException, CommandException {
return builder.getCodeVersionsBetween(from, to);
if (options.versionFilter != null) {
return builder.getCodeVersionsBetweenVersions(options.versionFilter);
}
if (options.dateFilter != null) {
return builder.getCodeVersionsBetweenDates(options.dateFilter);
}
return ImmutableList.copyOf(options.versions);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,16 +49,6 @@ class BazelBuilder implements Builder {
this.builderDir = builderDir;
}

@Override
public ImmutableList<String> getCodeVersionsBetween(String from, String to)
throws CommandException {
String[] gitLogCommand = {"git", "log", from + ".." + to, "--pretty=format:%H", "--reverse"};
Command cmd = new Command(gitLogCommand, null, builderDir.toFile());
CommandResult result = cmd.execute();
String output = new String(result.getStdout(), UTF_8).trim();
return ImmutableList.copyOf(output.split("\n"));
}

@Override
public Path getBuildBinary(String codeVersion) throws IOException, CommandException {
if (buildBinary != null && currentCodeVersion.equals(codeVersion)) {
Expand Down Expand Up @@ -135,6 +125,23 @@ public void prepare() throws IOException, CommandException {
prepareFromGitRepo(DEFAULT_GIT_REPO);
}

@Override
public ImmutableList<String> getCodeVersionsBetweenVersions(VersionFilter versionFilter)
throws CommandException {
return getListOfOutputFromCommand(
"git", "log",
versionFilter.getFrom() + ".." + versionFilter.getTo(), "--pretty=format:%H", "--reverse");
}

@Override
public ImmutableList<String> getCodeVersionsBetweenDates(DateFilter dateFilter)
throws CommandException {
return getListOfOutputFromCommand(
"git", "log",
"--after", dateFilter.getFromString(),
"--before", dateFilter.getToString(), "--pretty=format:%H", "--reverse");
}

void prepareFromGitRepo(String gitRepo) throws IOException, CommandException {
if (builderDir.toFile().exists() && !builderDir.toFile().isDirectory()) {
try {
Expand All @@ -156,4 +163,12 @@ void prepareFromGitRepo(String gitRepo) throws IOException, CommandException {
}
// Assume the directory is what we need if not empty
}

private ImmutableList<String> getListOfOutputFromCommand(String... command)
throws CommandException{
Command cmd = new Command(command, null, builderDir.toFile());
CommandResult result = cmd.execute();
String output = new String(result.getStdout(), UTF_8).trim();
return ImmutableList.copyOf(output.split("\n"));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,42 +17,56 @@
import com.google.devtools.common.options.Option;
import com.google.devtools.common.options.OptionsBase;

import java.util.List;

/** Class that contains arguments for running the benchmark. */
public class BenchmarkOptions extends OptionsBase {

@Option(
name = "workspace",
defaultValue = "",
category = "benchmark",
valueHelp = "path",
help = "Directory where we put all the code and results."
name = "workspace",
defaultValue = "",
category = "benchmark",
valueHelp = "path",
help = "Directory where we put all the code and results."
)
public String workspace;

@Option(
name = "output",
defaultValue = "",
category = "benchmark",
valueHelp = "path",
help = "Path to put benchmark result (json format)."
name = "output",
defaultValue = "",
category = "benchmark",
valueHelp = "path",
help = "Path to put benchmark result (json format)."
)
public String output;

@Option(
name = "from",
defaultValue = "",
category = "benchmark",
valueHelp = "string",
help = "Use code versions from this (not included)."
name = "version_between",
defaultValue = "",
category = "version filter",
valueHelp = "string",
help = "Use code versions between two versions, eg. 'abcedf..uvwxyz'.",
converter = VersionFilterConverter.class
)
public VersionFilter versionFilter;

@Option(
name = "time_between",
defaultValue = "",
category = "time filter",
valueHelp = "string",
help = "Use code versions between two time, eg. '2017-01-01 13:00..2017-01-02 08:00'.",
converter = DateFilterConverter.class
)
public String from;
public DateFilter dateFilter;

@Option(
name = "to",
defaultValue = "",
category = "benchmark",
valueHelp = "string",
help = "Use code versions to this (included)."
name = "versions",
defaultValue = "",
category = "version",
valueHelp = "list of strings",
allowMultiple = true,
help = "Use code versions as listed."
)
public String to;
public List<String> versions;
}
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,9 @@ interface BuildCase {
ImmutableList<BuildTargetConfig> getBuildTargetConfigs();

/**
* Returns a list of code versions (can be anything you specified) in {@code (from, to]} (can be
* any interval you specified) of {@code builder}.
* Returns a list of code versions (can be anything you specified) of {@code builder}.
*/
ImmutableList<String> getCodeVersions(Builder builder, String from, String to)
ImmutableList<String> getCodeVersions(Builder builder, BenchmarkOptions options)
throws IOException, CommandException;

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@ class BuildGroupRunner {
this.workspace = workspace;
}

BuildGroupResult run(String from, String to) throws IOException, CommandException {
BuildGroupResult run(BenchmarkOptions opt)
throws IOException, CommandException {
BuildCase buildCase = new BazelBuildCase();
ImmutableList<BuildTargetConfig> buildTargetConfigs = buildCase.getBuildTargetConfigs();
ImmutableList<BuildEnvConfig> buildEnvConfigs = buildCase.getBuildEnvConfigs();
Expand All @@ -44,7 +45,12 @@ BuildGroupResult run(String from, String to) throws IOException, CommandExceptio
prepareBuilder();
System.out.println("Done preparing builder.");

ImmutableList<String> codeVersions = buildCase.getCodeVersions(builder, from, to);
// Get code versions (commit hashtag for Bazel)
ImmutableList<String> codeVersions = buildCase.getCodeVersions(builder, opt);
System.out.println("Ready to run benchmark for the following versions:");
for (String version : codeVersions) {
System.out.println(version);
}

BuildGroupResult.Builder buildGroupResultBuilder =
getBuildGroupResultBuilder(buildTargetConfigs, buildEnvConfigs, codeVersions);
Expand All @@ -57,11 +63,14 @@ BuildGroupResult run(String from, String to) throws IOException, CommandExceptio
// Get builder binary (build Bazel binary)
Path buildBinary = buildBinary = builder.getBuildBinary(version);

// Repeat several times to calculate average result
for (int t = 0; t < REPEAT_TIMES; ++t) {
// Environment config
for (int envIndex = 0; envIndex < buildEnvConfigs.size(); ++envIndex) {
BuildEnvConfig envConfig = buildEnvConfigs.get(envIndex);
System.out.println("Started config: " + envConfig.getDescription());

// Target config
for (int targetIndex = 0; targetIndex < buildTargetConfigs.size(); ++targetIndex) {
BuildTargetConfig targetConfig = buildTargetConfigs.get(targetIndex);
System.out.println(targetConfig.getDescription());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,13 @@ interface Builder {
/** Returns the binary path of the build tool of a specific {@code codeVersion}. */
Path getBuildBinary(String codeVersion) throws IOException, CommandException;

/** Return a list of code versions in {@code (from, to]}. */
ImmutableList<String> getCodeVersionsBetween(String from, String to) throws CommandException;
/** Returns the code versions of the build tool between versions {@code (from, to]}. */
ImmutableList<String> getCodeVersionsBetweenVersions(VersionFilter versionFilter)
throws CommandException;

/** Returns the code versions of the build tool between dates {@code [from, to]}. */
ImmutableList<String> getCodeVersionsBetweenDates(DateFilter dateFilter)
throws CommandException;

/** Returns a command for build under specific config. */
ImmutableList<String> getCommandFromConfig(
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
// Copyright 2017 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.devtools.build.benchmark;

import java.text.SimpleDateFormat;
import java.util.Date;

/**
* Contains start date and end date.
*/
class DateFilter {

static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");

private final Date from;
private final Date to;

public DateFilter(Date from, Date to) {
this.from = from;
this.to = to;
}

public String getFromString() {
return DATE_FORMAT.format(from);
}

public String getToString() {
return DATE_FORMAT.format(to);
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
// Copyright 2017 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.devtools.build.benchmark;

import com.google.devtools.common.options.Converter;
import com.google.devtools.common.options.OptionsParsingException;

import java.text.ParseException;
import java.util.Date;

/**
* A converter class that convert an input string to a {@link DateFilter} object.
*/
public class DateFilterConverter implements Converter<DateFilter> {

public DateFilterConverter() {
super();
}

@Override
public DateFilter convert(String input) throws OptionsParsingException {
if (input.isEmpty()) {
return null;
}

String[] parts = input.split("\\.\\.");
if (parts.length != 2) {
throw new OptionsParsingException("Error parsing time_between option: no '..' found.");
}
if (parts[0].isEmpty()) {
throw new OptionsParsingException(
"Error parsing time_between option: start date not found");
}
if (parts[1].isEmpty()) {
throw new OptionsParsingException(
"Error parsing time_between option: end date not found");
}

// TODO(yueg): support more date formats
try {
Date from = DateFilter.DATE_FORMAT.parse(parts[0]);
Date to = DateFilter.DATE_FORMAT.parse(parts[1]);
return new DateFilter(from, to);
} catch (ParseException e) {
throw new OptionsParsingException(
"Error parsing datetime, format should be: yyyy-MM-ddTHH:mm:ss");
}
}

@Override
public String getTypeDescription() {
return "A date filter in format: <start date(time)>..<end date(time)>";
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ public static void main(String[] args) {
BuildGroupRunner runner = new BuildGroupRunner(workspace.toPath());
BuildGroupResult result = null;
try {
result = runner.run(opt.from, opt.to);
result = runner.run(opt);
} catch (Exception e) {
logger.log(Level.SEVERE, e.getMessage());
System.exit(1);
Expand All @@ -82,12 +82,24 @@ public static void main(String[] args) {
public static BenchmarkOptions parseArgs(String[] args) throws OptionsParsingException {
BenchmarkOptions opt = Options.parse(BenchmarkOptions.class, args).getOptions();

// Check options
if (opt.workspace.isEmpty() || opt.from.isEmpty() || opt.to.isEmpty() || opt.output.isEmpty()) {
// Missing options
if (opt.workspace.isEmpty() || opt.output.isEmpty()) {
System.err.println(Options.getUsage(BenchmarkOptions.class));
throw new IllegalArgumentException("Argument value should not be empty.");
throw new IllegalArgumentException("Argument --workspace and --output should not be empty.");
}
// Should use exact one argument between from/to, after/before and versions
int emptyNum = booleanToInt(opt.versionFilter == null)
+ booleanToInt(opt.dateFilter == null)
+ booleanToInt(opt.versions.isEmpty());
if (emptyNum != 2) {
System.err.println(Options.getUsage(BenchmarkOptions.class));
throw new IllegalArgumentException("Please use exact one type of version filter at a time.");
}

return opt;
}

private static int booleanToInt(boolean b) {
return b ? 1 : 0;
}
}
Loading

0 comments on commit b3589a0

Please sign in to comment.