Adding new class to evaluate a test run against a baseline

Adding new class to evaluate a test run against a baseline and output the relative performance.

harness/src/main/java/org/apache/geode/perftest/analysis/BenchmarkRunAnalyzer.java

@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.geode.perftest.analysis;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.Writer;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Analyzer that takes in benchmarks, probes, and result directories and produces
+ * a comparison of the results to a provided writer.
+ *
+ * This currently handles data in the format
+ * <pre>
+ *   Result1
+ *     /BenchmarkA
+ *       /client1
+ *         /Probe1.csv
+ *       /client2
+ *         /Probe2.csv
+ *     /BenchmarkB
+ *         ...
+ * </pre>
+ */
+public class BenchmarkRunAnalyzer {
+  private final List<SensorData> benchmarks = new ArrayList<>();
+  private final List<ProbeResultParser> probes = new ArrayList<>();
+
+  /**
+   * Add a benchmark to be analyzed. The benchmark is expected to exist
+   * in both result directories passed to {@link #analyzeTestRun(File, File, Writer)}
+   */
+  public void addBenchmark(String name, String testResultDir, List<String> nodeNames) {
+    benchmarks.add(new SensorData(name, testResultDir, nodeNames));
+  }
+
+  /**
+   * Add a probe to produce a comparison for. The probe expects to find output files
+   * in the result directory for each node of each benchmark.
+   */
+  public void addProbe(ProbeResultParser probeResultParser) {
+    probes.add(probeResultParser);
+  }
+
+  public void analyzeTestRun(File testResultDir, File baselineResultDir, Writer output)
+      throws IOException {
+    PrintWriter stream = new PrintWriter(output);
+    for (SensorData benchmark : benchmarks) {
+      stream.println("-- " + benchmark.benchmarkName + " --");
+      for (ProbeResultParser probe : probes) {
+        stream.println(probe.getResultDescription());
+        for (String node : benchmark.nodesToParse) {
+          probe.parseResults(new File(new File(testResultDir, benchmark.benchmarkSubdirectory), node));
+        }
+        double testResult = probe.getProbeResult();
+        stream.println("Result: " + String.valueOf(testResult));
+        probe.reset();
+        for (String node : benchmark.nodesToParse) {
+          probe.parseResults(new File(new File(baselineResultDir, benchmark.benchmarkSubdirectory), node));
+        }
+        double baselineResult = probe.getProbeResult();
+        stream.println("Baseline: " + String.valueOf(baselineResult));
+        stream.println("Relative performance: " + String.valueOf(testResult / baselineResult));
+        stream.println();
+      }
+    }
+
+    stream.flush();
+  }
+
+  // TODO: depending on how run output is stored, this data may be excessive or insufficient
+  // The present assumption is each benchmark contains an arbitrarily named result directory
+  // containing subdirectories for each node.  Those subdirectories then contain the probe output
+  // files for the run, for that node.
+  private static class SensorData {
+    private final String benchmarkName;
+    private final String benchmarkSubdirectory;
+    private final List<String> nodesToParse;
+
+    public SensorData(String benchmarkName, String benchmarkSubdirectory, List<String> nodesNames) {
+      this.benchmarkName = benchmarkName;
+      this.benchmarkSubdirectory = benchmarkSubdirectory;
+      this.nodesToParse = nodesNames;
+    }
+  }
+
+}

harness/src/main/java/org/apache/geode/perftest/analysis/ProbeResultParser.java

@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.geode.perftest.analysis;
+
+import java.io.File;
+import java.io.IOException;
+
+public interface ProbeResultParser {
+  // Given a output directory for a benchmark, parse out the data for the desired probe.  Note that
+  // this method may be passed several csv files for a run and is expected to appropriately
+  // aggregate the result of interest.
+  void parseResults(File benchmarkOutputDir) throws IOException;
+
+  // Reset the parser to a clean state where parseResults can be called again
+  void reset();
+
+  // Get a single float value summarizing the data for the probe.
+  double getProbeResult();
+
+  // Get a text description of what the probe result is depicting
+  String getResultDescription();
+}

harness/src/main/java/org/apache/geode/perftest/yardstick/YardstickPercentileSensorParser.java → harness/src/main/java/org/apache/geode/perftest/yardstick/analysis/YardstickPercentileSensorParser.java

@@ -12,7 +12,7 @@
  * or implied. See the License for the specific language governing permissions and limitations under
  * the License.
  */
-package org.apache.geode.perftest.yardstick;
+package org.apache.geode.perftest.yardstick.analysis;
 
 import static java.lang.Math.abs;
 
@@ -22,12 +22,21 @@
 import java.io.IOException;
 import java.util.ArrayList;
 
-public class YardstickPercentileSensorParser {
-  static final String sensorOutputFile = "PercentileProbe.csv";
+import org.yardstickframework.probes.PercentileProbe;
+
+import org.apache.geode.perftest.analysis.ProbeResultParser;
+
+/**
+ * Parses the output from {@link PercentileProbe} and reports the
+ * 99% percentile latency in microseconds.
+ */
+public class YardstickPercentileSensorParser implements ProbeResultParser {
+  public static final String sensorOutputFile = "PercentileProbe.csv";
+  public static final String probeResultDescription = "99th percentile latency";
 
   private class SensorBucket {
     public int latencyBucket;
-    public float bucketPercentage;
+    public double bucketPercentage;
 
     SensorBucket(String dataLine) throws IOException {
       String[] data = dataLine.split(",");
@@ -36,7 +45,7 @@ private class SensorBucket {
       }
       try {
         latencyBucket = Integer.parseInt(data[0]);
-        bucketPercentage = Float.parseFloat(data[1]);
+        bucketPercentage = Double.parseDouble(data[1]);
       } catch (NumberFormatException e) {
         throw new IOException("Invalid data line: " + dataLine);
       }
@@ -60,6 +69,22 @@ public void parseResults(File resultDir) throws IOException {
     }
   }
 
+  @Override
+  public void reset() {
+    buckets = new ArrayList<>();
+  }
+
+  @Override
+  // Default probe result is the 99th percentile latency for the benchmark
+  public double getProbeResult() {
+    return getPercentile(99);
+  }
+
+  @Override
+  public String getResultDescription() {
+    return probeResultDescription;
+  }
+
   private void normalizeBuckets() {
     float totalPercentage = 0;
     for (SensorBucket bucket : buckets) {
@@ -73,11 +98,11 @@ private void normalizeBuckets() {
     }
   }
 
-  public float getPercentile(int target) {
+  public double getPercentile(int target) {
     if (target < 0 || target > 100) {
       throw new RuntimeException("Percentile must be in the range (0, 100), invalid value: " + target);
     }
-    float targetPercent = target / 100f;
+    double targetPercent = target / 100.0;
     normalizeBuckets();
 
     if (buckets.size() == 1) {
@@ -86,7 +111,7 @@ public float getPercentile(int target) {
 
     SensorBucket[] bucketArray = buckets.toArray(new SensorBucket[buckets.size()]);
 
-    float accumulator = 0;
+    double accumulator = 0;
     int i = -1;
     while (targetPercent - accumulator > 0.0001) {
       ++i;
@@ -100,7 +125,7 @@ public float getPercentile(int target) {
         bucketArray[i + 1].latencyBucket - targetBucket.latencyBucket :
         targetBucket.latencyBucket - bucketArray[i - 1].latencyBucket;
 
-    float percentileLocationInTargetBucket = 1.0f - ((accumulator - targetPercent) / targetBucket.bucketPercentage);
+    double percentileLocationInTargetBucket = 1.0 - ((accumulator - targetPercent) / targetBucket.bucketPercentage);
 
     return targetBucket.latencyBucket + bucketSize * percentileLocationInTargetBucket;
   }

harness/src/main/java/org/apache/geode/perftest/yardstick/YardstickThroughputSensorParser.java → harness/src/main/java/org/apache/geode/perftest/yardstick/analysis/YardstickThroughputSensorParser.java

@@ -12,7 +12,7 @@
  * or implied. See the License for the specific language governing permissions and limitations under
  * the License.
  */
-package org.apache.geode.perftest.yardstick;
+package org.apache.geode.perftest.yardstick.analysis;
 
 import java.io.BufferedReader;
 import java.io.File;
@@ -21,28 +21,17 @@
 import java.util.ArrayList;
 import java.util.List;
 
-public class YardstickThroughputSensorParser {
-  static final String sensorOutputFile = "ThroughputLatencyProbe.csv";
+import org.yardstickframework.probes.ThroughputLatencyProbe;
 
-  private class SensorDatapoint {
-    public int second;
-    public float opsPerSec;
-    public float avgLatency;
+import org.apache.geode.perftest.analysis.ProbeResultParser;
 
-    SensorDatapoint(String dataLine) throws IOException {
-      String[] data = dataLine.split(",");
-      if (data.length != 3) {
-        throw new IOException("Invalid data line: " + dataLine);
-      }
-      try {
-        second = Integer.parseInt(data[0]);
-        opsPerSec = Float.parseFloat(data[1]);
-        avgLatency = Float.parseFloat(data[2]);
-      } catch (NumberFormatException e) {
-        throw new IOException("Invalid data line: " + dataLine);
-      }
-    }
-  }
+/**
+ * Parses the results from a {@link ThroughputLatencyProbe} and
+ * reports the average throughput in operations/second.
+ */
+public class YardstickThroughputSensorParser implements ProbeResultParser {
+  public static final String sensorOutputFile = "ThroughputLatencyProbe.csv";
+  public static final String probeResultDescription = "average ops/second";
 
   private List<SensorDatapoint> datapoints = new ArrayList<>();
 
@@ -61,11 +50,46 @@ public void parseResults(File resultDir) throws IOException {
     }
   }
 
-  public float getAverageThroughput() {
-    float accumulator = 0;
+  @Override
+  public void reset() {
+    datapoints = new ArrayList<>();
+  }
+
+  @Override
+  public double getProbeResult() {
+    return getAverageThroughput();
+  }
+
+  @Override
+  public String getResultDescription() {
+    return probeResultDescription;
+  }
+
+  public double getAverageThroughput() {
+    double accumulator = 0;
     for (SensorDatapoint datapoint : datapoints) {
       accumulator += datapoint.opsPerSec;
     }
     return  accumulator / datapoints.size();
   }
+
+  private static class SensorDatapoint {
+    private int second;
+    private double opsPerSec;
+    private double avgLatency;
+
+    SensorDatapoint(String dataLine) throws IOException {
+      String[] data = dataLine.split(",");
+      if (data.length != 3) {
+        throw new IOException("Invalid data line: " + dataLine);
+      }
+      try {
+        second = Integer.parseInt(data[0]);
+        opsPerSec = Float.parseFloat(data[1]);
+        avgLatency = Float.parseFloat(data[2]);
+      } catch (NumberFormatException e) {
+        throw new IOException("Invalid data line: " + dataLine);
+      }
+    }
+  }
 }