Build: Run weekly JMH Benchmarks & visualize results (apache#6441)

.github/workflows/recurring-jmh-benchmarks.yml

@@ -0,0 +1,72 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: "Recurring JMH Benchmarks"
+on:
+  schedule:
+    # * is a special character in YAML so you have to quote this string
+    # this schedules a workflow to run at specific UTC times using POSIX cron syntax -> https://crontab.guru/
+    # we're running benchmarks every Sunday at 00:00 UTC
+    - cron:  '0 0 * * 0'
+
+jobs:
+  run-benchmark:
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        # TODO: "IcebergSortCompactionBenchmark" seems to run indefinitely
+        benchmark: ["SparkParquetReadersFlatDataBenchmark", "SparkParquetReadersNestedDataBenchmark",
+                    "SparkParquetWritersFlatDataBenchmark", "SparkParquetWritersNestedDataBenchmark",
+                    "IcebergSourceFlatParquetDataFilterBenchmark",
+                    "IcebergSourceFlatParquetDataReadBenchmark", "IcebergSourceFlatParquetDataWriteBenchmark",
+                    "IcebergSourceNestedListParquetDataWriteBenchmark", "IcebergSourceNestedParquetDataFilterBenchmark",
+                    "IcebergSourceNestedParquetDataReadBenchmark", "IcebergSourceNestedParquetDataWriteBenchmark",
+                    "IcebergSourceParquetEqDeleteBenchmark", "IcebergSourceParquetMultiDeleteFileBenchmark",
+                    "IcebergSourceParquetPosDeleteBenchmark", "IcebergSourceParquetWithUnrelatedDeleteBenchmark"]
+        spark_version: ['iceberg-spark-3.3']
+    env:
+      SPARK_LOCAL_IP: localhost
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          repository: ${{ github.event.inputs.repo }}
+          ref: ${{ github.event.inputs.ref }}
+      - uses: actions/setup-java@v3
+        with:
+          distribution: zulu
+          java-version: 11
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~/.gradle/caches
+            ~/.gradle/wrapper
+          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
+          restore-keys: ${{ runner.os }}-gradle-
+      - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
+
+      - name: Run Benchmark
+        run: ./gradlew :iceberg-spark:${{ matrix.spark_version }}:jmh -PjmhIncludeRegex=${{ matrix.benchmark }} -PjmhOutputPath=benchmark/${{ matrix.benchmark }}.txt -PjmhJsonOutputPath=benchmark/${{ matrix.benchmark }}.json
+
+      - uses: actions/upload-artifact@v3
+        if: ${{ always() }}
+        with:
+          name: benchmark-results
+          path: |
+            **/benchmark/*

build.gradle

@@ -31,6 +31,7 @@ buildscript {
     classpath 'com.diffplug.spotless:spotless-plugin-gradle:6.12.0'
     classpath 'gradle.plugin.org.inferred:gradle-processors:3.7.0'
     classpath 'me.champeau.jmh:jmh-gradle-plugin:0.6.8'
+    classpath 'gradle.plugin.io.morethan.jmhreport:gradle-jmh-report:0.9.0'
     classpath "com.github.alisiikh:gradle-scalastyle-plugin:3.4.1"
     classpath 'com.palantir.gradle.revapi:gradle-revapi:1.7.0'
     classpath 'com.gorylenko.gradle-git-properties:gradle-git-properties:2.4.1'

gradle.properties

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 jmhOutputPath=build/reports/jmh/human-readable-output.txt
+jmhJsonOutputPath=build/reports/jmh/results.json
 jmhIncludeRegex=.*
 systemProp.defaultFlinkVersions=1.16
 systemProp.knownFlinkVersions=1.14,1.15,1.16

jmh.gradle

@@ -45,20 +45,33 @@ jmhProjects.add(project(":iceberg-data"))
 
 configure(jmhProjects) {
   apply plugin: 'me.champeau.jmh'
+  apply plugin: 'io.morethan.jmhreport'
+
+  def jmhReportDir = project.property("jmhJsonOutputPath").toString().replace(".json", "")
+  mkdir(file(jmhReportDir))
 
   jmh {
     jmhVersion = '1.32'
     failOnError = true
     forceGC = true
     includeTests = true
     humanOutputFile = file(jmhOutputPath)
+    resultsFile = file(jmhJsonOutputPath)
+    resultFormat = 'JSON'
     includes = [jmhIncludeRegex]
     zip64 = true
   }
 
+  jmhReport {
+    jmhResultPath = file(jmhJsonOutputPath)
+    jmhReportOutput = file(jmhReportDir)
+  }
+
   jmhCompileGeneratedClasses {
     pluginManager.withPlugin('com.palantir.baseline-error-prone') {
       options.errorprone.enabled = false
     }
   }
+
+  tasks.jmh.finalizedBy tasks.jmhReport
 }