GEODE-8763: run benchmark tests up to 5 times (apache#5864)

* run benchmark tests up to 5 times, this time correctly reporting failure if still didn't pass on the 5th try
* increase benchmark timeouts
* run benchmarks on every commit even when multiple commits come in close together

ci/pipelines/geode-build/jinja.template.yml

@@ -34,6 +34,7 @@
     trigger: true
     passed:
     - Build
+    version: every
 {% endmacro %}
 
 {%- macro deep_merge(a, b): %}
@@ -476,6 +477,7 @@ jobs:
     - get: geode-build-version
       trigger: true
       passed: *benchmark-inputs
+      version: every
     - put: concourse-metadata-resource
   - do:
     - task: run_benchmarks{{ run_var.title }}

ci/pipelines/shared/jinja.variables.yml

@@ -29,12 +29,12 @@ benchmarks:
     flag: '-PwithSsl -PtestJVM=/usr/lib/jvm/bellsoft-java11-amd64'
     options: '--tests=*GetBenchmark --tests=*PutBenchmark'
     max_in_flight: 1
-    timeout: 3h
+    timeout: 8h
   - title: '_with_security_manager'
     flag: '-PwithSecurityManager'
     options: '--tests=Partitioned*'
     max_in_flight: 2
-    timeout: 5h
+    timeout: 12h
 
 build_test:
   ARTIFACT_SLUG: build

ci/scripts/run_benchmarks.sh

@@ -85,10 +85,14 @@ do
   ./run_on_cluster.sh -t ${CLUSTER_TAG} -- rm /home/geode/locator10334view.dat;
 
   if ./run_against_baseline.sh -t ${CLUSTER_TAG} -b ${GEODE_SHA} -r ${GEODE_REPO} -p ${BENCHMARKS_REPO} ${BASELINE_OPTION} -e ${BENCHMARKS_BRANCH} -o ${RESULTS_DIR} -m "'source':'geode-ci',${METADATA_BASELINE},'baseline_branch':'${BASELINE_BRANCH}','geode_branch':'${GEODE_SHA}'" --ci -- ${FLAGS} ${TEST_OPTIONS} ; then
+    STATUS=0
     break
+  else
+    STATUS=1
   fi
 
   set -e
 done
 
 popd
+exit $STATUS