Introduce DSL for configuring BWC tests (elastic#78597)

this makes configuring bwc Tests less errorprone due to strange groovy behaviour
vistaarjuneja · Oct 4, 2021 · e684d5e · e684d5e
1 parent 7e7cfe3
commit e684d5e
Show file tree

Hide file tree

Showing 15 changed files with 183 additions and 223 deletions.
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcVersions.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcVersions.java
@@ -21,7 +21,9 @@
 import java.util.Set;
 import java.util.SortedSet;
 import java.util.TreeSet;
+import java.util.function.BiConsumer;
 import java.util.function.Consumer;
+import java.util.function.Predicate;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
@@ -340,6 +342,14 @@ public List<Version> getIndexCompatible() {
         return unmodifiableList(filterSupportedVersions(indexCompatibles));
     }
 
+    public void withIndexCompatiple(BiConsumer<Version, String> versionAction) {
+        getIndexCompatible().forEach(v -> versionAction.accept(v, "v"+v.toString()));
+    }
+
+    public void withIndexCompatiple(Predicate<Version> filter, BiConsumer<Version, String> versionAction) {
+        getIndexCompatible().stream().filter(filter).forEach(v -> versionAction.accept(v, "v"+v.toString()));
+    }
+
     public List<Version> getWireCompatible() {
         List<Version> wireCompat = new ArrayList<>();
         List<Version> prevMajors = groupByMajor.get(currentVersion.getMajor() - 1);
@@ -353,6 +363,14 @@ public List<Version> getWireCompatible() {
         return unmodifiableList(filterSupportedVersions(wireCompat));
     }
 
+    public void withWireCompatiple(BiConsumer<Version, String> versionAction) {
+        getWireCompatible().forEach(v -> versionAction.accept(v, "v"+v.toString()));
+    }
+
+    public void withWireCompatiple(Predicate<Version> filter, BiConsumer<Version, String> versionAction) {
+        getWireCompatible().stream().filter(filter).forEach(v -> versionAction.accept(v, "v"+v.toString()));
+    }
+
     private List<Version> filterSupportedVersions(List<Version> wireCompat) {
         return Architecture.current() == Architecture.AARCH64
             ? wireCompat.stream().filter(version -> version.onOrAfter("7.12.0")).collect(Collectors.toList())

diff --git a/qa/ccs-rolling-upgrade-remote-cluster/build.gradle b/qa/ccs-rolling-upgrade-remote-cluster/build.gradle
@@ -6,7 +6,6 @@
  * Side Public License, v 1.
  */
 
-import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.internal.info.BuildParams
 import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
 
@@ -19,9 +18,7 @@ dependencies {
   testImplementation project(':client:rest-high-level')
 }
 
-for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) {
-  String bwcVersionStr = bwcVersion.toString()
-  String baseName = "v" + bwcVersionStr
+BuildParams.bwcVersions.withWireCompatiple { bwcVersion, baseName ->
 
   /**
    * We execute tests 3 times.
@@ -32,13 +29,13 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) {
    */
   def localCluster = testClusters.register("${baseName}-local") {
     numberOfNodes = 2
-    versions = [bwcVersionStr, project.version]
+    versions = [bwcVersion.toString(), project.version]
     setting 'cluster.remote.node.attr', 'gateway'
     setting 'xpack.security.enabled', 'false'
   }
   def remoteCluster = testClusters.register("${baseName}-remote") {
     numberOfNodes = 3
-    versions = [bwcVersionStr, project.version]
+    versions = [bwcVersion.toString(), project.version]
     firstNode.setting 'node.attr.gateway', 'true'
     lastNode.setting 'node.attr.gateway', 'true'
     setting 'xpack.security.enabled', 'false'
@@ -48,7 +45,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) {
   tasks.withType(StandaloneRestIntegTestTask).matching { it.name.startsWith("${baseName}#") }.configureEach {
     useCluster localCluster
     useCluster remoteCluster
-    systemProperty 'tests.upgrade_from_version', bwcVersionStr.replace('-SNAPSHOT', '')
+    systemProperty 'tests.upgrade_from_version', bwcVersion.toString().replace('-SNAPSHOT', '')
 
     doFirst {
       nonInputProperties.systemProperty('tests.rest.cluster', localCluster.map(c -> c.allHttpSocketURI.join(",")))

diff --git a/qa/full-cluster-restart/build.gradle b/qa/full-cluster-restart/build.gradle
@@ -7,7 +7,6 @@
  */
 
 
-import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.internal.info.BuildParams
 import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
 
@@ -16,12 +15,9 @@ apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.internal-test-artifact'
 apply plugin: 'elasticsearch.bwc-test'
 
-for (Version bwcVersion : BuildParams.bwcVersions.indexCompatible) {
-  def bwcVersionString = bwcVersion.toString();
-  String baseName = "v" + bwcVersionString
-
+BuildParams.bwcVersions.withIndexCompatiple { bwcVersion, baseName ->
   def baseCluster = testClusters.register(baseName) {
-      versions =  [bwcVersionString, project.version]
+      versions =  [bwcVersion.toString(), project.version]
       numberOfNodes = 2
       // some tests rely on the translog not being flushed
       setting 'indices.memory.shard_inactive_time', '60m'
@@ -48,7 +44,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.indexCompatible) {
     systemProperty 'tests.is_old_cluster', 'false'
   }
 
-  String oldVersion = bwcVersionString.minus("-SNAPSHOT")
+  String oldVersion = bwcVersion.toString().minus("-SNAPSHOT")
   tasks.matching { it.name.startsWith(baseName) && it.name.endsWith("ClusterTest") }.configureEach {
     it.systemProperty 'tests.old_cluster_version', oldVersion
     it.systemProperty 'tests.path.repo', "${buildDir}/cluster/shared/repo/${baseName}"

diff --git a/qa/mixed-cluster/build.gradle b/qa/mixed-cluster/build.gradle
@@ -6,7 +6,6 @@
  * Side Public License, v 1.
  */
 
-import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.internal.info.BuildParams
 import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
@@ -22,51 +21,46 @@ restResources {
   }
 }
 
-for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) {
-  if (bwcVersion == VersionProperties.getElasticsearchVersion()) {
-    // Not really a mixed cluster
-    continue;
-  }
-
-  String bwcVersionString = bwcVersion.toString()
-  String baseName = "v" + bwcVersionString
+BuildParams.bwcVersions.withWireCompatiple { bwcVersion, baseName ->
 
-  /* This project runs the core REST tests against a 4 node cluster where two of
+  if (bwcVersion != VersionProperties.getElasticsearchVersion()) {
+    /* This project runs the core REST tests against a 4 node cluster where two of
      the nodes has a different minor.  */
-  def baseCluster = testClusters.register(baseName) {
-    versions = [bwcVersionString, project.version]
-    numberOfNodes = 4
-    setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
-    setting 'xpack.security.enabled', 'false'
-  }
+    def baseCluster = testClusters.register(baseName) {
+      versions = [bwcVersion.toString(), project.version]
+      numberOfNodes = 4
+      setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
+      setting 'xpack.security.enabled', 'false'
+    }
 
-  tasks.register("${baseName}#mixedClusterTest", StandaloneRestIntegTestTask) {
-    useCluster baseCluster
-    mustRunAfter("precommit")
-    doFirst {
-      delete("${buildDir}/cluster/shared/repo/${baseName}")
-      // Getting the endpoints causes a wait for the cluster
-      println "Test cluster endpoints are: ${-> baseCluster.get().allHttpSocketURI.join(",")}"
-      println "Upgrading one node to create a mixed cluster"
-      if (BuildParams.isSnapshotBuild() == false) {
-           baseCluster.get().nodes."${baseName}-0".systemProperty 'es.index_mode_feature_flag_registered', 'true'
+    tasks.register("${baseName}#mixedClusterTest", StandaloneRestIntegTestTask) {
+      useCluster baseCluster
+      mustRunAfter("precommit")
+      doFirst {
+        delete("${buildDir}/cluster/shared/repo/${baseName}")
+        // Getting the endpoints causes a wait for the cluster
+        println "Test cluster endpoints are: ${-> baseCluster.get().allHttpSocketURI.join(",")}"
+        println "Upgrading one node to create a mixed cluster"
+        if (BuildParams.isSnapshotBuild() == false) {
+          baseCluster.get().nodes."${baseName}-0".systemProperty 'es.index_mode_feature_flag_registered', 'true'
+        }
+        baseCluster.get().nextNodeToNextVersion()
+        // Getting the endpoints causes a wait for the cluster
+        println "Upgrade complete, endpoints are: ${-> baseCluster.get().allHttpSocketURI.join(",")}"
+        println "Upgrading another node to create a mixed cluster"
+        if (BuildParams.isSnapshotBuild() == false) {
+          baseCluster.get().nodes."${baseName}-1".systemProperty 'es.index_mode_feature_flag_registered', 'true'
+        }
+        baseCluster.get().nextNodeToNextVersion()
+        nonInputProperties.systemProperty('tests.rest.cluster', baseCluster.map(c -> c.allHttpSocketURI.join(",")))
+        nonInputProperties.systemProperty('tests.clustername', baseName)
       }
-      baseCluster.get().nextNodeToNextVersion()
-      // Getting the endpoints causes a wait for the cluster
-      println "Upgrade complete, endpoints are: ${-> baseCluster.get().allHttpSocketURI.join(",")}"
-      println "Upgrading another node to create a mixed cluster"
-      if (BuildParams.isSnapshotBuild() == false) {
-        baseCluster.get().nodes."${baseName}-1".systemProperty 'es.index_mode_feature_flag_registered', 'true'
-      }
-      baseCluster.get().nextNodeToNextVersion()
-      nonInputProperties.systemProperty('tests.rest.cluster', baseCluster.map(c -> c.allHttpSocketURI.join(",")))
-      nonInputProperties.systemProperty('tests.clustername', baseName)
+      systemProperty 'tests.path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
+      onlyIf { project.bwc_tests_enabled }
     }
-    systemProperty 'tests.path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
-    onlyIf { project.bwc_tests_enabled }
-  }
 
-  tasks.register(bwcTaskName(bwcVersion)) {
-    dependsOn "${baseName}#mixedClusterTest"
+    tasks.register(bwcTaskName(bwcVersion)) {
+      dependsOn "${baseName}#mixedClusterTest"
+    }
   }
 }
diff --git a/qa/repository-multi-version/build.gradle b/qa/repository-multi-version/build.gradle
@@ -6,7 +6,6 @@
  * Side Public License, v 1.
  */
 
-import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.internal.info.BuildParams
 import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
 
@@ -20,8 +19,7 @@ dependencies {
   testImplementation project(':client:rest-high-level')
 }
 
-for (Version bwcVersion : BuildParams.bwcVersions.indexCompatible) {
-  String baseName = "v${bwcVersion}"
+BuildParams.bwcVersions.withIndexCompatiple { bwcVersion, baseName ->
   String oldClusterName = "${baseName}-old"
   String newClusterName = "${baseName}-new"
 

diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle
@@ -6,7 +6,6 @@
  * Side Public License, v 1.
  */
 
-import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.internal.info.BuildParams
 import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
 
@@ -15,7 +14,7 @@ apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.bwc-test'
 apply plugin: 'elasticsearch.rest-resources'
 
-for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) {
+BuildParams.bwcVersions.withWireCompatiple { bwcVersion, baseName ->
   /*
    * The goal here is to:
    * <ul>
@@ -30,11 +29,8 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) {
    * </ul>
    */
 
-  String bwcVersionString = bwcVersion.toString()
-  String baseName = "v" + bwcVersionString
-
   def baseCluster = testClusters.register(baseName) {
-    versions = [bwcVersionString, project.version]
+    versions = [bwcVersion.toString(), project.version]
     numberOfNodes = 3
 
     setting 'repositories.url.allowed_urls', 'http://snapshot.test*'

diff --git a/qa/verify-version-constants/build.gradle b/qa/verify-version-constants/build.gradle
@@ -6,7 +6,6 @@
  * Side Public License, v 1.
  */
 
-import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.internal.info.BuildParams
 import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
@@ -15,12 +14,9 @@ apply plugin: 'elasticsearch.internal-testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.bwc-test'
 
-for (Version bwcVersion : BuildParams.bwcVersions.indexCompatible) {
-  String bwcVersionString = bwcVersion.toString()
-  String baseName = "v${bwcVersionString}"
-
+BuildParams.bwcVersions.withIndexCompatiple { bwcVersion, baseName ->
   def baseCluster = testClusters.register(baseName) {
-      version = bwcVersionString
+      version = bwcVersion.toString()
       setting 'xpack.security.enabled', 'true'
       user username: 'admin', password: 'admin-password', role: 'superuser'
   }

diff --git a/x-pack/plugin/eql/qa/mixed-node/build.gradle b/x-pack/plugin/eql/qa/mixed-node/build.gradle
@@ -3,58 +3,50 @@ apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.bwc-test'
 apply plugin: 'elasticsearch.rest-test'
 
-import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.internal.info.BuildParams
 import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
 
 dependencies {
-  testImplementation project(':x-pack:qa')
-  testImplementation(project(xpackModule('ql:test-fixtures')))
-  testImplementation project(path: xpackModule('eql'), configuration: 'default')
+    testImplementation project(':x-pack:qa')
+    testImplementation(project(xpackModule('ql:test-fixtures')))
+    testImplementation project(path: xpackModule('eql'), configuration: 'default')
 }
 
-tasks.named("integTest").configure{ enabled = false}
-
-for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible.findAll { it.onOrAfter('7.10.0') }) {
-  if (bwcVersion == VersionProperties.getElasticsearchVersion()) {
-    // Not really a mixed cluster
-    continue;
-  }
-
-  String bwcVersionString = bwcVersion.toString()
-  String baseName = "v" + bwcVersion
-
-  def cluster = testClusters.register(baseName) {
-      versions = [bwcVersionString, project.version]
-      numberOfNodes = 3
-      testDistribution = 'DEFAULT'
-      setting 'xpack.security.enabled', 'false'
-      setting 'xpack.watcher.enabled', 'false'
-      setting 'xpack.ml.enabled', 'false'
-      setting 'xpack.eql.enabled', 'true'
-      setting 'xpack.license.self_generated.type', 'trial'
-      // for debugging purposes
-      // setting 'logger.org.elasticsearch.xpack.eql.plugin.TransportEqlSearchAction', 'TRACE'
-  }
-
-  tasks.register("${baseName}#mixedClusterTest", StandaloneRestIntegTestTask) {
-    useCluster cluster
-    mustRunAfter("precommit")
-    doFirst {
-      // Getting the endpoints causes a wait for the cluster
-      println "Endpoints are: ${-> testClusters."${baseName}".allHttpSocketURI.join(",")}"
-      println "Upgrading one node to create a mixed cluster"
-      cluster.get().nextNodeToNextVersion()
+tasks.named("integTest").configure { enabled = false }
+
+BuildParams.bwcVersions.withWireCompatiple(v -> v.onOrAfter("7.10.0") &&
+        v != VersionProperties.getElasticsearchVersion()) { bwcVersion, baseName ->
+    def cluster = testClusters.register(baseName) {
+        versions = [bwcVersion.toString(), project.version]
+        numberOfNodes = 3
+        testDistribution = 'DEFAULT'
+        setting 'xpack.security.enabled', 'false'
+        setting 'xpack.watcher.enabled', 'false'
+        setting 'xpack.ml.enabled', 'false'
+        setting 'xpack.eql.enabled', 'true'
+        setting 'xpack.license.self_generated.type', 'trial'
+        // for debugging purposes
+        // setting 'logger.org.elasticsearch.xpack.eql.plugin.TransportEqlSearchAction', 'TRACE'
+    }
 
-      println "Upgrade complete, endpoints are: ${-> testClusters.named(baseName).get().allHttpSocketURI.join(",")}"
-      nonInputProperties.systemProperty('tests.rest.cluster', cluster.map(c->c.allHttpSocketURI.join(",")))
-      nonInputProperties.systemProperty('tests.clustername', baseName)
+    tasks.register("${baseName}#mixedClusterTest", StandaloneRestIntegTestTask) {
+        useCluster cluster
+        mustRunAfter("precommit")
+        doFirst {
+            // Getting the endpoints causes a wait for the cluster
+            println "Endpoints are: ${-> testClusters."${baseName}".allHttpSocketURI.join(",")}"
+            println "Upgrading one node to create a mixed cluster"
+            cluster.get().nextNodeToNextVersion()
+
+            println "Upgrade complete, endpoints are: ${-> testClusters.named(baseName).get().allHttpSocketURI.join(",")}"
+            nonInputProperties.systemProperty('tests.rest.cluster', cluster.map(c -> c.allHttpSocketURI.join(",")))
+            nonInputProperties.systemProperty('tests.clustername', baseName)
+        }
+        onlyIf { project.bwc_tests_enabled }
     }
-    onlyIf { project.bwc_tests_enabled }
-  }
 
-  tasks.register(bwcTaskName(bwcVersion)) {
-    dependsOn "${baseName}#mixedClusterTest"
-  }
-}
+    tasks.register(bwcTaskName(bwcVersion)) {
+        dependsOn "${baseName}#mixedClusterTest"
+    }
+}
diff --git a/x-pack/plugin/sql/qa/jdbc/build.gradle b/x-pack/plugin/sql/qa/jdbc/build.gradle
@@ -71,10 +71,9 @@ subprojects {
     }
 
     // Configure compatibility testing tasks
-    for (Version bwcVersion : BuildParams.bwcVersions.indexCompatible) {
+    BuildParams.bwcVersions.withIndexCompatiple { bwcVersion, baseName ->
       // Compatibility testing for JDBC driver started with version 7.9.0
       if (bwcVersion.onOrAfter(Version.fromString("7.9.0")) && (bwcVersion.equals(VersionProperties.elasticsearchVersion) == false)) {
-        String baseName = "v${bwcVersion}"
         UnreleasedVersionInfo unreleasedVersion = BuildParams.bwcVersions.unreleasedInfo(bwcVersion)
         Configuration driverConfiguration = configurations.create("jdbcDriver${baseName}") {
           // TODO: Temporary workaround for https://github.com/elastic/elasticsearch/issues/73433