Use branch appropriate version of Elasticsearch

Carrying on from the work done in elastic#11958, update the gradle build to download the same version of Elasticsearch as is specified in the logstash version.yml file. This commit updates the standard integration tests to use the same version of Elasticsearch that is already downloaded for x-pack integration tests, and also fixes integration tests to allow for the different responses around hits generated by different versions of Elasticsearch.
tanglisha · Jun 16, 2020 · 8df9d2c · 8df9d2c
1 parent 78c7204
commit 8df9d2c
Show file tree

Hide file tree

Showing 6 changed files with 72 additions and 75 deletions.
diff --git a/build.gradle b/build.gradle
@@ -381,23 +381,75 @@ tasks.register("copyFilebeat", Copy){
     }
 }
 
+tasks.register("downloadEs", Download) {
+    dependsOn  configureArtifactInfo
+    description "Download ES Snapshot for current branch version: ${version}"
+
+    String downloadedElasticsearchName = "elasticsearch-${project.ext.get("stackArtifactSuffix")}"
+    project.ext.set("unpackedElasticsearchName", "elasticsearch-${project.ext.get("stackArtifactSuffixNoArch")}")
+
+    // find url of build artifact
+    String artifactApiUrl = "${project.ext.get("artifactApiVersionedBuildUrl")}/projects/elasticsearch/packages/${downloadedElasticsearchName}.tar.gz"
+    String apiResponse = artifactApiUrl.toURL().text
+    def buildUrls = new JsonSlurper().parseText(apiResponse)
+
+    project.ext.set("elasticsearchSnapshotURL", System.getenv("ELASTICSEARCH_SNAPSHOT_URL") ?: buildUrls["package"]["url"])
+    project.ext.set("elasticsearchDownloadLocation", "${projectDir}/build/${downloadedElasticsearchName}.tar.gz")
+
+    src project.ext.elasticsearchSnapshotURL
+    onlyIfNewer true
+    retries 3
+    inputs.file("${projectDir}/versions.yml")
+    outputs.file(project.ext.elasticsearchDownloadLocation)
+    dest new File(project.ext.elasticsearchDownloadLocation)
+
+    doLast {
+        System.out.println "Downloaded to ${project.ext.elasticsearchDownloadLocation}"
+    }
+}
+
+
+tasks.register("deleteLocalEs", Delete) {
+    delete ('./build/elasticsearch')
+}
+
+tasks.register("copyEs", Copy){
+    dependsOn = [downloadEs, deleteLocalEs]
+    from tarTree(resources.gzip(project.ext.elasticsearchDownloadLocation))
+    into "./build/"
+    doLast {
+        file("./build/${project.ext.unpackedElasticsearchName}").renameTo('./build/elasticsearch')
+        System.out.println "Unzipped ${project.ext.elasticsearchDownloadLocation} to ./build/elasticsearch"
+        System.out.println "Deleting ${project.ext.elasticsearchDownloadLocation}"
+        delete(project.ext.elasticsearchDownloadLocation)
+    }
+}
+
+
 def rubyIntegrationSpecs = project.hasProperty("rubyIntegrationSpecs") ? ((String) project.property("rubyIntegrationSpecs")).split(/\s+/).join(",") : "specs/**/*_spec.rb"
 def integrationTestPwd = "${projectDir}/qa/integration"
 
 project(":logstash-integration-tests") {
     tasks.getByPath(":logstash-integration-tests:integrationTests").configure {
-        systemProperty 'org.logstash.integration.specs', rubyIntegrationSpecs
-        environment "FEATURE_FLAG", System.getenv('FEATURE_FLAG')
-        workingDir integrationTestPwd
-        dependsOn installIntegrationTestGems
-    }
+      systemProperty 'org.logstash.integration.specs', rubyIntegrationSpecs
+      environment "FEATURE_FLAG", System.getenv('FEATURE_FLAG')
+      workingDir integrationTestPwd
+      dependsOn installIntegrationTestGems
+  }
 }
 
 tasks.register("runIntegrationTests"){
     dependsOn tasks.getByPath(":logstash-integration-tests:integrationTests")
+    dependsOn copyEs
     dependsOn copyFilebeat
 }
 
+bootstrap.dependsOn assemblyDeps
+
+runIntegrationTests.shouldRunAfter tasks.getByPath(":logstash-core:test")
+check.dependsOn runIntegrationTests
+
+
 tasks.register("generateLicenseReport", JavaExec) {
     dependsOn generateLicenseReportInputs
     dependsOn ":dependencies-report:assemble"
@@ -451,50 +503,6 @@ runIntegrationTests.shouldRunAfter tasks.getByPath(":logstash-core:test")
 check.dependsOn runIntegrationTests
 
 
-tasks.register("downloadEs", Download) {
-    dependsOn  configureArtifactInfo
-    description "Download ES Snapshot for current branch version: ${version}"
-
-    String downloadedElasticsearchName = "elasticsearch-${project.ext.get("stackArtifactSuffix")}"
-    project.ext.set("unpackedElasticsearchName", "elasticsearch-${project.ext.get("stackArtifactSuffixNoArch")}")
-
-    // find url of build artifact
-    String artifactApiUrl = "${project.ext.get("artifactApiVersionedBuildUrl")}/projects/elasticsearch/packages/${downloadedElasticsearchName}.tar.gz"
-    String apiResponse = artifactApiUrl.toURL().text
-    def buildUrls = new JsonSlurper().parseText(apiResponse)
-
-    project.ext.set("elasticsearchSnapshotURL", System.getenv("ELASTICSEARCH_SNAPSHOT_URL") ?: buildUrls["package"]["url"])
-    project.ext.set("elasticsearchDownloadLocation", "${projectDir}/build/${downloadedElasticsearchName}.tar.gz")
-
-    src project.ext.elasticsearchSnapshotURL
-    onlyIfNewer true
-    retries 3
-    inputs.file("${projectDir}/versions.yml")
-    outputs.file(project.ext.elasticsearchDownloadLocation)
-    dest new File(project.ext.elasticsearchDownloadLocation)
-
-    doLast {
-        System.out.println "Downloaded to ${project.ext.elasticsearchDownloadLocation}"
-    }
-}
-
-
-tasks.register("deleteLocalEs", Delete) {
-    delete ('./build/elasticsearch')
-}
-
-tasks.register("copyEs", Copy){
-    dependsOn = [downloadEs, deleteLocalEs]
-    from tarTree(resources.gzip(project.ext.elasticsearchDownloadLocation))
-    into "./build/"
-    doLast {
-        file("./build/${project.ext.unpackedElasticsearchName}").renameTo('./build/elasticsearch')
-        System.out.println "Unzipped ${project.ext.elasticsearchDownloadLocation} to ./build/elasticsearch"
-        System.out.println "Deleting ${project.ext.elasticsearchDownloadLocation}"
-        delete(project.ext.elasticsearchDownloadLocation)
-    }
-}
-
 Boolean oss = System.getenv('OSS').equals('true')
 
 if (!oss) {

diff --git a/qa/integration/services/elasticsearch_setup.sh b/qa/integration/services/elasticsearch_setup.sh
@@ -4,25 +4,7 @@ current_dir="$(dirname "$0")"
 
 source "$current_dir/helpers.sh"
 
-if [ -n "${ES_VERSION+1}" ]; then
-  echo "Elasticsearch version is $ES_VERSION"
-  version=$ES_VERSION
-else
-   version=6.5.4
-fi
-
-ES_HOME=$INSTALL_DIR/elasticsearch
-
-setup_es() {
-  if [ ! -d $ES_HOME ]; then
-      local version=$1
-      download_url=https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-$version.tar.gz
-      curl -sL $download_url > $INSTALL_DIR/elasticsearch.tar.gz
-      mkdir $ES_HOME
-      tar -xzf $INSTALL_DIR/elasticsearch.tar.gz --strip-components=1 -C $ES_HOME/.
-      rm $INSTALL_DIR/elasticsearch.tar.gz
-  fi
-}
+ES_HOME="$current_dir/../../../build/elasticsearch"
 
 start_es() {
   es_args=$@
@@ -38,7 +20,5 @@ start_es() {
   return 0
 }
 
-setup_install_dir
-setup_es $version
 export ES_JAVA_OPTS="-Xms512m -Xmx512m"
 start_es
diff --git a/qa/integration/services/elasticsearch_teardown.sh b/qa/integration/services/elasticsearch_teardown.sh
@@ -4,7 +4,7 @@ current_dir="$(dirname "$0")"
 
 source "$current_dir/helpers.sh"
 
-ES_HOME=$INSTALL_DIR/elasticsearch
+ES_HOME="$current_dir/../../../build/elasticsearch"
 
 stop_es() {
     pid=$(cat $ES_HOME/elasticsearch.pid)

diff --git a/qa/integration/specs/dlq_spec.rb b/qa/integration/specs/dlq_spec.rb
@@ -19,6 +19,8 @@
 require_relative '../framework/settings'
 require_relative '../services/logstash_service'
 require_relative '../framework/helpers'
+require_relative 'spec_helper.rb'
+
 require "logstash/devutils/rspec/spec_helper"
 
 describe "Test Dead Letter Queue" do
@@ -66,12 +68,11 @@
       try(60) do
         begin
           result = es_client.search(index: 'logstash-*', size: 0, q: '*')
-          hits = result["hits"]["total"]
         rescue Elasticsearch::Transport::Transport::Errors::ServiceUnavailable => e
           puts "Elasticsearch unavailable #{e.inspect}"
           hits = 0
         end
-        expect(hits).to eq(1000)
+        expect(result).to have_hits(1000)
       end
 
       result = es_client.search(index: 'logstash-*', size: 1, q: '*')

diff --git a/qa/integration/specs/es_output_how_spec.rb b/qa/integration/specs/es_output_how_spec.rb
@@ -18,6 +18,7 @@
 require_relative '../framework/fixture'
 require_relative '../framework/settings'
 require_relative '../services/logstash_service'
+require_relative 'spec_helper.rb'
 
 describe "Test Elasticsearch output" do
 
@@ -39,11 +40,11 @@
     # now we test if all data was indexed by ES, but first refresh manually
     es_client.indices.refresh
     result = es_client.search(index: 'logstash-*', size: 0, q: '*')
-    expect(result["hits"]["total"]).to eq(37)
+    expect(result).to have_hits(37)
 
     # randomly checked for results and structured fields
     result = es_client.search(index: 'logstash-*', size: 1, q: 'dynamic')
-    expect(result["hits"]["total"]).to eq(1)
+    expect(result).to have_hits(1)
     s = result["hits"]["hits"][0]["_source"]
     expect(s["bytes"]).to eq(18848)
     expect(s["response"]).to eq(200)

diff --git a/qa/integration/specs/spec_helper.rb b/qa/integration/specs/spec_helper.rb
@@ -22,3 +22,10 @@
 
   config.filter_run_excluding exclude_tags
 end
+
+RSpec::Matchers.define :have_hits do |expected|
+  match do |actual|
+    # For Elasticsearch versions 7+, the result is in a value field, just in total for > 6
+    expected == actual['hits']['total'].is_a?(Hash) ? actual['hits']['total']['value'] : actual['hits']['total']
+  end
+end