Add integration test for Elasticsearch using defaults (elastic#13353)

* Add integration test for Elasticsearch using defaults Currently, the elasticsearch integration test turns off `data_streams` and `ecs_compatibility`, hiding the out of the box experience for ingesting into elasticsearch * Removed debugging statement
AsherBond · Jan 20, 2022 · 57c16e1 · 57c16e1
1 parent ca501ac
commit 57c16e1
Show file tree

Hide file tree

Showing 2 changed files with 77 additions and 27 deletions.
diff --git a/qa/integration/fixtures/es_output_how_spec.yml b/qa/integration/fixtures/es_output_how_spec.yml
@@ -2,39 +2,70 @@
 services:
   - logstash
   - elasticsearch
-config: |-
-  input {
-    stdin { ecs_compatibility => disabled }
-  }
+config:
+  default: |-
+    input {
+      stdin { }
+    }
+
+    filter {
+      grok {
+        match => {
+          "message" => "%{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] \"%{WORD:verb} %{DATA:request} HTTP/%{NUMBER:httpversion}\" %{NUMBER:response:int} (?:-|%{NUMBER:bytes:int}) %{QS:referrer} %{QS:agent}"
+        }
+      }
 
-  filter {
-    grok {
-      match => {
-        "message" => "%{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] \"%{WORD:verb} %{DATA:request} HTTP/%{NUMBER:httpversion}\" %{NUMBER:response:int} (?:-|%{NUMBER:bytes:int}) %{QS:referrer} %{QS:agent}"
+      date {
+        match => [ "timestamp", "dd/MMM/YYYY:HH:mm:ss Z" ]
+        locale => en
+      }
+      geoip {
+        ecs_compatibility => disabled
+        source => "clientip"
+      }
+      useragent {
+        ecs_compatibility => disabled
+        source => "agent"
+        target => "useragent"
       }
     }
 
-    date {
-      match => [ "timestamp", "dd/MMM/YYYY:HH:mm:ss Z" ]
-      locale => en
+    output {
+      elasticsearch {}
     }
-    geoip {
-      ecs_compatibility => disabled
-      source => "clientip"
+  ds_ecs_off: |-
+    input {
+      stdin { ecs_compatibility => disabled }
     }
-    useragent {
-      ecs_compatibility => disabled
-      source => "agent"
-      target => "useragent"
+
+    filter {
+      grok {
+        match => {
+          "message" => "%{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] \"%{WORD:verb} %{DATA:request} HTTP/%{NUMBER:httpversion}\" %{NUMBER:response:int} (?:-|%{NUMBER:bytes:int}) %{QS:referrer} %{QS:agent}"
+        }
+      }
+
+      date {
+        match => [ "timestamp", "dd/MMM/YYYY:HH:mm:ss Z" ]
+        locale => en
+      }
+      geoip {
+        ecs_compatibility => disabled
+        source => "clientip"
+      }
+      useragent {
+        ecs_compatibility => disabled
+        source => "agent"
+        target => "useragent"
+      }
     }
-  }
-  output {
-    elasticsearch {
-      data_stream => "false"
-      ecs_compatibility => disabled
-      index => "logstash-integration-test"
+    output {
+      elasticsearch {
+        data_stream => "false"
+        ecs_compatibility => disabled
+        index => "logstash-integration-test"
+      }
     }
-  }
 
 input: how_sample.input
 teardown_script:
diff --git a/qa/integration/specs/es_output_how_spec.rb b/qa/integration/specs/es_output_how_spec.rb
@@ -32,10 +32,29 @@
     @fixture.teardown
   }
 
-  it "can ingest 37K log lines of sample apache logs" do
+  it "can ingest 37K log lines of sample apache logs with default settings" do
     logstash_service = @fixture.get_service("logstash")
     es_service = @fixture.get_service("elasticsearch")
-    logstash_service.start_with_input(@fixture.config, @fixture.input)
+    logstash_service.start_with_input(@fixture.config("default"), @fixture.input)
+    es_client = es_service.get_client
+    # now we test if all data was indexed by ES, but first refresh manually
+    es_client.indices.refresh
+    result = es_client.search(index: '.ds-logs-*', size: 0, q: '*')
+    expect(result).to have_hits(37)
+
+    # randomly checked for results and structured fields
+    result = es_client.search(index: '.ds-logs-*', size: 1, q: 'dynamic')
+    expect(result).to have_hits(1)
+    s = result["hits"]["hits"][0]["_source"]
+    expect(s["bytes"]).to eq(18848)
+    expect(s["response"]).to eq(200)
+    expect(s["clientip"]).to eq("213.113.233.227")
+  end
+
+  it "can ingest 37K log lines of sample apache logs with ecs and data streams off" do
+    logstash_service = @fixture.get_service("logstash")
+    es_service = @fixture.get_service("elasticsearch")
+    logstash_service.start_with_input(@fixture.config("ds_ecs_off"), @fixture.input)
     es_client = es_service.get_client
     # now we test if all data was indexed by ES, but first refresh manually
     es_client.indices.refresh