Merge pull request elastic#383 from electrical/kv_filter

Kv filter

lib/logstash/filters/kv.rb

@@ -27,7 +27,7 @@ class LogStash::Filters::KV < LogStash::Filters::Base
   plugin_status "beta"
 
   # The fields to perform 'key=value' searching on
-  config :fields, :validate => :array, :default => ["@message"]
+  config :fields, :validate => :array
 
   # A string of characters to trim from the value. This is useful if your
   # values are wrapped in brackets or are terminated by comma (like postfix
@@ -87,17 +87,50 @@ class LogStash::Filters::KV < LogStash::Filters::Base
   # Example, to place all keys into container kv:
   #
   #     filter { kv { container => "kv" } }
-  config :container, :validate => :string, :default => '@fields'
+  config :container, :validate => :string, :deprecated => true
+
+  # The fields to perform 'key=value' searching on
+  #
+  # Example, to use the @message field:
+  #
+  #     filter { kv { source => "@message" } }
+  config :source, :validate => :string, :default => '@message'
+
+  # The name of the container to put all of the key-value pairs into 
+  #
+  # Example, to place all keys into field kv:
+  #
+  #     filter { kv { target => "kv" } }
+  config :target, :validate => :string, :default => '@fields'
 
   def register
     @trim_re = Regexp.new("[#{@trim}]") if !@trim.nil?
+
+    #TODO(electrical): Remove this when removing the container variable
+    if @container
+      if @target
+        logger.error("'container' and 'target' are the same setting, but 'container' is deprecated. Please use only 'target'")
+      end
+      @target = @container
+    end
+
+    #TODO(electrical): Remove this when removing the fields variable
+    if @source
+      if @fields
+        logger.error("'fields' and 'source' are the same setting, but 'fields' is deprecated. Please use only 'source'")
+      end
+      @fields=Array.new if @fields.nil?
+      @fields << @source
+    end
+
   end # def register
 
   def filter(event)
     return unless filter?(event)
 
     kv_keys=Hash.new
 
+    #TODO(electrical): Remove this loop when we remove the fields variable
     @fields.each do |fieldname|
       value = event[fieldname]
 
@@ -111,10 +144,10 @@ def filter(event)
     end
     # If we have any keys, create/append the hash
     if kv_keys.length > 0
-      if !event[@container].nil?
-        event[@container].merge!(kv_keys)
+      if !event[@target].nil?
+        event[@target].merge!(kv_keys)
       else
-        event[@container] = kv_keys
+        event[@target]= kv_keys
       end
       filter_matched(event)
     end

spec/filters/kv.rb

@@ -106,7 +106,7 @@
 
   end
 
-  describe "test container" do
+  describe "test container (deprecated test)" do
     config <<-CONFIG
       filter {
         kv { container => 'kv' }
@@ -124,7 +124,7 @@
 
   end
 
-  describe "test empty container" do
+  describe "test empty container (deprecated test)" do
     config <<-CONFIG
       filter {
         kv { container => 'kv' }
@@ -215,4 +215,94 @@
       end
     end
   end
+
+  #New tests
+  describe "test target" do
+    config <<-CONFIG
+      filter {
+        kv { target => 'kv' }
+      }
+    CONFIG
+
+    sample "hello=world foo=bar baz=fizz doublequoted=\"hello world\" singlequoted='hello world'" do
+      insist { subject["kv"]["hello"] } == "world"
+      insist { subject["kv"]["foo"] } == "bar"
+      insist { subject["kv"]["baz"] } == "fizz"
+      insist { subject["kv"]["doublequoted"] } == "hello world"
+      insist { subject["kv"]["singlequoted"] } == "hello world"
+      insist {subject['@fields']["kv"].count } == 5
+    end
+
+  end
+
+  describe "test empty target" do
+    config <<-CONFIG
+      filter {
+        kv { target => 'kv' }
+      }
+    CONFIG
+
+    sample "hello:world:foo:bar:baz:fizz" do
+      insist { subject["kv"] } == nil
+      insist {subject['@fields'].count } == 0
+    end
+  end
+
+
+  describe "test data from specific sub source" do
+    config <<-CONFIG
+      filter {
+        kv {
+          source => "data"
+        }
+      }
+    CONFIG
+    sample({"@fields" => {"data" => "hello=world foo=bar baz=fizz doublequoted=\"hello world\" singlequoted='hello world'"}}) do
+      insist { subject["hello"] } == "world"
+      insist { subject["foo"] } == "bar"
+      insist { subject["baz"] } == "fizz"
+      insist { subject["doublequoted"] } == "hello world"
+      insist { subject["singlequoted"] } == "hello world"
+      insist { subject['@fields'].count } == 6
+    end
+  end
+
+  describe "test data from specific top source" do
+    config <<-CONFIG
+      filter {
+        kv {
+          source => "@data"
+        }
+      }
+    CONFIG
+    sample({"@data" => "hello=world foo=bar baz=fizz doublequoted=\"hello world\" singlequoted='hello world'"}) do
+      insist { subject["hello"] } == "world"
+      insist { subject["foo"] } == "bar"
+      insist { subject["baz"] } == "fizz"
+      insist { subject["doublequoted"] } == "hello world"
+      insist { subject["singlequoted"] } == "hello world"
+      insist { subject['@fields'].count } == 5
+    end
+  end
+
+
+  describe "test data from specific sub source and target" do
+    config <<-CONFIG
+      filter {
+        kv {
+          source => "data"
+          target => "kv"
+        }
+      }
+    CONFIG
+    sample({"@fields" => {"data" => "hello=world foo=bar baz=fizz doublequoted=\"hello world\" singlequoted='hello world'"}}) do
+      insist { subject["kv"]["hello"] } == "world"
+      insist { subject["kv"]["foo"] } == "bar"
+      insist { subject["kv"]["baz"] } == "fizz"
+      insist { subject["kv"]["doublequoted"] } == "hello world"
+      insist { subject["kv"]["singlequoted"] } == "hello world"
+      insist { subject['@fields']["kv"].count } == 5
+    end
+  end
+
 end