Merge branch 'master' of github.com:logstash/logstash

CHANGELOG

@@ -3,7 +3,7 @@
   * New input plugins: generator, heroku, pipe, ganglia
   * New output plugins: juggernaut, metricscatcher, nagios_ncsa, pipe,
                         opentsdb
-  * New filter plugins: zeromq, environment, xml, csv
+  * New filter plugins: zeromq, environment, xml, csv, syslog_pri
   * Fixes for gelf output
 
  ## IMPORTANT CHANGES FOR UPGRADES FROM 1.1.0
@@ -19,6 +19,12 @@
   - elasticsearch version is now 0.19.4
       This means your elasticsearch cluster must be running 0.19.x for
       compatibility reasons.
+  - grok pattern %{POSINT} used to match '0' -- now it does not. If you want
+    to match non-negative integers, there is now a %{NONNEGINT} pattern.
+  - bug in file input fixed that led to an extra leading slash in @source_path.
+    Previously, file input would have @source = 'file://host//var/log/foo' and
+    @source_path = '//var/log/foo'; now @source = 'file://host/var/log/foo'
+    and @source_path = '/var/log/foo'. [LOGSTASH-501]
 
   ## general
   - NOTE: gemspec removed; deploying logstash as a gem hasn't been supported

CONTRIBUTORS

@@ -44,6 +44,8 @@ Contributors:
 * Patrick Debois (jedi4ever)
 * bodik
 * Philippe Weber
+* Marc Huffnagle (mhuffnagle)
+* Oliver Gorwits (ollyg)
 
 
 Note: If you've sent me patches, bug reports, or otherwise contributed to

lib/logstash/filters/grep.rb

@@ -25,7 +25,8 @@ class LogStash::Filters::Grep < LogStash::Filters::Base
   # through.
   config :negate, :validate => :boolean, :default => false
 
-  # A hash of matches of field => regexp
+  # A hash of matches of field => regexp.  If multiple matches are specified,
+  # all must match for the grep to be considered successful.
   # Normal regular expressions are supported here.
   config :match, :validate => :hash, :default => {}
 

lib/logstash/filters/grok.rb

@@ -19,7 +19,6 @@ class LogStash::Filters::Grok < LogStash::Filters::Base
   # Multiple patterns is fine.
   config :pattern, :validate => :array
 
-  # Specify a path to a directory with grok pattern files in it
   # A hash of matches of field => value
   config :match, :validate => :hash, :default => {}
 

lib/logstash/filters/syslog_pri.rb

@@ -0,0 +1,106 @@
+require "logstash/filters/base"
+require "logstash/namespace"
+
+# Filter plugin for logstash to parse the PRI field from the front
+# of a Syslog (RFC3164) message.  If no priority is set, it will
+# default to 13 (per RFC).
+#
+# This filter is based on the original syslog.rb code shipped
+# with logstash.
+class LogStash::Filters::Syslog_pri < LogStash::Filters::Base
+  config_name "syslog_pri"
+
+  # set the status to experimental/beta/stable
+  plugin_status "experimental"
+
+  # Add human-readable names after parsing severity and facility from PRI
+  config :use_labels, :validate => :boolean, :default => true
+
+  # Name of field which passes in the extracted PRI part of the syslog message
+  config :syslog_pri_field_name, :validate => :string, :default => "syslog_pri"
+
+  # Labels for facility levels. This comes from RFC3164.
+  config :facility_labels, :validate => :array, :default => [
+    "kernel",
+    "user-level",
+    "mail",
+    "daemon",
+    "security/authorization",
+    "syslogd",
+    "line printer",
+    "network news",
+    "uucp",
+    "clock",
+    "security/authorization",
+    "ftp",
+    "ntp",
+    "log audit",
+    "log alert",
+    "clock",
+    "local0",
+    "local1",
+    "local2",
+    "local3",
+    "local4",
+    "local5",
+    "local6",
+    "local7"
+  ]
+
+  # Labels for severity levels. This comes from RFC3164.
+  config :severity_labels, :validate => :array, :default => [
+    "emergency",
+    "alert",
+    "critical",
+    "error",
+    "warning",
+    "notice",
+    "informational",
+    "debug"
+  ]
+
+  public
+  def register
+    # Nothing
+  end # def register
+
+  public
+  def filter(event)
+    return unless filter?(event)
+    parse_pri(event)
+    filter_matched(event)
+  end # def filter
+
+  private
+  def parse_pri(event)
+    # Per RFC3164, priority = (facility * 8) + severity
+    # = (facility << 3) & (severity)
+    if event.fields[@syslog_pri_field_name]
+      if event.fields[@syslog_pri_field_name].is_a?(Array)
+        priority = event.fields[@syslog_pri_field_name].first.to_i
+      else
+        priority = event.fields[@syslog_pri_field_name].to_i
+      end
+    else
+      priority = 13  # default
+    end
+    severity = priority & 7 # 7 is 111 (3 bits)
+    facility = priority >> 3
+    event.fields["syslog_severity_code"] = severity
+    event.fields["syslog_facility_code"] = facility
+
+    # Add human-readable names after parsing severity and facility from PRI
+    if @use_labels
+      facility_number = event.fields["syslog_facility_code"]
+      severity_number = event.fields["syslog_severity_code"]
+
+      if @facility_labels[facility_number]
+        event.fields["syslog_facility"] = @facility_labels[facility_number]
+      end
+
+      if @severity_labels[severity_number]
+        event.fields["syslog_severity"] = @severity_labels[severity_number]
+      end
+    end
+  end # def parse_pri
+end # class LogStash::Filters::SyslogPRI

lib/logstash/inputs/file.rb

@@ -1,6 +1,7 @@
 require "logstash/inputs/base"
 require "logstash/namespace"
 require "socket" # for Socket.gethostname
+require "uri"
 
 # Stream events from files.
 #
@@ -67,7 +68,7 @@ def run(queue)
     hostname = Socket.gethostname
 
     tail.subscribe do |path, line|
-      source = "file://#{hostname}/#{path}"
+      source = URI::Generic.new("file", nil, hostname, nil, nil, path, nil, nil, nil).to_s
       @logger.debug("Received line", :path => path, :line => line)
       e = to_event(line, source)
       if e

patterns/grok-patterns

@@ -6,7 +6,8 @@ NUMBER (?:%{BASE10NUM})
 BASE16NUM (?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+))
 BASE16FLOAT \b(?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\.[0-9A-Fa-f]*)?)|(?:\.[0-9A-Fa-f]+)))\b
 
-POSINT \b(?:[0-9]+)\b
+POSINT \b(?:[1-9][0-9]*)\b
+NONNEGINT \b(?:[0-9]+)\b
 WORD \b\w+\b
 NOTSPACE \S+
 SPACE \s*
@@ -31,7 +32,7 @@ HOSTPORT (?:%{IPORHOST=~/\./}:%{POSINT})
 PATH (?:%{UNIXPATH}|%{WINPATH})
 UNIXPATH (?:/(?:[\w_%!$@:.,-]+|\\.)*)+
 #UNIXPATH (?<![\w\/])(?:/[^\/\s?*]*)+
-LINUXTTY (?:/dev/pts/%{POSINT})
+LINUXTTY (?:/dev/pts/%{NONNEGINT})
 BSDTTY (?:/dev/tty[pq][a-z0-9])
 TTY (?:%{BSDTTY}|%{LINUXTTY})
 WINPATH (?:[A-Za-z]+:|\\)(?:\\[^\\?*]*)+

test/logstash/filters/test_syslog_pri.rb

@@ -0,0 +1,117 @@
+require "rubygems"
+require File.join(File.dirname(__FILE__), "..", "minitest")
+
+require "logstash/loadlibs"
+require "logstash"
+require "logstash/filters"
+require "logstash/filters/syslog_pri"
+require "logstash/event"
+
+describe LogStash::Filters::Syslog_pri do
+  before do
+    @filter = LogStash::Filters.from_name("syslog_pri", {})
+    @typename = "syslog_pri"
+  end
+
+  def config(cfg)
+    cfg["type"] = @typename
+    cfg.each_key do |key|
+      if cfg[key].is_a?(String)
+        cfg[key] = [cfg[key]]
+      end
+    end
+
+    @filter = LogStash::Filters::Syslog_pri.new(cfg)
+    @filter.register
+  end # def config
+
+  test "severity" do
+    event = LogStash::Event.new
+    event.type = @typename
+    event["syslog_pri"] = 75
+    @filter.filter(event)
+    assert_equal(3, event["syslog_severity_code"])
+  end
+
+  test "severity field name" do
+    event = LogStash::Event.new
+    event.type = @typename
+    event["syslog_pri"] = 75
+    @filter.filter(event)
+    assert_equal("error", event["syslog_severity"])
+  end
+
+  test "facility" do
+    event = LogStash::Event.new
+    event.type = @typename
+    event["syslog_pri"] = 75
+    @filter.filter(event)
+    assert_equal(9, event["syslog_facility_code"])
+  end
+
+  test "facility field name" do
+    event = LogStash::Event.new
+    event.type = @typename
+    event["syslog_pri"] = 75
+    @filter.filter(event)
+    assert_equal("clock", event["syslog_facility"])
+  end
+
+  test "no field names added with use_labels=false" do
+    config "use_labels" => "false"
+
+    event = LogStash::Event.new
+    event.type = @typename
+    event["syslog_pri"] = 75
+    @filter.filter(event)
+    assert_equal(nil, event["syslog_facility"])
+    assert_equal(nil, event["syslog_severity"])
+  end
+
+  test "default priority of 13" do
+    event = LogStash::Event.new
+    event.type = @typename
+    @filter.filter(event)
+    assert_equal(1, event["syslog_facility_code"])
+    assert_equal(5, event["syslog_severity_code"])
+  end
+
+  test "priority as array" do
+    event = LogStash::Event.new
+    event.type = @typename
+    event["syslog_pri"] = [75]
+    @filter.filter(event)
+    assert_equal(9, event["syslog_facility_code"])
+    assert_equal(3, event["syslog_severity_code"])
+  end
+
+  test "set priority field name" do
+    config "syslog_pri_field_name" => "syslog_pri2"
+
+    event = LogStash::Event.new
+    event.type = @typename
+    event["syslog_pri"] = 15
+    event["syslog_pri2"] = 75
+    @filter.filter(event)
+    assert_equal(9, event["syslog_facility_code"])
+    assert_equal(3, event["syslog_severity_code"])
+  end
+
+  test "custom facility labels" do
+    config "facility_labels" => ["a", "b", "c", "d", "e", "f"]
+
+    event = LogStash::Event.new
+    event.type = @typename
+    @filter.filter(event)
+    assert_equal("b", event["syslog_facility"])
+  end
+
+  test "custom severity labels" do
+    config "severity_labels" => ["a", "b", "c", "d", "e", "f"]
+
+    event = LogStash::Event.new
+    event.type = @typename
+    @filter.filter(event)
+    assert_equal("f", event["syslog_severity"])
+  end
+end # Test 'syslog_pri' filter

test/logstash/inputs/test_file.rb

@@ -0,0 +1,44 @@
+require "rubygems"
+require File.join(File.dirname(__FILE__), "..", "minitest")
+
+require "logstash/loadlibs"
+require "logstash/testcase"
+require "logstash/agent"
+require "logstash/logging"
+require "logstash/inputs/file"
+
+require "tempfile"
+
+describe LogStash::Inputs::File do
+  test "file input sets source_path properly for events" do
+    logfile = Tempfile.new("logstash")
+    begin
+      @input = LogStash::Inputs::File.new("type" => ["testing"], "path" => [logfile.path])
+      @input.register
+
+      queue = Queue.new
+
+      Thread.new { @input.run(queue) }
+
+      event = nil
+      while event.nil?
+        logfile.write("This is my log message.\n")
+        logfile.flush
+
+        begin
+          event = queue.pop(true)
+        rescue ThreadError => error
+          raise error unless error.to_s == "queue empty"
+          sleep(0.05)
+        end
+      end
+
+      @input.teardown
+
+      assert_equal(logfile.path, event["@source_path"])
+    ensure
+      logfile.close
+      logfile.unlink
+    end
+  end
+end # testing for LogStash::Inputs::File

test/logstash/inputs/test_gelf.rb

@@ -53,4 +53,4 @@
     assert_equal("Hello world", events.last.message)
     assert_equal("bar", events.last.fields["foo"])
   end # test gelf input defaults
-end # testing for LogStash::Outputs::ElasticSearch
+end # testing for LogStash::Inputs::Gelf