Merge branch 'master' into aws-iam-roles

CHANGELOG

@@ -1,12 +1,27 @@
-1.1.9 ( January X, 2013 )
+NEXT (?????)
+  ## filters
+  - bugfix: mutate: skip missing fields in 'convert' (#244, patch by Ralph Meijer)
+
   ## outputs
- - fix bug in elasticsearch_river where it wouldn't resolve %{} variables in index 
-   and changed index -> index_type in ES header. (LOGSTASH-819)
+  - improvement: gelf: new tunable 'ignore_metadata' flag to set which fields
+    to ignore if ship_metadata is set. (#244, patch by Ralph Meijer)
+  - improvement: gelf: make short_message's field name tunable (#244, patch by
+    Ralph Meijer)
 
-1.1.8 (January 3, 2013)
+1.1.8 (January 10, 2013)
   ## general
   - patched another work around for JRUBY-6970 (LOGSTASH-801)
 
+  ## inputs
+  - bugfix: tcp: 'Address in use' errors now report the host/port involved.
+    (LOGSTASH-831)
+  - bugfix: zeromq: fix bug where an invalid url could be given as a source
+    (LOGSTASH-821, #306)
+
+  ## outputs
+  - bugfix: elasticsearch_river: it now resolves evaluates %{} variables in
+    index and index_type settings. (LOGSTASH-819)
+
 1.1.7 (January 3, 2013)
  ## inputs
  - fix bug where @source_host was set to 'false' in many cases.

lib/logstash/filters/mutate.rb

@@ -219,7 +219,9 @@ def convert(event)
 
       # calls convert_{string,integer,float} depending on type requested.
       converter = method("convert_" + type)
-      if original.is_a?(Hash)
+      if original.nil?
+        next
+      elsif original.is_a?(Hash)
         @logger.debug("I don't know how to type convert a hash, skipping",
                       :field => field, :value => original)
         next

lib/logstash/inputs/tcp.rb

@@ -42,7 +42,13 @@ def initialize(*args)
   def register
     if server?
       @logger.info("Starting tcp input listener", :address => "#{@host}:#{@port}")
-      @server_socket = TCPServer.new(@host, @port)
+      begin
+        @server_socket = TCPServer.new(@host, @port)
+      rescue Errno::EADDRINUSE
+        @logger.error("Could not start TCP server: Address in use",
+                      :host => @host, :port => @port)
+        raise
+      end
     end
   end # def register
 

lib/logstash/inputs/zeromq.rb

@@ -1,6 +1,7 @@
 require "logstash/inputs/base"
 require "logstash/namespace"
 require "timeout"
+require "socket"
 
 # Read events over a 0MQ SUB socket.
 #
@@ -119,6 +120,7 @@ def server?
   end # def server?
 
   def run(output_queue)
+    host = Socket.gethostname
     begin
       loop do
         # Here's the unified receiver
@@ -138,7 +140,7 @@ def run(output_queue)
           @logger.debug("ZMQ receiving", :event => m2)
           msg = m2
         end
-        @sender ||= "zmq+#{@topology}://#{@type}/"
+        @sender ||= "zmq+#{@topology}://#{host}/#{@type}"
         e = self.to_event(msg, @sender)
         if e
           output_queue << e

lib/logstash/outputs/gelf.rb

@@ -53,6 +53,10 @@ class LogStash::Outputs::Gelf < LogStash::Outputs::Base
   # messages.
   config :ship_metadata, :validate => :boolean, :default => true
 
+  # Ignore these fields when ship_metadata is set. Typically this lists the
+  # fields used in dynamic values for GELF fields.
+  config :ignore_metadata, :validate => :array, :default => [ "severity", "source_host", "source_path", "short_message" ]
+
   # The GELF custom field mappings. GELF supports arbitrary attributes as custom
   # fields. This exposes that. Exclude the `_` portion of the field name
   # e.g. `custom_fields => ['foo_field', 'some_value']
@@ -62,12 +66,14 @@ class LogStash::Outputs::Gelf < LogStash::Outputs::Base
   # The GELF full message. Dynamic values like %{foo} are permitted here.
   config :full_message, :validate => :string, :default => "%{@message}"
 
+  # The GELF short message field name. If the field does not exist or is empty,
+  # the event message is taken instead.
+  config :short_message, :validate => :string, :default => "short_message"
+
   public
   def register
     require "gelf" # rubygem 'gelf'
     option_hash = Hash.new
-    #option_hash['level'] = @level
-    #option_hash['facility'] = @facility
 
     #@gelf = GELF::Notifier.new(@host, @port, @chunksize, option_hash)
     @gelf = GELF::Notifier.new(@host, @port, @chunksize)
@@ -103,8 +109,6 @@ def register
       "alert" => 1, "a" => 1,
       "emergency" => 0, "e" => 0,
      }
-
-     @ignore_fields = [ "facility", "full_message", "short_message", "host", "level", "line", "timestamp", "version", "file" ]
   end # def register
 
   public
@@ -114,11 +118,15 @@ def receive(event)
     # We have to make our own hash here because GELF expects a hash
     # with a specific format.
     m = Hash.new
-    if event.fields["short_message"]
-      v = event.fields["short_message"]
-      m["short_message"] = (v.is_a?(Array) && v.length == 1) ? v.first : v
-    else
-      m["short_message"] = event.message
+
+    m["short_message"] = event.message
+    if event.fields[@short_message]
+      v = event.fields[@short_message]
+      short_message = (v.is_a?(Array) && v.length == 1) ? v.first : v
+      short_message = short_message.to_s
+      if !short_message.empty?
+        m["short_message"] = short_message
+      end
     end
 
     m["full_message"] = event.sprintf(@full_message)
@@ -135,7 +143,7 @@ def receive(event)
         # Trim leading '_' in the event
         name = name[1..-1] if name.start_with?('_')
         name = "_id" if name == "id"  # "_id" is reserved, so use "__id"
-        if !value.nil? and !@ignore_fields.include?(name)
+        if !value.nil? and !@ignore_metadata.include?(name)
           if value.is_a?(Array)
             # collapse single-element arrays, otherwise leave as array
             m["_#{name}"] = (value.length == 1) ? value.first : value

lib/logstash/version.rb

@@ -1,5 +1,5 @@
 # The version of logstash.
-LOGSTASH_VERSION = "1.1.8.dev"
+LOGSTASH_VERSION = "1.1.9.dev"
 
 # Note to authors: this should not include dashes because 'gem' barfs if
 # you include a dash in the version string.

patterns/grok-patterns

@@ -55,7 +55,7 @@ MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])
 DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)
 
 # Years?
-YEAR [0-9]+
+YEAR (?>\d\d){1,2}
 # Time: HH:MM:SS
 #TIME \d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?
 # I'm still on the fence about using grok to perform the time match,