Merge remote-tracking branch 'upstream/master'

CHANGELOG

@@ -4,6 +4,13 @@
     threads in tools like top(1).
   - Java 5 is no longer supported (You must use Java 6 or newer).
 
+  ## inputs
+  - improvement: loggly: supports http proxying now (#276, patch by Richard
+    Pijnenburg)
+  - improvement: twitter: supports http proxying now (#276, patch by Richard
+    Pijnenburg)
+  - improvement: tcp: ssl now supported! (#318, patch by Matthew Richardson)
+
   ## filters
   - deprecation: the --grok-patterns-path flag is deprecated and will now
     warn you if you use it. (LOGSTASH-803)
@@ -13,7 +20,11 @@
   ## outputs
   - fix bug in mongo output that would fail to load bson_java support
     (LOGSTASH-849)
-
+  - Added tag support to gelf output. Returns tags as _tags field (LOGSTASH-880)
+  - bugfix: elasticsearch: Fix a race condition. (#340, patch by Raymond Feng)
+  - improvement: http: now supports a custom 'message' format for building your
+    own http bodies from an event. (#319, patch by Christian S)
+
 1.1.9 (January 10, 2013)
   ## inputs
   - bugfix: all inputs: fix bug where some @source values were not valid urls

Makefile

@@ -125,6 +125,10 @@ vendor/bundle: | vendor $(JRUBY)
 	@echo "=> Installing gems to $@..."
 	@#$(QUIET)GEM_HOME=$(GEM_HOME) $(JRUBY_CMD) --1.9 $(GEM_HOME)/bin/bundle install --deployment
 	$(QUIET)GEM_HOME=./vendor/bundle/jruby/1.9/ GEM_PATH= $(JRUBY_CMD) --1.9 ./gembag.rb logstash.gemspec
+	@# rubygems is rejecting pushes right now, so use a temporary locatin for lumberjack
+	$(QUIET)GEM_HOME=./vendor/bundle/jruby/1.9/ GEM_PATH= $(JRUBY_CMD) --1.9 -S gem uninstall jls-lumberjack
+	$(QUIET)rm jls-lumberjack*gem; wget http://jls.objects.dreamhost.com/gems/lumberjack/jls-lumberjack-0.0.16.gem
+	$(QUIET)GEM_HOME=./vendor/bundle/jruby/1.9/ GEM_PATH= $(JRUBY_CMD) --1.9 -S gem install --local jls-lumberjack-0.0.16.gem
 	@# Purge old version of json
 	#$(QUIET)GEM_HOME=./vendor/bundle/jruby/1.9/ GEM_PATH= $(JRUBY_CMD) --1.9 -S gem uninstall json -v 1.6.5
 	@# Purge old versions of gems installed because gembag doesn't do 

README.md

@@ -46,6 +46,11 @@ build with [fpm](https://github.com/jordansissel/fpm). If you are interested in
 seeing future releases include your favorite packaging format, please let me
 know.
 
+## Project Principles
+
+* Software: Make it work, then make it right, then make it fast.
+* Community: If a newbie has a bad time, it's a bug.
+
 ## Contributing
 
 All contributions are welcome: ideas, patches, documentation, bug reports,

lib/logstash/agent2.rb

@@ -0,0 +1,61 @@
+require "logstash/config/file"
+require "clamp"
+
+class LogStash::Agent2 < Clamp::Command
+  option ["-f", "--config"], "CONFIG_PATH",
+    "Load the logstash config from a specific file or directory. " \
+    "If a direcory is given, all files in that directory will " \
+    "be concatonated in lexicographical order and then parsed as " \
+    "a single config file. You can also specify wildcards (globs)" \
+    "and any matched files will be loaded in the order described above",
+    :attribute_name => :config_path
+
+  option "-e", "CONFIG_STRING",
+    "Use the given string as the configuration data. Same syntax as " \
+    "the config file. If not input is specified, then " \
+    "'stdin { type => stdin }' is the default input. If no output is " \
+    "specified, then 'stdout { debug => true }}' is default output.",
+    :attribute_name => :config_string
+
+  option ["-w", "--filterworkers"], "COUNT",
+    "Sets the number of filter workers to run.",
+    :attribute_name => :filter_workers, :default => 1, &:to_i
+
+  option "--watchdog-timeout", "SECONDS", 
+    "Set the filter watchdog timeout (in seconds). This timeout is used" \
+    " to detect stuck filters; stuck filters usually symptoms of bugs. " \
+    "When a filter takes longer than TIMEOUT seconds, it will cause " \
+    "logstash to abort.", :default => 10, &:to_f
+
+  option ["-l", "--log"], "FILE",
+    "Write logstash internal logs to the given file. Without this flag, " \
+    "logstash will emit logs to standard output."
+
+  verbosity = 0
+  option "-v", :flag, "Increase verbosity of logstash internal logs. " \
+    "Specifying once will show 'informational' logs. Specifying twice " \
+    "will show 'debug' logs.", :attribute_name => :verbosity do
+    verbosity += 1
+  end
+
+  option ["-V", "--version"], :flag,
+    "Emit the version of logstash and its friends" do
+    # TODO(sissel): This should emit the version of JRuby and ElasticSearch as
+    # well. Perhaps also the versions of all gems?
+    require "logstash/version"
+    puts "logstash #{LOGSTASH_VERSION}"
+    exit(0)
+  end
+
+  plugin_paths = []
+  option ["-p", "--pluginpath"] , "PATH",
+    "A colon-delimited path of where to find plugins. Plugins are expected " \
+    "to be in a specific directory hierarchy: PATH/logstash/TYPE/NAME.rb - " \
+    "where TYPE is 'input' 'filter' or 'output' and NAME is the name of the" \
+    "plugin.", :attribute_name => :plugin_path  do |value|
+    plugin_paths << value unless plugin_paths.include?(value)
+  end
+
+  def execute
+  end # def execute
+end # class LogStash::Agent2

lib/logstash/config/file.rb

@@ -1,7 +1,6 @@
 require "logstash/namespace"
 require "logstash/config/grammar"
 require "logstash/config/registry"
-require "logstash/agent"
 require "logger"
 
 class LogStash::Config::File

lib/logstash/filters/alter.rb

@@ -33,7 +33,7 @@ class LogStash::Filters::Alter < LogStash::Filters::Base
   # Example:
   #
   #     filter {
-  #       alter => {
+  #       alter {
   #         condrewriteother => [ 
   #              "field_name", "expected_value", "field_name_to_change", "value",
   #              "field_name2", "expected_value2, "field_name_to_change2", "value2",
@@ -48,7 +48,7 @@ class LogStash::Filters::Alter < LogStash::Filters::Base
   # Example:
   #
   #     filter {
-  #       alter => {
+  #       alter {
   #         coalesce => [
   #              "field_name", "value1", "value2", "value3", ...
   #         ]

lib/logstash/filters/csv.rb

@@ -14,32 +14,66 @@ class LogStash::Filters::CSV < LogStash::Filters::Base
   # The CSV data in the value of the source field will be expanded into a
   # datastructure in the "dest" field.  Note: if the "dest" field
   # already exists, it will be overridden.
-  config /[A-Za-z0-9_-]+/, :validate => :string
+  config /[A-Za-z0-9_-]+/, :validate => :string, :deprecated => true
 
   # Define a list of field names (in the order they appear in the CSV,
   # as if it were a header line). If this is not specified or there
   # are not enough fields specified, the default field name is "fieldN"
   # (where N is the field number, starting from 1).
   # Optional.
-  config :fields, :validate => :array, :default => []
+  config :fields, :validate => :array, :deprecated => true
+
+  # The CSV data in the value of the source field will be expanded into a
+  # datastructure.
+  # This deprecates the regexp '[A-Za-z0-9_-]' variable.
+  config :source, :validate => :string
+
+  # Define a list of column names (in the order they appear in the CSV,
+  # as if it were a header line). If this is not specified or there
+  # are not enough columns specified, the default column name is "columnX"
+  # (where X is the field number, starting from 1).
+  # This deprecates the 'fields' variable.
+  # Optional.
+  config :columns, :validate => :array, :default => []
 
   # Define the column separator value. If this is not specified the default
   # is a comma ','
   # Optional.
   config :separator, :validate => :string, :default => ","
 
+  # Define target for placing the data
+  # Defaults to @fields
+  # Optional
+  config :target, :validate => :string, :default => "@fields"
+
   public
   def register
-    @csv = {}
 
+    #TODO(electrical): At some point 'fields' and the regexp parts need to be removed.
+    if @fields
+      if @columns
+        @logger.error("'fields' and 'columns' are the same setting, but 'fields' is deprecated. Please use only 'columns'")
+      end
+      @columns = @fields
+    end
+
+    @csv = {}
+    #TODO(electrical): At some point this can be removed
     @config.each do |field, dest|
-      next if (RESERVED + ["fields", "separator"]).member?(field)
+      next if (RESERVED + ["fields", "separator", "source", "columns", "target"]).member?(field)
+      @logger.warn("#{self.class.config_name}: You used a deprecated setting '#{field} => #{dest}'. You should use 'source => \"#{field}\"' and 'target => \"#{dest}\"'")
       @csv[field] = dest
     end
 
-    # Default to parsing @message and dumping into @fields
+    #TODO(electrical): Will we make @source required or not?
+    if @source
+      #Add the source field to the list.
+      @csv[@source] = @target
+    end
+
+    # Default to parsing @message and dumping into @target
     if @csv.empty?
-      @csv["@message"] = "@fields"
+      @csv["@message"] = @target
     end
   end # def register
 
@@ -50,6 +84,7 @@ def filter(event)
     @logger.debug("Running csv filter", :event => event)
 
     matches = 0
+    #TODO(electrical): When old stuff can be removed. this block will need to be changed also
     @csv.each do |key, dest|
       if event[key]
         if event[key].is_a?(String)
@@ -68,7 +103,7 @@ def filter(event)
           values = CSV.parse_line(raw, {:col_sep => @separator})
           data = {}
           values.each_index do |i|
-            field_name = @fields[i] || "field#{i+1}"
+            field_name = @columns[i] || "column#{i+1}"
             data[field_name] = values[i]
           end
 

lib/logstash/filters/date.rb

@@ -119,9 +119,10 @@ def register
     locale = parseLocale(@config["locale"][0]) if @config["locale"] != nil and @config["locale"][0] != nil
     missing = []
     @config.each do |field, value|
-      next if (RESERVED + ["locale"]).include?(field)
-      next if (RESERVED + ["match"]).include?(field)
+      next if (RESERVED + ["locale", "match"]).include?(field)
 
+      recommended_setting = value.map { |v| "\"#{v}\"" }.join(", ")
+      @logger.warn("#{self.class.config_name}: You used a deprecated setting '#{field} => #{value}'. You should use 'match => [ \"#{field}\", #{recommended_setting} ]'")
       # values here are an array of format strings for the given field.
       setupMatcher(field, locale, missing, value) # value.each
     end # @config.each

lib/logstash/filters/grok.rb

@@ -299,6 +299,7 @@ def register
       patterns.each do |pattern|
         @logger.debug? and @logger.debug("regexp: #{@type}/#{field}", :pattern => pattern)
         @patterns[field].compile(pattern)
+        @logger.warn("#{self.class.config_name}: You used a deprecated setting '#{field} => #{pattern}'. You should use 'match => [ \"#{field}\", \"#{pattern}\" ]'")
       end
     end # @config.each
   end # def register

lib/logstash/filters/json.rb

@@ -24,17 +24,53 @@ class LogStash::Filters::Json < LogStash::Filters::Base
   # JSON in the value of the source field will be expanded into a
   # datastructure in the "dest" field.  Note: if the "dest" field
   # already exists, it will be overridden.
-  config /[A-Za-z0-9_@-]+/, :validate => :string
+  config /[A-Za-z0-9_@-]+/, :validate => :string, :deprecated => true
+
+  # Config for json is:
+  #
+  #     source => source_field
+  #
+  # For example, if you have json data in the @message field:
+  #
+  #     filter {
+  #       json {
+  #         source => "@message"
+  #       }
+  #     }
+  #
+  # The above would parse the xml from the @message field
+  config :source, :validate => :string
+
+  # Define target for placing the data
+  #
+  # for example if you want the data to be put in the 'doc' field:
+  #
+  #     filter {
+  #       json {
+  #         target => "doc"
+  #       }
+  #     }
+  #
+  # json in the value of the source field will be expanded into a
+  # datastructure in the "target" field.
+  # Note: if the "target" field already exists, it will be overridden
+  # Required
+  config :target, :validate => :string
 
   public
   def register
     @json = {}
 
     @config.each do |field, dest|
-      next if RESERVED.member?(field)
-
+      next if (RESERVED + ["source", "target"]).member?(field)
+      @logger.warn("#{self.class.config_name}: You used a deprecated setting '#{field} => #{dest}'. You should use 'source => \"#{field}\"' and 'target => \"#{dest}\"'")
       @json[field] = dest
     end
+
+    if @source
+      @json[@source] = @target
+    end
+
   end # def register
 
   public

lib/logstash/filters/translate.rb

@@ -0,0 +1,71 @@
+require "logstash/filters/base"
+require "logstash/namespace"
+
+# Originally written to translate HTTP response codes 
+# but turned into a general translation tool which uses
+# .yaml files as a dictionary.
+# response codes in default dictionary were scraped from 
+# 'gem install cheat; cheat status_codes'
+
+class LogStash::Filters::Translate < LogStash::Filters::Base
+  config_name "translate"
+  plugin_status "experimental"
+
+
+  # The field containing a response code If this field is an
+  # array, only the first value will be used.
+  config :field, :validate => :string, :required => true
+
+  # name with full path of external dictionary file.    
+  # format of the table should be a YAML file. 
+  # make sure you encase any integer based keys in quotes.
+  # For simple string search and replacements for just a few values
+  # use the gsub function of the mutate filter.
+  config :dictionary_path, :validate => :string, :required => true
+
+  # The destination you wish to populate with the response code.    
+  # default is http_response_code.  set to the same value as source
+  # if you want to do a substitution.
+  config :destination, :validate => :string, :default => "translation"
+
+  # set to false if you want to match multiple terms.   
+  # a large dictionary could get expensive if set to false.
+  config :exact, :validate => :boolean, :default => true
+
+
+
+  public
+  def register
+    if File.exists?(@dictionary_path)
+      begin
+        @dictionary = YAML.load_file(@dictionary_path)
+      rescue Exception => e
+        raise "Bad Syntax in dictionary file" 
+      end
+    end # if File.exists?
+    @logger.info("Dictionary - ", :dictionary => @dictionary)
+    if @exact
+      @logger.info("Dictionary translation method - Exact")
+    else 
+      @logger.info("Dictionary translation method - Fuzzy")
+    end # if @exact
+  end # def register
+
+  public
+  def filter(event)
+    return unless filter?(event)
+      begin
+        source = event[@field]
+        source = source.first if source.is_a? Array # if array,  just use first value 
+        source = source.to_s # make sure its a string.  Is this really needed?
+        if @exact
+          translation = @dictionary[source] if @dictionary.include?(source)
+        else 
+          translation = source.gsub(Regexp.union(@dictionary.keys), @dictionary)
+        end # if @exact
+      rescue Exception => e
+          @logger.error("Something went wrong when attempting to translate from dictionary", :exception => e, :field => @field, :event => event)
+      end
+      event[@destination] = translation
+  end # def filter
+end # class LogStash::Filters::Translate