Merge pull request elastic#83 from lusis/docgen-changes

Add support for plugin_status to plugins

docs/docgen.rb

@@ -23,6 +23,7 @@ def initialize
       /^ *class.*< *LogStash::(Outputs|Filters|Inputs)::Base/ => \
         lambda { |m| set_class_description },
       /^ *config +[^=].*/ => lambda { |m| add_config(m[0]) },
+      /^ *plugin_status .*/ => lambda { |m| set_plugin_status(m[0]) },
       /^ *config_name .*/ => lambda { |m| set_config_name(m[0]) },
       /^ *flag[( ].*/ => lambda { |m| add_flag(m[0]) },
       /^ *(class|def|module) / => lambda { |m| clear_comments },
@@ -96,6 +97,11 @@ def set_config_name(code)
     @name = name
   end # def set_config_name
 
+  def set_plugin_status(code)
+    status = eval(code)
+    @plugin_status = status
+  end
+
   # pretend to be the config DSL and just get the name
   def config(name, opts={})
     return name, opts
@@ -113,6 +119,11 @@ def config_name(name)
     return name
   end # def config_name
 
+  # pretend to be the config dsl's 'plugin_status' method
+  def plugin_status(status)
+    return status
+  end # def plugin_status
+
   def clear_comments
     @comments.clear
   end # def clear_comments
@@ -126,6 +137,7 @@ def generate(file, settings)
     @comments = []
     @settings = {}
     @class_description = ""
+    @plugin_status = ""
 
     # parse base first
     parse(File.new(File.join(File.dirname(file), "base.rb"), "r").read)

docs/docs.html.erb

@@ -3,6 +3,7 @@ title: logstash docs for <%= section %>s/<%= name %>
 layout: content_right
 ---
 <h2><%= name %></h2>
+<h3>Status: <a href="plugin-status"><%= @plugin_status %></a></h3>
 
 <%= description %>
 

docs/plugin-status.md

@@ -0,0 +1,23 @@
+---
+title: Plugin Status - logstash
+layout: content_right
+---
+# Plugin Status
+
+Plugins (inputs/outputs/filters) have a status in logstash. This is to provide an indicator to the end-user as to the 'state' of the plugin.
+
+Terminology is still being worked out but there are three general states - experimental, unstable, stable.
+
+The desire here is to allow people to quickly iterate on possible new plugins while conveying to the end-user a set of expectations about that plugin. This allows you to make more informed decisions about when and where to use the functionality provided by the new plugin.
+
+## Experimental
+When a plugin is in the `experimental` state, it is essentially untested. This does not mean that it does not have any associated unit tests. This applies more to in-the-wild usage. Most new plugins will probably fit in this category. There is a chance that experimental plugins may be removed at some point. It is possible that an experimental plugin will be broken mid-release.
+
+## Unstable
+Unstable plugins are plugins that are in the process of being stabalized into a final form. Unstable plugins will have a bit more wide-spread usage in the community. The API for these plugins has stabilized and is unlikely to change mid-release. Test cases may or may not exist.
+
+## Stable
+Stable plugins are plugins that you can comfortably rely on in production. These have full test cases.
+
+# A note about output plugins
+It's worth reminding users that `output` plugins are currently blocking. If any output plugin fails, all output plugins are blocked. Please keep this in mind when using experimental output plugins as it could cause unintended side-effects.

lib/logstash/config/mixin.rb

@@ -99,6 +99,11 @@ def config_name(name=nil)
       return @config_name
     end
 
+    def plugin_status(status=nil)
+      @plugin_status = status if !status.nil?
+      return @plugin_status
+    end
+
     # Define a new configuration setting
     def config(name, opts={})
       @config ||= Hash.new
@@ -157,13 +162,30 @@ def validate(params)
       @logger = LogStash::Logger.new(STDOUT)
       is_valid = true
 
+      is_valid &&= validate_plugin_status
       is_valid &&= validate_check_invalid_parameter_names(params)
       is_valid &&= validate_check_required_parameter_names(params)
       is_valid &&= validate_check_parameter_values(params)
 
       return is_valid
     end # def validate
 
+    def validate_plugin_status
+      case @plugin_status
+      when "experimental"
+        @logger.warn("Using experimental plugin #{@config_name}. This plugin is untested. Use at your own risk")
+      when "unstable"
+        @logger.info("Using unstable plugin #{@config_name}.")
+      when "stable"
+        # This is cool.
+      when nil
+        raise "#{@config_name} must set a plugin_status"
+      else
+        raise "#{@config_name} set an invalid plugin status #{@plugin_status}. Valid values are experimental, unstable and stable"
+      end
+      return true
+    end
+
     def validate_check_invalid_parameter_names(params)
       invalid_params = params.keys
       # Filter out parameters that match regexp keys.

lib/logstash/filters/date.rb

@@ -21,6 +21,7 @@
 class LogStash::Filters::Date < LogStash::Filters::Base
 
   config_name "date"
+  plugin_status "unstable"
 
   # Config for date is:
   #   fieldname => dateformat

lib/logstash/filters/dns.rb

@@ -30,6 +30,7 @@
 class LogStash::Filters::DNS < LogStash::Filters::Base
 
   config_name "dns"
+  plugin_status "unstable"
 
   # Reverse resolve one or more fields.
   config :reverse, :validate => :array

lib/logstash/filters/gelfify.rb

@@ -5,6 +5,7 @@
 # corresponding GELF levels.
 class LogStash::Filters::Gelfify < LogStash::Filters::Base
   config_name "gelfify"
+  plugin_status "unstable"
 
   SYSLOG_LEVEL_MAP = {
     0 => 3, # Emergency => FATAL

lib/logstash/filters/grep.rb

@@ -9,6 +9,7 @@
 class LogStash::Filters::Grep < LogStash::Filters::Base
 
   config_name "grep"
+  plugin_status "unstable"
 
   # Drop events that don't match
   #

lib/logstash/filters/grok.rb

@@ -11,6 +11,7 @@
 # your own trivially. (See the patterns_dir setting)
 class LogStash::Filters::Grok < LogStash::Filters::Base
   config_name "grok"
+  plugin_status "unstable"
 
   # Specify a pattern to parse with. This will match the '@message' field.
   #

lib/logstash/filters/grokdiscovery.rb

@@ -6,6 +6,7 @@
 class LogStash::Filters::Grokdiscovery < LogStash::Filters::Base
 
   config_name "grokdiscovery"
+  plugin_status "experimental"
 
   public
   def initialize(config = {})

lib/logstash/filters/json.rb

@@ -6,6 +6,7 @@
 class LogStash::Filters::Json < LogStash::Filters::Base
 
   config_name "json"
+  plugin_status "unstable"
 
   # Config for json is:
   #   source: dest

lib/logstash/filters/multiline.rb

@@ -61,6 +61,7 @@
 class LogStash::Filters::Multiline < LogStash::Filters::Base
 
   config_name "multiline"
+  plugin_status "unstable"
 
   # The regular expression to match
   config :pattern, :validate => :string, :require => true

lib/logstash/filters/mutate.rb

@@ -8,6 +8,7 @@
 # TODO(sissel): Support regexp replacements like String#gsub ?
 class LogStash::Filters::Mutate < LogStash::Filters::Base
   config_name "mutate"
+  plugin_status "unstable"
 
   # Rename one or more fields.
   config :rename, :validate => :hash

lib/logstash/filters/split.rb

@@ -13,6 +13,7 @@
 class LogStash::Filters::Split < LogStash::Filters::Base
 
   config_name "split"
+  plugin_status "unstable"
 
   # The string to split on. This is usually a line terminator, but can be any
   # string.

lib/logstash/inputs/amqp.rb

@@ -12,6 +12,7 @@
 class LogStash::Inputs::Amqp < LogStash::Inputs::Base
 
   config_name "amqp"
+  plugin_status "unstable"
 
   # Your amqp server address
   config :host, :validate => :string, :required => true

lib/logstash/inputs/exec.rb

@@ -15,6 +15,7 @@
 class LogStash::Inputs::Exec < LogStash::Inputs::Base
 
   config_name "exec"
+  plugin_status "unstable"
 
   # Set this to true to enable debugging on an input.
   config :debug, :validate => :boolean, :default => false

lib/logstash/inputs/file.rb

@@ -11,6 +11,7 @@
 # is detected and handled by this input.
 class LogStash::Inputs::File < LogStash::Inputs::Base
   config_name "file"
+  plugin_status "unstable"
 
   # The path to the file to use as an input.
   # You can use globs here, such as "/var/log/*.log"

lib/logstash/inputs/gelf.rb

@@ -13,6 +13,7 @@
 #
 class LogStash::Inputs::Gelf < LogStash::Inputs::Base
   config_name "gelf"
+  plugin_status "unstable"
 
   # The address to listen on
   config :host, :validate => :string, :default => "0.0.0.0"

lib/logstash/inputs/redis.rb

@@ -8,6 +8,7 @@
 class LogStash::Inputs::Redis < LogStash::Inputs::Base
 
   config_name "redis"
+  plugin_status "unstable"
 
   # Name is used for logging in case there are multiple instances.
   # This feature has no real function and will be removed in future versions.

lib/logstash/inputs/stdin.rb

@@ -10,6 +10,8 @@ class LogStash::Inputs::Stdin < LogStash::Inputs::Base
 
   config_name "stdin"
 
+  plugin_status "unstable"
+
   public
   def register
     @host = Socket.gethostname

lib/logstash/inputs/stomp.rb

@@ -4,6 +4,7 @@
 
 class LogStash::Inputs::Stomp < LogStash::Inputs::Base
   config_name "stomp"
+  plugin_status "unstable"
 
   # The address of the STOMP server.
   config :host, :validate => :string, :default => "localhost", :required => true
@@ -69,4 +70,3 @@ def run(output_queue)
     subscription_handler
   end # def run
 end # class LogStash::Inputs::Stomp
-

lib/logstash/inputs/syslog.rb

@@ -20,6 +20,7 @@
 # Note: this input will start listeners on both TCP and UDP
 class LogStash::Inputs::Syslog < LogStash::Inputs::Base
   config_name "syslog"
+  plugin_status "unstable"
 
   # The address to listen on
   config :host, :validate => :string, :default => "0.0.0.0"

lib/logstash/inputs/tcp.rb

@@ -12,6 +12,7 @@
 class LogStash::Inputs::Tcp < LogStash::Inputs::Base
 
   config_name "tcp"
+  plugin_status "unstable"
 
   # When mode is `server`, the address to listen on.
   # When mode is `client`, the address to connect to.

lib/logstash/inputs/twitter.rb

@@ -8,7 +8,8 @@
 class LogStash::Inputs::Twitter < LogStash::Inputs::Base
 
   config_name "twitter"
-
+  plugin_status "unstable"
+
   # Your twitter username
   config :user, :validate => :string, :required => true
 

lib/logstash/inputs/xmpp.rb

@@ -9,6 +9,7 @@
 class LogStash::Inputs::Xmpp < LogStash::Inputs::Base
 
   config_name "xmpp"
+  plugin_status "unstable"
 
   # The user or resource ID, like [email protected].
   config :user, :validate => :string, :required => :true
@@ -67,4 +68,3 @@ def run(queue)
   end # def run
 
 end # def class LogStash:Inputs::Xmpp
-

lib/logstash/inputs/zmq.rb

@@ -15,6 +15,7 @@
 class LogStash::Inputs::Zmq < LogStash::Inputs::Base
 
   config_name "zmq"
+  plugin_status "experimental"
 
   # 0mq socket address to connect or bind to
   config :address, :validate => :string, :default => "tcp://127.0.0.1:2120"

lib/logstash/outputs/amqp.rb

@@ -10,6 +10,7 @@ class LogStash::Outputs::Amqp < LogStash::Outputs::Base
   MQTYPES = [ "fanout", "direct", "topic" ]
 
   config_name "amqp"
+  plugin_status "unstable"
 
   # Your amqp server address
   config :host, :validate => :string, :required => true

lib/logstash/outputs/elasticsearch.rb

@@ -12,6 +12,7 @@
 class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
 
   config_name "elasticsearch"
+  plugin_status "stable"
 
   # ElasticSearch server name. This is optional if your server is discoverable.
   config :host, :validate => :string

lib/logstash/outputs/elasticsearch_river.rb

@@ -17,6 +17,7 @@
 class LogStash::Outputs::ElasticSearchRiver < LogStash::Outputs::Base
 
   config_name "elasticsearch_river"
+  plugin_status "unstable"
 
   config :debug, :validate => :boolean, :default => false
 
@@ -172,4 +173,3 @@ def receive(event)
     @mq.receive_raw(index_message)
   end # def receive
 end # LogStash::Outputs::ElasticSearchRiver
-

lib/logstash/outputs/file.rb

@@ -8,7 +8,8 @@
 class LogStash::Outputs::File < LogStash::Outputs::Base
 
   config_name "file"
-
+  plugin_status "unstable"
+
   # The path to the file to write. Event fields can be used here, 
   # like "/var/log/logstash/%{@source_host}/%{application}"
   config :path, :validate => :string, :required => true

lib/logstash/outputs/ganglia.rb

@@ -5,6 +5,7 @@
 # ganglia's gmond. This is heavily based on the graphite output.
 class LogStash::Outputs::Ganglia < LogStash::Outputs::Base
   config_name "ganglia"
+  plugin_status "unstable"
 
   # The address of the graphite server.
   config :host, :validate => :string, :default => "localhost"

lib/logstash/outputs/gelf.rb

@@ -8,7 +8,8 @@
 class LogStash::Outputs::Gelf < LogStash::Outputs::Base
 
   config_name "gelf"
-
+  plugin_status "unstable"
+
   # graylog2 server address
   config :host, :validate => :string, :required => true
 

lib/logstash/outputs/graphite.rb

@@ -10,6 +10,7 @@
 # I can capture the metric values from the logs and emit them to graphite.
 class LogStash::Outputs::Graphite < LogStash::Outputs::Base
   config_name "graphite"
+  plugin_status "unstable"
 
   # The address of the graphite server.
   config :host, :validate => :string, :default => "localhost"

lib/logstash/outputs/internal.rb

@@ -5,6 +5,7 @@
 # is not useful for general deployment.
 class LogStash::Outputs::Internal < LogStash::Outputs::Base
   config_name "internal"
+  plugin_status "stable"
 
   attr_accessor :callback
 

lib/logstash/outputs/loggly.rb

@@ -25,6 +25,7 @@ def rbuf_fill
 # and 'json logging' enabled.
 class LogStash::Outputs::Loggly < LogStash::Outputs::Base
   config_name "loggly"
+  plugin_status "unstable"
 
   # The hostname to send logs to. This should target the loggly http input
   # server which is usually "logs.loggly.com"

lib/logstash/outputs/mongodb.rb

@@ -4,6 +4,7 @@
 class LogStash::Outputs::Mongodb < LogStash::Outputs::Base
 
   config_name "mongodb"
+  plugin_status "unstable"
 
   # your mongodb host
   config :host, :validate => :string, :required => true