From a3ebbec089515a581faa458db6d9c999fffca407 Mon Sep 17 00:00:00 2001 From: Jordan Sissel Date: Sat, 19 Feb 2011 19:12:55 -0800 Subject: [PATCH] - More work towards a config file format. --- bin/logstash | 19 +++++----- lib/logstash/agent.rb | 18 ++++++---- lib/logstash/config/grammar.rl | 15 +++++--- lib/logstash/config/mixin.rb | 1 + lib/logstash/config/parser.rb | 65 ---------------------------------- lib/logstash/outputs/amqp.rb | 1 + lib/logstash/outputs/base.rb | 1 - 7 files changed, 33 insertions(+), 87 deletions(-) delete mode 100644 lib/logstash/config/parser.rb diff --git a/bin/logstash b/bin/logstash index f14c247dcce..5e6fc91920a 100755 --- a/bin/logstash +++ b/bin/logstash @@ -5,7 +5,7 @@ $: << File.dirname($0) + "/../lib" require "rubygems" require "logstash/agent" require "optparse" -require "yaml" +#require "yaml" Settings = Struct.new(:config_file, :daemonize, :logfile, :verbose) @@ -52,12 +52,12 @@ if !File.exist?(settings.config_file) $stderr.puts "Config file '#{settings.config_file}' does not exist." end -begin - config = YAML::load_file(settings.config_file) -rescue => e - $stderr.puts "Loading config file '#{settings.config_file}' failed: #{e}" - exit 1 -end +#begin + #config = YAML::load_file(settings.config_file) +#rescue => e + #$stderr.puts "Loading config file '#{settings.config_file}' failed: #{e}" + ##exit 1 +#end if settings.daemonize if Process.fork == nil @@ -78,8 +78,5 @@ elsif settings.daemonize STDERR.reopen(devnull) end -agent = LogStash::Agent.new(config) -#if settings.logfile - #agent.log_to(settings.logfile) -#end +agent = LogStash::Agent.new(settings) agent.run diff --git a/lib/logstash/agent.rb b/lib/logstash/agent.rb index b0e60ca1956..1d4c17e0044 100644 --- a/lib/logstash/agent.rb +++ b/lib/logstash/agent.rb @@ -4,6 +4,7 @@ require "logstash/multiqueue" require "logstash/namespace" require "logstash/outputs" +require "logstash/config/file" require "java" require "uri" @@ -17,18 +18,14 @@ class LogStash::Agent attr_reader :filters public - def initialize(config) + def initialize(settings) log_to(STDERR) - @config = config + @settings = settings @threads = {} @outputs = [] @inputs = [] @filters = [] - # Config should have: - # - list of logs to monitor - # - log config - # - where to ship to Thread::abort_on_exception = true end # def initialize @@ -40,6 +37,14 @@ def log_to(target) public def run + # Load the config file + p @settings.config_file + config = LogStash::Config::File.new(@settings.config_file) + config.parse do |plugin| + ap plugin + end + exit + if @config["inputs"].length == 0 or @config["outputs"].length == 0 raise "Must have both inputs and outputs configured." end @@ -171,6 +176,7 @@ def register_signal_handler end @sigchannel.subscribe do |msg| + # TODO(sissel): Make this a function. case msg when :USR1 counts = Hash.new { |h,k| h[k] = 0 } diff --git a/lib/logstash/config/grammar.rl b/lib/logstash/config/grammar.rl index 2c8f484ec80..34304c1d428 100644 --- a/lib/logstash/config/grammar.rl +++ b/lib/logstash/config/grammar.rl @@ -81,7 +81,10 @@ require "logstash/namespace" @config[name] += @components } - ws = ([ \t\n])** ; + comment = "#" (any - "\n")* ; + ws = ([ \t\n] | comment)** ; + #ws = ([ \t\n])** ; + # TODO(sissel): Support floating point values? numeric = ( ("+" | "-")? [0-9] :>> [0-9]** ) >mark %stack_numeric; quoted_string = ( @@ -106,9 +109,13 @@ require "logstash/namespace" # } component_implementation = ( - naked_string ws "{" ws - parameters - ws "}" + ( + naked_string ws "{" ws + parameters + ws "}" + ) | ( + naked_string ws "{" ws "}" + ) ) %component_implementation ; component = ( diff --git a/lib/logstash/config/mixin.rb b/lib/logstash/config/mixin.rb index 9a47c7a8cf2..fb6e7cf3895 100644 --- a/lib/logstash/config/mixin.rb +++ b/lib/logstash/config/mixin.rb @@ -49,6 +49,7 @@ def config(cfg=nil) end # def config def dsl_gen + # TODO(sissel): Deprecated. puts "#{@dsl_parent.config_name} { #parent" if @dsl_parent config = [] config << "#{@config_name} { #node" diff --git a/lib/logstash/config/parser.rb b/lib/logstash/config/parser.rb deleted file mode 100644 index b630f196c4b..00000000000 --- a/lib/logstash/config/parser.rb +++ /dev/null @@ -1,65 +0,0 @@ -require "logstash/namespace" -require "logstash/config/grammar" -require "logstash/config/registry" -require "logstash/agent" - -class LogStash::Config::File - def initialize(file) - @file = file - end - - def parse - grammar = LogStash::Config::Grammar.new - grammar.parse(File.new(@file).read) - @config = grammar.config - end - - def each(&block) - #ap @config - - # First level is the components - # Like: - # inputs { - # ... - # } - @config.each do |type, plugin_config_array| - # plugin_config_array has arrays of each component config: - # inputs { - # amqp { ... } - # file { ... } - # file { ... } - # } - plugin_config_array.each do |plugin_config| - yield({ - :type => type, - :plugin => plugin_config.keys.first, - :parameters => plugin_config.values.first - }) - end - end # @config.each - end # def each -end # class LogStash::Config::Parser - -#agent = LogStash::Agent.new({}) -cfg = LogStash::Config::File.new(ARGV[0]) -cfg.parse -registry = LogStash::Config::Registry::registry -cfg.each do |o| - ap ["Config", o] - - # Load the base class for the type given (like inputs/base, or filters/base) - # TODO(sissel): Error handling - require "logstash/#{o[:type]}s/base" - type = registry[o[:type]] - - # Load the plugin itself (inputs/file, outputs/amqp, etc) - # TODO(sissel): Error handling - require "logstash/#{o[:type]}s/#{o[:plugin].downcase}" - config_class = registry[o[:plugin]] - if type - ap :got => [type, config_class] - else - puts "Unknown config #{o.awesome_inspect}" - end -end -#agent.config(cfg) diff --git a/lib/logstash/outputs/amqp.rb b/lib/logstash/outputs/amqp.rb index f0033e2f296..4de6b0e7d0d 100644 --- a/lib/logstash/outputs/amqp.rb +++ b/lib/logstash/outputs/amqp.rb @@ -7,6 +7,7 @@ class LogStash::Outputs::Amqp < LogStash::Outputs::Base MQTYPES = [ "fanout", "queue", "topic" ] + config_name "amqp" config :host => :string config :queue_type => :string config :queue_name => :string diff --git a/lib/logstash/outputs/base.rb b/lib/logstash/outputs/base.rb index 0835b626644..5235b8f5e74 100644 --- a/lib/logstash/outputs/base.rb +++ b/lib/logstash/outputs/base.rb @@ -11,7 +11,6 @@ class LogStash::Outputs::Base attr_accessor :logger config_name "outputs" - dsl_parent nil public def initialize(url)