- Update packaging

- Split config files

Rakefile

@@ -35,3 +35,13 @@ task :tar do
   sh "cd /tmp/logstash-build && " \
      "tar -czf #{outdir}/logstash-#{version}.tar.gz logstash-#{version}"
 end
+
+task :package do
+  system("gem build logstash.gemspec")
+end
+
+task :publish do
+  latest_gem = %x{ls -t logstash*.gem}.split("\n").first
+  system("gem push #{latest_gem}")
+end
+

bin/logstash

@@ -3,50 +3,56 @@
 require "rubygems"
 require "eventmachine"
 require "logstash/agent"
+require "yaml"
+require "optparse"
 
-case ARGV[0]
-when "client"
-  config = {
-    "input" => [
-      "/var/log/messages",
-      "/var/log/apache2/access.log",
-    ],
-    "output" => [
-      "amqp://localhost/topic/testing",
-      "websocket:///",
-      "mongodb://localhost/testdb",
-    ],
-  }
-when "server"
-  config = {
-    "input" => [
-      "amqp://localhost/topic/testing",
-    ],
-    "filter" => [
-      "grok",
-    ],
-    "output" => [
-      "stdout:///",
-      "amqp://localhost/topic/parsed",
-    ],
-  }
-when "standalone"
-  config = {
-    "input" => [
-      "/var/log/messages",
-      "/var/log/apache2/access.log",
-    ],
-    "filter" => [
-      "grok",
-    ],
-    "output" => [
-      "mongodb://localhost/testdb",
-    ],
-  }
-else
-  puts "Don't know what you want me to do."
+Settings = Struct.new(:config_file, :daemonize)
+
+settings = Settings.new
+settings.daemonize = false
+settings.config_file = nil
+
+opts = OptionParser.new do |opts|
+  opts.banner = "Usage; #{$0} [options]"
+
+  opts.on("-f CONFIGFILE", "--config CONFIGFILE",
+          "Load the logstash config from a specific file") do |arg|
+    settings.config_file = arg
+  end
+
+  #opts.on("-d", "--daemonize", "Daemonize (default is run in foreground)") do 
+    #settings.daemonize = true
+  #end
+end
+
+opts.parse!(ARGV)
+
+# TODO(sissel): put the config management stuff in LogStash::Agent
+if settings.config_file == "" or settings.config_file == nil
+  $stderr.puts "No config file given. (missing -f or --config flag?)"
   exit 1
 end
 
+if !File.exist?(settings.config_file)
+  $stderr.puts "Config file '#{settings.config_file}' does not exist."
+  exit 1
+end
+
+begin
+  config = YAML::load_file(settings.config_file)
+rescue => e
+  $stderr.puts "Loading config file '#{settings.config_file}' failed: #{e}"
+  exit 1
+end
+
+#config = nil
+#confignames = []
+#YAML::load_documents(file) do |doc|
+  #confignames << doc["configname"]
+  #if doc["configname"] == ARGV[0]
+    #config = doc
+  #end
+#end
+
 agent = LogStash::Agent.new(config)
 agent.run

etc/logstash-mongodb-storage.yaml

@@ -0,0 +1,5 @@
+# Example config that reads parsed logs from AMQP and dumps results into mongodb
+inputs:
+  - amqp://localhost/topic/parsedlogs
+outputs:
+  - mongodb://localhost/parsedlogs

etc/logstash-parser.yaml

@@ -0,0 +1,20 @@
+# Example config that parses rawlogs with grok and puts them on another AMQP topic
+inputs:
+  - amqp://localhost/topic/rawlogs
+outputs:
+  - amqp://localhost/topic/parsedlogs
+  - stdout:///
+filters:
+  grok:
+    linux-syslog: # for logs tagged 'linux-syslog'
+      timestamp: 
+        key: date
+        format: %b %e %H:%M:%S
+      patterns:
+      - %{SYSLOGLINE}
+    apache-access: # for logs tagged 'apache-error'
+      timestamp:
+        key: timestamp
+        format: %d/%b/%Y:%H:%M:%S %Z
+      patterns:
+      - %{COMBINEDAPACHELOG}

etc/logstash-reader.yaml

@@ -0,0 +1,8 @@
+# Example config that reads parsed logs from AMQP and prints to stdout
+inputs:
+  - amqp://localhost/topic/parsedlogs
+#filters:
+  #field:
+  #- progname.include?("tester")
+outputs:
+  - stdout:///

etc/logstash-shipper.yaml

@@ -0,0 +1,18 @@
+--- 
+configname: shipper
+# Example config that only ships log data from files to an AMQP topic
+inputs:
+  linux-syslog:
+  - /var/log/messages
+  - /var/log/kern.log
+  - /var/log/auth.log
+  - /var/log/user.log
+  apache-access:
+  - /var/log/apache2/access.log
+  - /b/access
+  apache-error:
+  - /var/log/apache2/access.log
+  unknown:
+  - /b/randomdata
+outputs:
+- amqp://localhost/topic/rawlogs