Added grok parser for bro http.log

Fix parsing http.log. Tested on 200000 logs.

Added test for bro http.log

Added parsed BRO DNS, changed HTTP in BRO_HTTP

Added parser BRO_CONN

Added bro files

Fixes logstash-plugins#71

patterns/bro

@@ -0,0 +1,13 @@
+# https://www.bro.org/sphinx/script-reference/log-files.html
+
+# http.log
+BRO_HTTP %{NUMBER:ts}\t%{NOTSPACE:uid}\t%{IP:orig_h}\t%{INT:orig_p}\t%{IP:resp_h}\t%{INT:resp_p}\t%{INT:trans_depth}\t%{GREEDYDATA:method}\t%{GREEDYDATA:domain}\t%{GREEDYDATA:uri}\t%{GREEDYDATA:referrer}\t%{GREEDYDATA:user_agent}\t%{NUMBER:request_body_len}\t%{NUMBER:response_body_len}\t%{GREEDYDATA:status_code}\t%{GREEDYDATA:status_msg}\t%{GREEDYDATA:info_code}\t%{GREEDYDATA:info_msg}\t%{GREEDYDATA:filename}\t%{GREEDYDATA:bro_tags}\t%{GREEDYDATA:username}\t%{GREEDYDATA:password}\t%{GREEDYDATA:proxied}\t%{GREEDYDATA:orig_fuids}\t%{GREEDYDATA:orig_mime_types}\t%{GREEDYDATA:resp_fuids}\t%{GREEDYDATA:resp_mime_types}
+
+# dns.log
+BRO_DNS %{NUMBER:ts}\t%{NOTSPACE:uid}\t%{IP:orig_h}\t%{INT:orig_p}\t%{IP:resp_h}\t%{INT:resp_p}\t%{WORD:proto}\t%{INT:trans_id}\t%{GREEDYDATA:query}\t%{GREEDYDATA:qclass}\t%{GREEDYDATA:qclass_name}\t%{GREEDYDATA:qtype}\t%{GREEDYDATA:qtype_name}\t%{GREEDYDATA:rcode}\t%{GREEDYDATA:rcode_name}\t%{GREEDYDATA:AA}\t%{GREEDYDATA:TC}\t%{GREEDYDATA:RD}\t%{GREEDYDATA:RA}\t%{GREEDYDATA:Z}\t%{GREEDYDATA:answers}\t%{GREEDYDATA:TTLs}\t%{GREEDYDATA:rejected}
+
+# conn.log
+BRO_CONN %{NUMBER:ts}\t%{NOTSPACE:uid}\t%{IP:orig_h}\t%{INT:orig_p}\t%{IP:resp_h}\t%{INT:resp_p}\t%{WORD:proto}\t%{GREEDYDATA:service}\t%{NUMBER:duration}\t%{NUMBER:orig_bytes}\t%{NUMBER:resp_bytes}\t%{GREEDYDATA:conn_state}\t%{GREEDYDATA:local_orig}\t%{GREEDYDATA:missed_bytes}\t%{GREEDYDATA:history}\t%{GREEDYDATA:orig_pkts}\t%{GREEDYDATA:orig_ip_bytes}\t%{GREEDYDATA:resp_pkts}\t%{GREEDYDATA:resp_ip_bytes}\t%{GREEDYDATA:tunnel_parents}
+
+# files.log
+BRO_FILES %{NUMBER:ts}\t%{NOTSPACE:fuid}\t%{IP:tx_hosts}\t%{IP:rx_hosts}\t%{NOTSPACE:conn_uids}\t%{GREEDYDATA:source}\t%{GREEDYDATA:depth}\t%{GREEDYDATA:analyzers}\t%{GREEDYDATA:mime_type}\t%{GREEDYDATA:filename}\t%{GREEDYDATA:duration}\t%{GREEDYDATA:local_orig}\t%{GREEDYDATA:is_orig}\t%{GREEDYDATA:seen_bytes}\t%{GREEDYDATA:total_bytes}\t%{GREEDYDATA:missing_bytes}\t%{GREEDYDATA:overflow_bytes}\t%{GREEDYDATA:timedout}\t%{GREEDYDATA:parent_fuid}\t%{GREEDYDATA:md5}\t%{GREEDYDATA:sha1}\t%{GREEDYDATA:sha256}\t%{GREEDYDATA:extracted}

spec/patterns/bro.rb

@@ -0,0 +1,126 @@
+# encoding: utf-8
+require "spec_helper"
+require "logstash/patterns/core"
+
+describe "HTTP" do
+
+  let(:value)   { "1432555199.633017	COpk6E3vkURP8QQNKl	192.168.9.35	55281	178.236.7.146	80	4	POST	www.amazon.it	/xa/dealcontent/v2/GetDeals?nocache=1432555199326	http://www.amazon.it/	Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36	223	1859	200	OK	-	-	-	(empty)	-	-	-	FrLEcY3AUPKdcYGf29	text/plain	FOJpbGzIMh9syPxH8	text/plain" }
+  let(:grok)    { grok_match(subject, value) }
+
+  it "a pattern pass the grok expression" do
+    expect(grok).to pass
+  end
+
+  it "matches a simple message" do
+    expect(subject).to match(value)
+  end
+
+  it "generates the ts field" do
+    expect(grok).to include("ts" => "1432555199.633017")
+  end
+
+  it "generates the uid field" do
+    expect(grok).to include("uid" => "COpk6E3vkURP8QQNKl")
+  end
+
+  it "generates the orig_h field" do
+    expect(grok).to include("orig_h" => "192.168.9.35")
+  end
+
+  it "generates the orig_p field" do
+    expect(grok).to include("orig_p" => "55281")
+  end
+
+  it "generates the resp_h field" do
+    expect(grok).to include("resp_h" => "178.236.7.146")
+  end
+
+  it "generates the resp_p field" do
+    expect(grok).to include("resp_p" => "80")
+  end
+
+  it "generates the trans_depth field" do
+    expect(grok).to include("trans_depth" => "4")
+  end
+
+  it "generates the method field" do
+    expect(grok).to include("method" => "POST")
+  end
+
+  it "generates the domain field" do
+    expect(grok).to include("domain" => "www.amazon.it")
+  end
+
+  it "generates the uri field" do
+    expect(grok).to include("uri" => "/xa/dealcontent/v2/GetDeals?nocache=1432555199326")
+  end
+
+  it "generates the referrer field" do
+    expect(grok).to include("referrer" => "http://www.amazon.it/")
+  end
+
+  it "generates the user_agent field" do
+    expect(grok).to include("user_agent" => "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36")
+  end
+
+  it "generates the request_body_len field" do
+    expect(grok).to include("request_body_len" => "223")
+  end
+
+  it "generates the response_body_len field" do
+    expect(grok).to include("response_body_len" => "1859")
+  end
+
+  it "generates the status_code field" do
+    expect(grok).to include("status_code" => "200")
+  end
+
+  it "generates the status_msg field" do
+    expect(grok).to include("status_msg" => "OK")
+  end
+
+  it "generates the info_code field" do
+    expect(grok).to include("info_code" => "-")
+  end
+
+  it "generates the info_msg field" do
+    expect(grok).to include("info_msg" => "-")
+  end
+
+  it "generates the filename field" do
+    expect(grok).to include("filename" => "-")
+  end
+
+  it "generates the bro_tags field" do
+    expect(grok).to include("bro_tags" => "(empty)")
+  end
+
+  it "generates the username field" do
+    expect(grok).to include("username" => "-")
+  end
+
+  it "generates the password field" do
+    expect(grok).to include("password" => "-")
+  end
+
+  it "generates the proxied field" do
+    expect(grok).to include("proxied" => "-")
+  end
+
+  it "generates the orig_fuids field" do
+    expect(grok).to include("orig_fuids" => "FrLEcY3AUPKdcYGf29")
+  end
+
+  it "generates the orig_mime_types field" do
+    expect(grok).to include("orig_mime_types" => "text/plain")
+  end
+
+  it "generates the resp_fuids field" do
+    expect(grok).to include("resp_fuids" => "FOJpbGzIMh9syPxH8")
+  end
+
+  it "generates the resp_mime_types field" do
+    expect(grok).to include("resp_mime_types" => "text/plain")
+  end
+
+end