diff --git a/.editorconfig b/.editorconfig index 7a1535ad1b44..ff8d698c9a8a 100644 --- a/.editorconfig +++ b/.editorconfig @@ -32,3 +32,7 @@ indent_style = tab [Vagrantfile] indent_size = 2 indent_style = space + +[*.rl] +indent_size = 4 +indent_style = space diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 8c51873bb932..6ceba82f0899 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -16,6 +16,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -1318,6 +1319,129 @@ type: keyword -- +[[exported-fields-cef]] +== Decode CEF processor fields fields + +Common Event Format (CEF) data. + + + +[float] +=== cef + +By default the `decode_cef` processor writes all data from the CEF message to this `cef` object. It contains the CEF header fields and the extension data. + + + +*`cef.version`*:: ++ +-- +Version of the CEF specification used by the message. + + +type: keyword + +-- + +*`cef.device.vendor`*:: ++ +-- +Vendor of the device that produced the message. + + +type: keyword + +-- + +*`cef.device.product`*:: ++ +-- +Product of the device that produced the message. + + +type: keyword + +-- + +*`cef.device.version`*:: ++ +-- +Version of the product that produced the message. + + +type: keyword + +-- + +*`cef.device.event_class_id`*:: ++ +-- +Unique identifier of the event type. + + +type: keyword + +-- + +*`cef.severity`*:: ++ +-- +Importance of the event. The valid string values are Unknown, Low, Medium, High, and Very-High. The valid integer values are 0-3=Low, 4-6=Medium, 7- 8=High, and 9-10=Very-High. + + +type: keyword + +example: Very-High + +-- + +*`cef.name`*:: ++ +-- +Short description of the event. + + +type: keyword + +-- + +*`cef.extensions`*:: ++ +-- +Collection of key-value pairs carried in the CEF extension field. + + +type: object + +-- + +*`observer.product`*:: ++ +-- +Product name. + +type: keyword + +-- + +*`source.service.name`*:: ++ +-- +Service that is the source of the event. + +type: keyword + +-- + +*`destination.service.name`*:: ++ +-- +Service that is the target of the event. + +type: keyword + +-- + [[exported-fields-cisco]] == Cisco fields diff --git a/filebeat/docs/index.asciidoc b/filebeat/docs/index.asciidoc index eb6fe9b13a12..ab91ca58abb2 100644 --- a/filebeat/docs/index.asciidoc +++ b/filebeat/docs/index.asciidoc @@ -18,6 +18,7 @@ include::{asciidoc-dir}/../../shared/attributes.asciidoc[] :has_solutions: :ignores_max_retries: :has_docker_label_ex: +:has_decode_cef_processor: :has_decode_csv_fields_processor: :has_script_processor: :has_timestamp_processor: diff --git a/filebeat/magefile.go b/filebeat/magefile.go index 802b1e95f507..864c3fac316a 100644 --- a/filebeat/magefile.go +++ b/filebeat/magefile.go @@ -148,6 +148,7 @@ func fieldDocs() error { devtools.XPackBeatDir("module"), devtools.OSSBeatDir("input"), devtools.XPackBeatDir("input"), + devtools.XPackBeatDir("processors"), } output := devtools.CreateDir("build/fields/fields.all.yml") if err := devtools.GenerateFieldsYAMLTo(output, inputs...); err != nil { diff --git a/libbeat/docs/processors-using.asciidoc b/libbeat/docs/processors-using.asciidoc index 1a6eb06df748..29d878134d63 100644 --- a/libbeat/docs/processors-using.asciidoc +++ b/libbeat/docs/processors-using.asciidoc @@ -211,6 +211,9 @@ The supported processors are: * <> * <> * <> +ifdef::has_decode_cef_processor[] +* <> +endif::[] ifdef::has_decode_csv_fields_processor[] * <> endif::[] @@ -793,6 +796,45 @@ Adds the environment field to every event: } ------------------------------------------------------------------------------- +ifdef::has_decode_cef_processor[] +[[processor-decode-cef]] +[role="xpack"] +=== Decode CEF + +beta[] + +The `decode_cef` processor decodes Common Event Format (CEF) messages. This +processor is available in Filebeat. + +Below is an example configuration that decodes the `message` field as CEF after +renaming it to `event.original`. It is best to rename `message` to +`event.original` because the decoded CEF data contains its own `message` field. + +[source,yaml] +---- +processors: +- rename: + fields: + - {from: "message", to: "event.original"} +- decode_cef: + field: event.original +---- + +The `decode_cef` processor has the following configuration settings. + +.Decode CEF options +[options="header"] +|====== +| `field` | no | message | Source field containing the CEF message to be parsed. | +| `target_field` | no | cef | Target field where the parsed CEF object will be written. | +| `ecs` | no | true | Generate Elastic Common Schema (ECS) fields from the CEF data. + Certain CEF header and extension values will be used to populate ECS fields. | +| `ignore_missing` | no | false | Ignore errors when the source field is missing. | +| `ignore_failure` | no | false | Ignore failures when the source field does not contain a CEF message. | +| `id` | no | | An identifier for this processor instance. Useful for debugging. | +|====== +endif::[] + ifdef::has_decode_csv_fields_processor[] [[decode-csv-fields]] === Decode CSV fields diff --git a/x-pack/filebeat/include/list.go b/x-pack/filebeat/include/list.go index 8eb3ddf136f0..c96c7896cbf3 100644 --- a/x-pack/filebeat/include/list.go +++ b/x-pack/filebeat/include/list.go @@ -24,4 +24,5 @@ import ( _ "github.com/elastic/beats/x-pack/filebeat/module/rabbitmq" _ "github.com/elastic/beats/x-pack/filebeat/module/suricata" _ "github.com/elastic/beats/x-pack/filebeat/module/zeek" + _ "github.com/elastic/beats/x-pack/filebeat/processors/decode_cef" ) diff --git a/x-pack/filebeat/magefile.go b/x-pack/filebeat/magefile.go index f9353665cb8e..2ccc0ff0b143 100644 --- a/x-pack/filebeat/magefile.go +++ b/x-pack/filebeat/magefile.go @@ -81,10 +81,10 @@ func TestPackages() error { return devtools.TestPackages() } -// Fields generates the fields.yml file and a fields.go for each module and -// input. +// Fields generates the fields.yml file and a fields.go for each module, input, +// and processor. func Fields() { - mg.Deps(fieldsYML, moduleFieldsGo, inputFieldsGo) + mg.Deps(fieldsYML, moduleFieldsGo, inputFieldsGo, processorsFieldsGo) } func inputFieldsGo() error { @@ -95,9 +95,13 @@ func moduleFieldsGo() error { return devtools.GenerateModuleFieldsGo("module") } +func processorsFieldsGo() error { + return devtools.GenerateModuleFieldsGo("processors") +} + // fieldsYML generates a fields.yml based on filebeat + x-pack/filebeat/modules. func fieldsYML() error { - return devtools.GenerateFieldsYAML(devtools.OSSBeatDir("module"), "module", "input") + return devtools.GenerateFieldsYAML(devtools.OSSBeatDir("module"), "module", "input", "processors") } // Dashboards collects all the dashboards and generates index patterns. @@ -130,7 +134,7 @@ func Update() { } func includeList() error { - return devtools.GenerateIncludeListGo([]string{"input/*"}, []string{"module"}) + return devtools.GenerateIncludeListGo([]string{"input/*", "processors/*"}, []string{"module"}) } // IntegTest executes integration tests (it uses Docker to run the tests). diff --git a/x-pack/filebeat/processors/decode_cef/_meta/fields.yml b/x-pack/filebeat/processors/decode_cef/_meta/fields.yml new file mode 100644 index 000000000000..402c7b68b01a --- /dev/null +++ b/x-pack/filebeat/processors/decode_cef/_meta/fields.yml @@ -0,0 +1,70 @@ +- key: cef + title: Decode CEF processor fields + description: > + Common Event Format (CEF) data. + fields: + - name: cef + type: group + description: > + By default the `decode_cef` processor writes all data from the CEF + message to this `cef` object. It contains the CEF header fields and the + extension data. + fields: + - name: version + type: keyword + description: > + Version of the CEF specification used by the message. + + - name: device.vendor + type: keyword + description: > + Vendor of the device that produced the message. + + - name: device.product + type: keyword + description: > + Product of the device that produced the message. + + - name: device.version + type: keyword + description: > + Version of the product that produced the message. + + - name: device.event_class_id + type: keyword + description: > + Unique identifier of the event type. + + - name: severity + type: keyword + example: Very-High + description: > + Importance of the event. The valid string values are Unknown, Low, + Medium, High, and Very-High. The valid integer values are 0-3=Low, + 4-6=Medium, 7- 8=High, and 9-10=Very-High. + + - name: name + type: keyword + description: > + Short description of the event. + + - name: extensions + type: object + object_type: keyword + description: > + Collection of key-value pairs carried in the CEF extension field. + + - name: observer.product + type: keyword + description: + Product name. + + - name: source.service.name + type: keyword + description: + Service that is the source of the event. + + - name: destination.service.name + type: keyword + description: + Service that is the target of the event. diff --git a/x-pack/filebeat/processors/decode_cef/cef/.gitignore b/x-pack/filebeat/processors/decode_cef/cef/.gitignore new file mode 100644 index 000000000000..e92895af780a --- /dev/null +++ b/x-pack/filebeat/processors/decode_cef/cef/.gitignore @@ -0,0 +1,2 @@ +cef.svg +*.dot diff --git a/x-pack/filebeat/processors/decode_cef/cef/cef.go b/x-pack/filebeat/processors/decode_cef/cef/cef.go new file mode 100644 index 000000000000..d5b82c03e050 --- /dev/null +++ b/x-pack/filebeat/processors/decode_cef/cef/cef.go @@ -0,0 +1,138 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cef + +import "bytes" + +// Parser is generated from a ragel state machine using the following command: +//go:generate ragel -Z -G1 cef.rl -o parser.go +//go:generate go fmt parser.go + +// An SVG rendering of the state machine can be viewed by opening cef.svg in +// Chrome / Firefox. +//go:generate ragel -V -p cef.rl -o cef.dot +//go:generate dot -T svg cef.dot -o cef.svg + +// Event is a single CEF message. +type Event struct { + // CEF version. + Version int `json:"version"` + + // Vendor of the sending device. + DeviceVendor string `json:"device_vendor"` + + // Product of the sending device. + DeviceProduct string `json:"device_product"` + + // Version of the sending device. + DeviceVersion string `json:"device_version"` + + // Device Event Class ID identifies the type of event reported + DeviceEventClassID string `json:"device_event_class_id"` + + // Human-readable and understandable description of the event. + Name string `json:"name"` + + // Importance of the event. The valid string values are Unknown, Low, + // Medium, High, and Very-High. The valid integer values are 0-3=Low, + // 4-6=Medium, 7- 8=High, and 9-10=Very-High. + Severity string `json:"severity"` + + // Extensions is a collection of key-value pairs. The keys are part of a + // predefined set. The standard allows for including additional keys as + // outlined in "ArcSight Extension Directory". An event can contain any + // number of key-value pairs in any order. + Extensions map[string]string `json:"extensions,omitempty"` +} + +func (e *Event) init() { + e.Version = -1 + e.DeviceVendor = "" + e.DeviceProduct = "" + e.DeviceVersion = "" + e.DeviceEventClassID = "" + e.Name = "" + e.Severity = "" + e.Extensions = nil +} + +func (e *Event) pushExtension(key []byte, value []byte) { + if e.Extensions == nil { + e.Extensions = map[string]string{} + } + e.Extensions[string(key)] = string(value) +} + +// Unpack unpacks a common event format (CEF) message. The data is expected to +// be UTF-8 encoded and must begin with the CEF message header (e.g. starts +// with "CEF:"). +// +// The CEF message consists of a header followed by a series of key-value pairs. +// +// CEF:Version|Device Vendor|Device Product|Device Version|Device Event Class ID|Name|Severity|[Extension] +// +// The header is a series of pipe delimited values. If a pipe (|) is used in a +// header value, it has to be escaped with a backslash (\). If a backslash is +// used is must be escaped with another backslash. +// +// The extension contains key-value pairs. The equals sign (=) separates each +// key from value. And key-value pairs are separated by a single space +// (e.g. "src=1.2.3.4 dst=8.8.8.8"). If an equals sign is used as part of the +// value then it must be escaped with a backslash (\). If a backslash is used is +// must be escaped with another backslash. +// +// Extension keys must begin with an alphanumeric or underscore (_) character +// and may contain alphanumeric, underscore (_), period (.), comma (,), and +// brackets ([) (]). This is less strict than the CEF specification, but aligns +// the key names used in practice. +func (e *Event) Unpack(data []byte, opts ...Option) error { + var settings Settings + for _, opt := range opts { + opt.Apply(&settings) + } + + err := e.unpack(data) + + if settings.fullExtensionNames { + for key, v := range e.Extensions { + fullName, found := fullNameMapping[key] + if !found || key == fullName { + continue + } + + e.Extensions[fullName] = v + delete(e.Extensions, key) + } + } + + return err +} + +var ( + backslash = []byte(`\`) + escapedBackslash = []byte(`\\`) + + pipe = []byte(`|`) + escapedPipe = []byte(`\|`) + + equalsSign = []byte(`=`) + escapedEqualsSign = []byte(`\=`) +) + +func replaceHeaderEscapes(b []byte) []byte { + if bytes.IndexByte(b, '\\') != -1 { + b = bytes.ReplaceAll(b, escapedBackslash, backslash) + b = bytes.ReplaceAll(b, escapedPipe, pipe) + } + return b +} + +func replaceExtensionEscapes(b []byte) []byte { + if bytes.IndexByte(b, '\\') != -1 { + b = bytes.ReplaceAll(b, escapedBackslash, backslash) + b = bytes.ReplaceAll(b, escapedEqualsSign, equalsSign) + } + return b +} diff --git a/x-pack/filebeat/processors/decode_cef/cef/cef.rl b/x-pack/filebeat/processors/decode_cef/cef/cef.rl new file mode 100644 index 000000000000..55248ab1534b --- /dev/null +++ b/x-pack/filebeat/processors/decode_cef/cef/cef.rl @@ -0,0 +1,159 @@ +// Code generated by ragel DO NOT EDIT. +package cef + +import ( + "fmt" + "strconv" + + "go.uber.org/multierr" +) + +%%{ + machine cef; + write data; + variable p p; + variable pe pe; +}%% + +// unpack unpacks a CEF message. +func (e *Event) unpack(data []byte) error { + cs, p, pe, eof := 0, 0, len(data), len(data) + mark := 0 + + // Extension key. + var extKey []byte + + // Extension value start and end indices. + extValueStart, extValueEnd := 0, 0 + + // recoveredErrs are problems with the message that the parser was able to + // recover from (though the parsing might not be "correct"). + var recoveredErrs []error + + e.init() + + %%{ + # Actions to execute while executing state machine. + action mark { + mark = p + } + action version { + e.Version, _ = strconv.Atoi(string(data[mark:p])) + } + action device_vendor { + e.DeviceVendor = string(replaceHeaderEscapes(data[mark:p])) + } + action device_product { + e.DeviceProduct = string(replaceHeaderEscapes(data[mark:p])) + } + action device_version { + e.DeviceVersion = string(replaceHeaderEscapes(data[mark:p])) + } + action device_event_class_id { + e.DeviceEventClassID = string(replaceHeaderEscapes(data[mark:p])) + } + action name { + e.Name = string(replaceHeaderEscapes(data[mark:p])) + } + action severity { + e.Severity = string(data[mark:p]) + } + action extension_key { + // A new extension key marks the end of the last extension value. + if len(extKey) > 0 && extValueStart <= mark - 1 { + e.pushExtension(extKey, replaceExtensionEscapes(data[extValueStart:mark-1])) + extKey, extValueStart, extValueEnd = nil, 0, 0 + } + extKey = data[mark:p] + } + action extension_value_start { + extValueStart = p; + extValueEnd = p + } + action extension_value_mark { + extValueEnd = p+1 + } + action extension_eof { + // Reaching the EOF marks the end of the final extension value. + if len(extKey) > 0 && extValueStart <= extValueEnd { + e.pushExtension(extKey, replaceExtensionEscapes(data[extValueStart:extValueEnd])) + extKey, extValueStart, extValueEnd = nil, 0, 0 + } + } + action extension_err { + recoveredErrs = append(recoveredErrs, fmt.Errorf("malformed value for %s at pos %d", extKey, p+1)) + fhold; fgoto gobble_extension; + } + action recover_next_extension { + extKey, extValueStart, extValueEnd = nil, 0, 0 + // Resume processing at p, the start of the next extension key. + p = mark; + fgoto extensions; + } + + # Define what header characters are allowed. + pipe = "|"; + escape = "\\"; + escape_pipe = escape pipe; + backslash = "\\\\"; + device_chars = backslash | escape_pipe | (any -- pipe -- escape); + severity_chars = ( alpha | digit | "-" ); + + # Header fields. + version = "CEF:" digit+ >mark %version; + device_vendor = device_chars* >mark %device_vendor; + device_product = device_chars* >mark %device_product; + device_version = device_chars* >mark %device_version; + device_event_class_id = device_chars* >mark %device_event_class_id; + name = device_chars* >mark %name; + severity = severity_chars* >mark %severity; + + header = version pipe + device_vendor pipe + device_product pipe + device_version pipe + device_event_class_id pipe + name pipe + severity pipe; + + # Define what extension characters are allowed. + equal = "="; + escape_equal = escape equal; + # Only alnum is defined in the CEF spec. The other characters allow + # non-conforming extension keys to be parsed. + extension_key_start_chars = alnum | '_'; + extension_key_chars = extension_key_start_chars | '.' | ',' | '[' | ']'; + extension_key_pattern = extension_key_start_chars extension_key_chars*; + extension_value_chars = backslash | escape_equal | (any -- equal -- escape); + + # Extension fields. + extension_key = extension_key_pattern >mark %extension_key; + extension_value = (extension_value_chars @extension_value_mark)* >extension_value_start $err(extension_err); + extension = extension_key equal extension_value %/extension_eof; + extensions = " "* extension (" " extension)*; + + # gobble_extension attempts recovery from a malformed value by trying to + # advance to the next extension key and re-entering the main state machine. + gobble_extension := any* (" " >mark) extension_key_pattern equal @recover_next_extension; + + # CEF message. + cef = header extensions?; + + main := cef; + write init; + write exec; + }%% + + // Check if state machine completed. + if cs < cef_first_final { + // Reached an early end. + if p == pe { + return multierr.Append(multierr.Combine(recoveredErrs...), fmt.Errorf("unexpected end of CEF event")) + } + + // Encountered invalid input. + return multierr.Append(multierr.Combine(recoveredErrs...), fmt.Errorf("error in CEF event at pos %d", p+1)) + } + + return multierr.Combine(recoveredErrs...) +} diff --git a/x-pack/filebeat/processors/decode_cef/cef/cef_test.go b/x-pack/filebeat/processors/decode_cef/cef/cef_test.go new file mode 100644 index 000000000000..4e7f2c8f1306 --- /dev/null +++ b/x-pack/filebeat/processors/decode_cef/cef/cef_test.go @@ -0,0 +1,350 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cef + +import ( + "crypto/sha1" + "encoding/hex" + "flag" + "io/ioutil" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +var generateCorpus = flag.Bool("corpus", false, "generate fuzz corpus from test cases") + +const ( + standardMessage = `CEF:26|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232 eventId=1` + + headerOnly = `CEF:26|security|threatmanager|1.0|100|trojan successfully stopped|10|` + + emptyDeviceFields = `CEF:0|||1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232` + + escapedPipeInHeader = `CEF:26|security|threat\|->manager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232` + + equalsSignInHeader = `CEF:26|security|threat=manager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst=12.121.122.82 spt=1232` + + emptyExtensionValue = `CEF:26|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 dst= spt=1232` + + leadingWhitespace = `CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10| src=10.0.0.192 dst=12.121.122.82 spt=1232` + + escapedPipeInExtension = `CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this\|has an escaped pipe` + + pipeInMessage = `CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this|has an pipe` + + equalsInMessage = `CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|moo=this =has = equals\=` + + escapesInExtension = `CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|msg=a+b\=c x=c\\d\=z` + + malformedExtensionEscape = `CEF:0|FooBar|Web Gateway|1.2.3.45.67|200|Success|2|rt=Sep 07 2018 14:50:39 cat=Access Log dst=1.1.1.1 dhost=foo.example.com suser=redacted src=2.2.2.2 requestMethod=POST request='https://foo.example.com/bar/bingo/1' requestClientApplication='Foo-Bar/2018.1.7; =Email:user@example.com; Guid:test=' cs1= cs1Label=Foo Bar` + + multipleMalformedExtensionValues = `CEF:0|vendor|product|version|event_id|name|Very-High| msg=Hello World error=Failed because id==old_id user=root angle=106.7<=180` +) + +var testMessages = []string{ + standardMessage, + headerOnly, + emptyDeviceFields, + escapedPipeInHeader, + equalsSignInHeader, + emptyExtensionValue, + leadingWhitespace, + escapedPipeInExtension, + pipeInMessage, + equalsInMessage, + escapesInExtension, + malformedExtensionEscape, + multipleMalformedExtensionValues, +} + +func TestGenerateFuzzCorpus(t *testing.T) { + if !*generateCorpus { + t.Skip("-corpus is not enabled") + } + + for _, m := range testMessages { + h := sha1.New() + h.Write([]byte(m)) + name := hex.EncodeToString(h.Sum(nil)) + + ioutil.WriteFile(filepath.Join("fuzz/corpus", name), []byte(m), 0644) + } +} + +func TestEventUnpack(t *testing.T) { + t.Run("standardMessage", func(t *testing.T) { + var e Event + err := e.Unpack([]byte(standardMessage)) + assert.NoError(t, err) + assert.Equal(t, 26, e.Version) + assert.Equal(t, "security", e.DeviceVendor) + assert.Equal(t, "threatmanager", e.DeviceProduct) + assert.Equal(t, "1.0", e.DeviceVersion) + assert.Equal(t, "100", e.DeviceEventClassID) + assert.Equal(t, "trojan successfully stopped", e.Name) + assert.Equal(t, "10", e.Severity) + assert.Equal(t, map[string]string{ + "src": "10.0.0.192", + "dst": "12.121.122.82", + "spt": "1232", + "eventId": "1", + }, e.Extensions) + }) + + t.Run("headerOnly", func(t *testing.T) { + var e Event + err := e.Unpack([]byte(headerOnly)) + assert.NoError(t, err) + assert.Equal(t, 26, e.Version) + assert.Equal(t, "security", e.DeviceVendor) + assert.Equal(t, "threatmanager", e.DeviceProduct) + assert.Equal(t, "1.0", e.DeviceVersion) + assert.Equal(t, "100", e.DeviceEventClassID) + assert.Equal(t, "trojan successfully stopped", e.Name) + assert.Equal(t, "10", e.Severity) + assert.Nil(t, e.Extensions) + }) + + t.Run("escapedPipeInHeader", func(t *testing.T) { + var e Event + err := e.Unpack([]byte(escapedPipeInHeader)) + assert.NoError(t, err) + assert.Equal(t, 26, e.Version) + assert.Equal(t, "security", e.DeviceVendor) + assert.Equal(t, "threat|->manager", e.DeviceProduct) + assert.Equal(t, "1.0", e.DeviceVersion) + assert.Equal(t, "100", e.DeviceEventClassID) + assert.Equal(t, "trojan successfully stopped", e.Name) + assert.Equal(t, "10", e.Severity) + assert.Equal(t, map[string]string{ + "src": "10.0.0.192", + "dst": "12.121.122.82", + "spt": "1232", + }, e.Extensions) + }) + + t.Run("equalsSignInHeader", func(t *testing.T) { + var e Event + err := e.Unpack([]byte(equalsSignInHeader)) + assert.NoError(t, err) + assert.Equal(t, 26, e.Version) + assert.Equal(t, "security", e.DeviceVendor) + assert.Equal(t, "threat=manager", e.DeviceProduct) + assert.Equal(t, "1.0", e.DeviceVersion) + assert.Equal(t, "100", e.DeviceEventClassID) + assert.Equal(t, "trojan successfully stopped", e.Name) + assert.Equal(t, "10", e.Severity) + assert.Equal(t, map[string]string{ + "src": "10.0.0.192", + "dst": "12.121.122.82", + "spt": "1232", + }, e.Extensions) + }) + + t.Run("emptyExtensionValue", func(t *testing.T) { + var e Event + err := e.Unpack([]byte(emptyExtensionValue)) + assert.NoError(t, err) + assert.Equal(t, 26, e.Version) + assert.Equal(t, "security", e.DeviceVendor) + assert.Equal(t, "threatmanager", e.DeviceProduct) + assert.Equal(t, "1.0", e.DeviceVersion) + assert.Equal(t, "100", e.DeviceEventClassID) + assert.Equal(t, "trojan successfully stopped", e.Name) + assert.Equal(t, "10", e.Severity) + assert.Equal(t, map[string]string{ + "src": "10.0.0.192", + "dst": "", + "spt": "1232", + }, e.Extensions) + }) + + t.Run("emptyDeviceFields", func(t *testing.T) { + var e Event + err := e.Unpack([]byte(emptyDeviceFields)) + assert.NoError(t, err) + assert.Equal(t, 0, e.Version) + assert.Equal(t, "", e.DeviceVendor) + assert.Equal(t, "", e.DeviceProduct) + assert.Equal(t, "1.0", e.DeviceVersion) + assert.Equal(t, "100", e.DeviceEventClassID) + assert.Equal(t, "trojan successfully stopped", e.Name) + assert.Equal(t, "10", e.Severity) + assert.Equal(t, map[string]string{ + "src": "10.0.0.192", + "dst": "12.121.122.82", + "spt": "1232", + }, e.Extensions) + }) + + t.Run("errorEscapedPipeInExtension", func(t *testing.T) { + var e Event + err := e.Unpack([]byte(escapedPipeInExtension)) + assert.Equal(t, 0, e.Version) + assert.Equal(t, "security", e.DeviceVendor) + assert.Equal(t, "threatmanager", e.DeviceProduct) + assert.Equal(t, "1.0", e.DeviceVersion) + assert.Equal(t, "100", e.DeviceEventClassID) + assert.Equal(t, "trojan successfully stopped", e.Name) + assert.Equal(t, "10", e.Severity) + assert.Nil(t, e.Extensions) + + // Pipes in extensions should not be escaped. + assert.Error(t, err) + }) + + t.Run("leadingWhitespace", func(t *testing.T) { + var e Event + err := e.Unpack([]byte(leadingWhitespace)) + assert.NoError(t, err) + assert.Equal(t, 0, e.Version) + assert.Equal(t, "security", e.DeviceVendor) + assert.Equal(t, "threatmanager", e.DeviceProduct) + assert.Equal(t, "1.0", e.DeviceVersion) + assert.Equal(t, "100", e.DeviceEventClassID) + assert.Equal(t, "trojan successfully stopped", e.Name) + assert.Equal(t, "10", e.Severity) + assert.Equal(t, map[string]string{ + "src": "10.0.0.192", + "dst": "12.121.122.82", + "spt": "1232", + }, e.Extensions) + }) + + t.Run("pipeInMessage", func(t *testing.T) { + var e Event + err := e.Unpack([]byte(pipeInMessage)) + assert.NoError(t, err) + assert.Equal(t, 0, e.Version) + assert.Equal(t, "security", e.DeviceVendor) + assert.Equal(t, "threatmanager", e.DeviceProduct) + assert.Equal(t, "1.0", e.DeviceVersion) + assert.Equal(t, "100", e.DeviceEventClassID) + assert.Equal(t, "trojan successfully stopped", e.Name) + assert.Equal(t, "10", e.Severity) + assert.Equal(t, map[string]string{ + "moo": "this|has an pipe", + }, e.Extensions) + }) + + t.Run("errorEqualsInMessage", func(t *testing.T) { + var e Event + err := e.Unpack([]byte(equalsInMessage)) + assert.Equal(t, 0, e.Version) + assert.Equal(t, "security", e.DeviceVendor) + assert.Equal(t, "threatmanager", e.DeviceProduct) + assert.Equal(t, "1.0", e.DeviceVersion) + assert.Equal(t, "100", e.DeviceEventClassID) + assert.Equal(t, "trojan successfully stopped", e.Name) + assert.Equal(t, "10", e.Severity) + assert.Nil(t, e.Extensions) + + // moo contains unescaped equals signs. + assert.Error(t, err) + }) + + t.Run("escapesInExtension", func(t *testing.T) { + var e Event + err := e.Unpack([]byte(escapesInExtension)) + assert.NoError(t, err) + assert.Equal(t, 0, e.Version) + assert.Equal(t, "security", e.DeviceVendor) + assert.Equal(t, "threatmanager", e.DeviceProduct) + assert.Equal(t, "1.0", e.DeviceVersion) + assert.Equal(t, "100", e.DeviceEventClassID) + assert.Equal(t, "trojan successfully stopped", e.Name) + assert.Equal(t, "10", e.Severity) + assert.Equal(t, map[string]string{ + "msg": "a+b=c", + "x": `c\d=z`, + }, e.Extensions) + }) + + t.Run("errorMalformedExtensionEscape", func(t *testing.T) { + var e Event + err := e.Unpack([]byte(malformedExtensionEscape)) + assert.Equal(t, 0, e.Version) + assert.Equal(t, "FooBar", e.DeviceVendor) + assert.Equal(t, "Web Gateway", e.DeviceProduct) + assert.Equal(t, "1.2.3.45.67", e.DeviceVersion) + assert.Equal(t, "200", e.DeviceEventClassID) + assert.Equal(t, "Success", e.Name) + assert.Equal(t, "2", e.Severity) + assert.Equal(t, map[string]string{ + "rt": "Sep 07 2018 14:50:39", + "cat": "Access Log", + "dst": "1.1.1.1", + "dhost": "foo.example.com", + "suser": "redacted", + "src": "2.2.2.2", + "requestMethod": "POST", + "request": `'https://foo.example.com/bar/bingo/1'`, + "cs1": "", + "cs1Label": "Foo Bar", + }, e.Extensions) + + // requestClientApplication is not valid because it contains an unescaped + // equals sign. + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "requestClientApplication") + } + }) + + t.Run("errorMultipleMalformedExtensionValues", func(t *testing.T) { + var e Event + err := e.Unpack([]byte(multipleMalformedExtensionValues)) + assert.Equal(t, 0, e.Version) + assert.Equal(t, "vendor", e.DeviceVendor) + assert.Equal(t, "product", e.DeviceProduct) + assert.Equal(t, "version", e.DeviceVersion) + assert.Equal(t, "event_id", e.DeviceEventClassID) + assert.Equal(t, "name", e.Name) + assert.Equal(t, "Very-High", e.Severity) + assert.Equal(t, map[string]string{ + "msg": "Hello World", + "error": "Failed because", + "user": "root", + }, e.Extensions) + + // Both id and angle contain unescaped equals signs. + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "id") + assert.Contains(t, err.Error(), "malformed") + } + }) + + t.Run("empty", func(t *testing.T) { + var e Event + err := e.Unpack([]byte("CEF:0|||||||a=")) + assert.NoError(t, err) + }) +} + +func TestEventUnpackWithFullExtensionNames(t *testing.T) { + var e Event + err := e.Unpack([]byte(standardMessage), WithFullExtensionNames()) + assert.NoError(t, err) + assert.Equal(t, map[string]string{ + "sourceAddress": "10.0.0.192", + "destinationAddress": "12.121.122.82", + "sourcePort": "1232", + "eventId": "1", + }, e.Extensions) +} + +func BenchmarkEventUnpack(b *testing.B) { + var messages [][]byte + for _, m := range testMessages { + messages = append(messages, []byte(m)) + } + b.ResetTimer() + + for i := 0; i < b.N; i++ { + var e Event + e.Unpack(messages[i%len(messages)]) + } +} diff --git a/x-pack/filebeat/processors/decode_cef/cef/cmd/cef2json/.gitignore b/x-pack/filebeat/processors/decode_cef/cef/cmd/cef2json/.gitignore new file mode 100644 index 000000000000..17a45d0177e3 --- /dev/null +++ b/x-pack/filebeat/processors/decode_cef/cef/cmd/cef2json/.gitignore @@ -0,0 +1,2 @@ +cef2json +cef2json.exe diff --git a/x-pack/filebeat/processors/decode_cef/cef/cmd/cef2json/cef2json.go b/x-pack/filebeat/processors/decode_cef/cef/cmd/cef2json/cef2json.go new file mode 100644 index 000000000000..e11b9309eb2b --- /dev/null +++ b/x-pack/filebeat/processors/decode_cef/cef/cmd/cef2json/cef2json.go @@ -0,0 +1,64 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package main + +import ( + "bufio" + "bytes" + "encoding/json" + "flag" + "fmt" + "log" + "os" + + "github.com/elastic/beats/x-pack/filebeat/processors/decode_cef/cef" +) + +var ( + fullExtensionNames bool +) + +func init() { + flag.BoolVar(&fullExtensionNames, "full", true, "Use full extension key names.") +} + +var cefMarker = []byte("CEF:") + +func main() { + log.SetFlags(0) + flag.Parse() + + var opts []cef.Option + if fullExtensionNames { + opts = append(opts, cef.WithFullExtensionNames()) + } + + s := bufio.NewScanner(os.Stdin) + for s.Scan() { + line := s.Bytes() + if len(line) == 0 { + continue + } + + begin := bytes.Index(line, cefMarker) + if begin == -1 { + continue + } + line = line[begin:] + + var e cef.Event + if err := e.Unpack(line, opts...); err != nil { + log.Println("ERROR:", err, "in:", string(line)) + } + + jsonData, err := json.Marshal(e) + if err != nil { + log.Println("ERROR:", err) + continue + } + + fmt.Println(string(jsonData)) + } +} diff --git a/x-pack/filebeat/processors/decode_cef/cef/fuzz/.gitignore b/x-pack/filebeat/processors/decode_cef/cef/fuzz/.gitignore new file mode 100644 index 000000000000..45ef0afd007d --- /dev/null +++ b/x-pack/filebeat/processors/decode_cef/cef/fuzz/.gitignore @@ -0,0 +1,4 @@ +corpus +crashers +suppressions +*.zip diff --git a/x-pack/filebeat/processors/decode_cef/cef/fuzz/Makefile b/x-pack/filebeat/processors/decode_cef/cef/fuzz/Makefile new file mode 100644 index 000000000000..5e928e83034c --- /dev/null +++ b/x-pack/filebeat/processors/decode_cef/cef/fuzz/Makefile @@ -0,0 +1,7 @@ +fuzz: + go test ../. -corpus + go get github.com/dvyukov/go-fuzz/go-fuzz github.com/dvyukov/go-fuzz/go-fuzz-build + go-fuzz-build + go-fuzz + +.PHONY: fuzz diff --git a/x-pack/filebeat/processors/decode_cef/cef/fuzz/fuzz.go b/x-pack/filebeat/processors/decode_cef/cef/fuzz/fuzz.go new file mode 100644 index 000000000000..427daae679d8 --- /dev/null +++ b/x-pack/filebeat/processors/decode_cef/cef/fuzz/fuzz.go @@ -0,0 +1,18 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package fuzz + +import ( + cef2 "github.com/elastic/beats/x-pack/filebeat/processors/decode_cef/cef" +) + +// Fuzz is the entry point that go-fuzz uses to fuzz the parser. +func Fuzz(data []byte) int { + var e cef2.Event + if err := e.Unpack(data); err != nil { + return 1 + } + return 0 +} diff --git a/x-pack/filebeat/processors/decode_cef/cef/keys.go b/x-pack/filebeat/processors/decode_cef/cef/keys.go new file mode 100644 index 000000000000..4ce02d65d55f --- /dev/null +++ b/x-pack/filebeat/processors/decode_cef/cef/keys.go @@ -0,0 +1,169 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cef + +// fullNameMapping is a mapping of CEF key names to full field names. +// This mapping was generated from tables contained in "Micro Focus Security +// ArcSight Common Event Format Version 25" dated September 28, 2017. +var fullNameMapping = map[string]string{ + "agt": "agentAddress", + "agentDnsDomain": "agentDnsDomain", + "ahost": "agentHostName", + "aid": "agentId", + "amac": "agentMacAddress", + "agentNtDomain": "agentNtDomain", + "art": "agentReceiptTime", + "atz": "agentTimeZone", + "agentTranslatedAddress": "agentTranslatedAddress", + "agentTranslatedZoneExternalID": "agentTranslatedZoneExternalID", + "agentTranslatedZoneURI": "agentTranslatedZoneURI", + "at": "agentType", + "av": "agentVersion", + "agentZoneExternalID": "agentZoneExternalID", + "agentZoneURI": "agentZoneURI", + "app": "applicationProtocol", + "cnt": "baseEventCount", + "in": "bytesIn", + "out": "bytesOut", + "customerExternalID": "customerExternalID", + "customerURI": "customerURI", + "dst": "destinationAddress", + "destinationDnsDomain": "destinationDnsDomain", + "dlat": "destinationGeoLatitude", + "dlong": "destinationGeoLongitude", + "dhost": "destinationHostName", + "dmac": "destinationMacAddress", + "dntdom": "destinationNtDomain", + "dpt": "destinationPort", + "dpid": "destinationProcessId", + "dproc": "destinationProcessName", + "destinationServiceName": "destinationServiceName", + "destinationTranslatedAddress": "destinationTranslatedAddress", + "destinationTranslatedPort": "destinationTranslatedPort", + "destinationTranslatedZoneExternalID": "destinationTranslatedZoneExternalID", + "destinationTranslatedZoneURI": "destinationTranslatedZoneURI", + "duid": "destinationUserId", + "duser": "destinationUserName", + "dpriv": "destinationUserPrivileges", + "destinationZoneExternalID": "destinationZoneExternalID", + "destinationZoneURI": "destinationZoneURI", + "act": "deviceAction", + "dvc": "deviceAddress", + "cfp1Label": "deviceCustomFloatingPoint1Label", + "cfp3Label": "deviceCustomFloatingPoint3Label", + "cfp4Label": "deviceCustomFloatingPoint4Label", + "deviceCustomDate1": "deviceCustomDate1", + "deviceCustomDate1Label": "deviceCustomDate1Label", + "deviceCustomDate2": "deviceCustomDate2", + "deviceCustomDate2Label": "deviceCustomDate2Label", + "cfp1": "deviceCustomFloatingPoint1", + "cfp2": "deviceCustomFloatingPoint2", + "cfp2Label": "deviceCustomFloatingPoint2Label", + "cfp3": "deviceCustomFloatingPoint3", + "cfp4": "deviceCustomFloatingPoint4", + "c6a1Label": "deviceCustomIPv6Address1Label", + "c6a4": "deviceCustomIPv6Address4", + "C6a4Label": "deviceCustomIPv6Address4Label", + "c6a1": "deviceCustomIPv6Address1", + "c6a3": "deviceCustomIPv6Address3", + "c6a3Label": "deviceCustomIPv6Address3Label", + "cn1": "deviceCustomNumber1", + "cn1Label": "deviceCustomNumber1Label", + "cn2": "DeviceCustomNumber2", + "cn2Label": "deviceCustomNumber2Label", + "cn3": "deviceCustomNumber3", + "cn3Label": "deviceCustomNumber3Label", + "cs1": "deviceCustomString1", + "cs1Label": "deviceCustomString1Label", + "cs2": "deviceCustomString2", + "cs2Label": "deviceCustomString2Label", + "cs3": "deviceCustomString3", + "cs3Label": "deviceCustomString3Label", + "cs4": "deviceCustomString4", + "cs4Label": "deviceCustomString4Label", + "cs5": "deviceCustomString5", + "cs5Label": "deviceCustomString5Label", + "cs6": "deviceCustomString6", + "cs6Label": "deviceCustomString6Label", + "deviceDirection": "deviceDirection", + "deviceDnsDomain": "deviceDnsDomain", + "cat": "deviceEventCategory", + "deviceExternalId": "deviceExternalId", + "deviceFacility": "deviceFacility", + "dvchost": "deviceHostName", + "deviceInboundInterface": "deviceInboundInterface", + "dvcmac": "deviceMacAddress", + "deviceNtDomain": "deviceNtDomain", + "DeviceOutboundInterface": "deviceOutboundInterface", + "DevicePayloadId": "devicePayloadId", + "dvcpid": "deviceProcessId", + "deviceProcessName": "deviceProcessName", + "rt": "deviceReceiptTime", + "dtz": "deviceTimeZone", + "deviceTranslatedAddress": "deviceTranslatedAddress", + "deviceTranslatedZoneExternalID": "deviceTranslatedZoneExternalID", + "deviceTranslatedZoneURI": "deviceTranslatedZoneURI", + "deviceZoneExternalID": "deviceZoneExternalID", + "deviceZoneURI": "deviceZoneURI", + "end": "endTime", + "eventId": "eventId", + "outcome": "eventOutcome", + "externalId": "externalId", + "fileCreateTime": "fileCreateTime", + "fileHash": "fileHash", + "fileId": "fileId", + "fileModificationTime": "fileModificationTime", + "fname": "filename", + "filePath": "filePath", + "filePermission": "filePermission", + "fsize": "fileSize", + "fileType": "fileType", + "flexDate1": "flexDate1", + "flexDate1Label": "flexDate1Label", + "flexString1": "flexString1", + "flexString2": "flexString2", + "flexString1Label": "flexString2Label", + "flexString2Label": "flexString2Label", + "msg": "message", + "oldFileCreateTime": "oldFileCreateTime", + "oldFileHash": "oldFileHash", + "oldFileId": "oldFileId", + "oldFileModificationTime": "oldFileModificationTime", + "oldFileName": "oldFileName", + "oldFilePath": "oldFilePath", + "oldFilePermission": "oldFilePermission", + "oldFileSize": "oldFileSize", + "oldFileType": "oldFileType", + "rawEvent": "rawEvent", + "reason": "Reason", + "requestClientApplication": "requestClientApplication", + "requestContext": "requestContext", + "requestCookies": "requestCookies", + "requestMethod": "requestMethod", + "request": "requestUrl", + "src": "sourceAddress", + "sourceDnsDomain": "sourceDnsDomain", + "slat": "sourceGeoLatitude", + "slong": "sourceGeoLongitude", + "shost": "sourceHostName", + "smac": "sourceMacAddress", + "sntdom": "sourceNtDomain", + "spt": "sourcePort", + "spid": "sourceProcessId", + "sproc": "sourceProcessName", + "sourceServiceName": "sourceServiceName", + "sourceTranslated": "Address", + "sourceTranslatedPort": "sourceTranslatedPort", + "sourceTranslatedZoneExternalID": "sourceTranslatedZoneExternalID", + "sourceTranslatedZoneURI": "sourceTranslatedZoneURI", + "suid": "sourceUserId", + "suser": "sourceUserName", + "spriv": "sourceUserPrivileges", + "sourceZoneExternalID": "sourceZoneExternalID", + "sourceZoneURI": "sourceZoneURI", + "start": "startTime", + "proto": "transportProtocol", + "type": "type", +} diff --git a/x-pack/filebeat/processors/decode_cef/cef/option.go b/x-pack/filebeat/processors/decode_cef/cef/option.go new file mode 100644 index 000000000000..9fae32f011cd --- /dev/null +++ b/x-pack/filebeat/processors/decode_cef/cef/option.go @@ -0,0 +1,27 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cef + +// Option controls Setting used in unpacking messages. +type Option interface { + Apply(*Settings) +} + +// Settings for unpacking messages. +type Settings struct { + fullExtensionNames bool +} + +type withFullExtensionNames struct{} + +func (w withFullExtensionNames) Apply(s *Settings) { + s.fullExtensionNames = true +} + +// WithFullExtensionNames causes CEF extension key names to be translated into +// their full key names (e.g. src -> sourceAddress). +func WithFullExtensionNames() Option { + return withFullExtensionNames{} +} diff --git a/x-pack/filebeat/processors/decode_cef/cef/parser.go b/x-pack/filebeat/processors/decode_cef/cef/parser.go new file mode 100644 index 000000000000..f7ee7802dad7 --- /dev/null +++ b/x-pack/filebeat/processors/decode_cef/cef/parser.go @@ -0,0 +1,1001 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//line cef.rl:1 +// Code generated by ragel DO NOT EDIT. +package cef + +import ( + "fmt" + "strconv" + + "go.uber.org/multierr" +) + +//line parser.go:15 +var _cef_eof_actions []byte = []byte{ + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 16, 16, 0, 0, 0, 0, + 19, 22, 22, 22, 19, 22, 22, 22, + 0, +} + +const cef_start int = 1 +const cef_first_final int = 31 +const cef_error int = 0 + +const cef_en_gobble_extension int = 28 +const cef_en_main int = 1 +const cef_en_main_cef_extensions int = 24 + +//line cef.rl:16 + +// unpack unpacks a CEF message. +func (e *Event) unpack(data []byte) error { + cs, p, pe, eof := 0, 0, len(data), len(data) + mark := 0 + + // Extension key. + var extKey []byte + + // Extension value start and end indices. + extValueStart, extValueEnd := 0, 0 + + // recoveredErrs are problems with the message that the parser was able to + // recover from (though the parsing might not be "correct"). + var recoveredErrs []error + + e.init() + +//line parser.go:55 + { + cs = cef_start + } + +//line parser.go:60 + { + if (p) == (pe) { + goto _test_eof + } + if cs == 0 { + goto _out + } + _resume: + switch cs { + case 1: + if data[(p)] == 67 { + goto tr0 + } + goto tr1 + case 0: + goto _out + case 2: + if data[(p)] == 69 { + goto tr2 + } + goto tr1 + case 3: + if data[(p)] == 70 { + goto tr3 + } + goto tr1 + case 4: + if data[(p)] == 58 { + goto tr4 + } + goto tr1 + case 5: + if 48 <= data[(p)] && data[(p)] <= 57 { + goto tr5 + } + goto tr1 + case 6: + if data[(p)] == 124 { + goto tr7 + } + if 48 <= data[(p)] && data[(p)] <= 57 { + goto tr6 + } + goto tr1 + case 7: + switch data[(p)] { + case 92: + goto tr9 + case 124: + goto tr10 + } + goto tr8 + case 8: + switch data[(p)] { + case 92: + goto tr12 + case 124: + goto tr13 + } + goto tr11 + case 9: + switch data[(p)] { + case 92: + goto tr11 + case 124: + goto tr11 + } + goto tr1 + case 10: + switch data[(p)] { + case 92: + goto tr15 + case 124: + goto tr16 + } + goto tr14 + case 11: + switch data[(p)] { + case 92: + goto tr18 + case 124: + goto tr19 + } + goto tr17 + case 12: + switch data[(p)] { + case 92: + goto tr17 + case 124: + goto tr17 + } + goto tr1 + case 13: + switch data[(p)] { + case 92: + goto tr21 + case 124: + goto tr22 + } + goto tr20 + case 14: + switch data[(p)] { + case 92: + goto tr24 + case 124: + goto tr25 + } + goto tr23 + case 15: + switch data[(p)] { + case 92: + goto tr23 + case 124: + goto tr23 + } + goto tr1 + case 16: + switch data[(p)] { + case 92: + goto tr27 + case 124: + goto tr28 + } + goto tr26 + case 17: + switch data[(p)] { + case 92: + goto tr30 + case 124: + goto tr31 + } + goto tr29 + case 18: + switch data[(p)] { + case 92: + goto tr29 + case 124: + goto tr29 + } + goto tr1 + case 19: + switch data[(p)] { + case 92: + goto tr33 + case 124: + goto tr34 + } + goto tr32 + case 20: + switch data[(p)] { + case 92: + goto tr36 + case 124: + goto tr37 + } + goto tr35 + case 21: + switch data[(p)] { + case 92: + goto tr35 + case 124: + goto tr35 + } + goto tr1 + case 22: + switch data[(p)] { + case 45: + goto tr38 + case 124: + goto tr39 + } + switch { + case data[(p)] < 65: + if 48 <= data[(p)] && data[(p)] <= 57 { + goto tr38 + } + case data[(p)] > 90: + if 97 <= data[(p)] && data[(p)] <= 122 { + goto tr38 + } + default: + goto tr38 + } + goto tr1 + case 23: + switch data[(p)] { + case 45: + goto tr40 + case 124: + goto tr41 + } + switch { + case data[(p)] < 65: + if 48 <= data[(p)] && data[(p)] <= 57 { + goto tr40 + } + case data[(p)] > 90: + if 97 <= data[(p)] && data[(p)] <= 122 { + goto tr40 + } + default: + goto tr40 + } + goto tr1 + case 31: + switch data[(p)] { + case 32: + goto tr42 + case 95: + goto tr43 + } + switch { + case data[(p)] < 65: + if 48 <= data[(p)] && data[(p)] <= 57 { + goto tr43 + } + case data[(p)] > 90: + if 97 <= data[(p)] && data[(p)] <= 122 { + goto tr43 + } + default: + goto tr43 + } + goto tr1 + case 24: + switch data[(p)] { + case 32: + goto tr42 + case 95: + goto tr43 + } + switch { + case data[(p)] < 65: + if 48 <= data[(p)] && data[(p)] <= 57 { + goto tr43 + } + case data[(p)] > 90: + if 97 <= data[(p)] && data[(p)] <= 122 { + goto tr43 + } + default: + goto tr43 + } + goto tr1 + case 25: + switch data[(p)] { + case 44: + goto tr44 + case 46: + goto tr44 + case 61: + goto tr45 + case 93: + goto tr44 + case 95: + goto tr44 + } + switch { + case data[(p)] < 65: + if 48 <= data[(p)] && data[(p)] <= 57 { + goto tr44 + } + case data[(p)] > 91: + if 97 <= data[(p)] && data[(p)] <= 122 { + goto tr44 + } + default: + goto tr44 + } + goto tr1 + case 32: + switch data[(p)] { + case 32: + goto tr54 + case 61: + goto tr46 + case 92: + goto tr55 + } + goto tr53 + case 33: + switch data[(p)] { + case 32: + goto tr56 + case 61: + goto tr46 + case 92: + goto tr57 + } + goto tr48 + case 34: + switch data[(p)] { + case 32: + goto tr56 + case 61: + goto tr46 + case 92: + goto tr57 + case 95: + goto tr58 + } + switch { + case data[(p)] < 65: + if 48 <= data[(p)] && data[(p)] <= 57 { + goto tr58 + } + case data[(p)] > 90: + if 97 <= data[(p)] && data[(p)] <= 122 { + goto tr58 + } + default: + goto tr58 + } + goto tr48 + case 35: + switch data[(p)] { + case 32: + goto tr56 + case 44: + goto tr59 + case 46: + goto tr59 + case 61: + goto tr60 + case 92: + goto tr57 + case 95: + goto tr59 + } + switch { + case data[(p)] < 65: + if 48 <= data[(p)] && data[(p)] <= 57 { + goto tr59 + } + case data[(p)] > 93: + if 97 <= data[(p)] && data[(p)] <= 122 { + goto tr59 + } + default: + goto tr59 + } + goto tr48 + case 36: + switch data[(p)] { + case 32: + goto tr62 + case 61: + goto tr46 + case 92: + goto tr63 + } + goto tr61 + case 37: + switch data[(p)] { + case 32: + goto tr64 + case 61: + goto tr46 + case 92: + goto tr65 + } + goto tr47 + case 38: + switch data[(p)] { + case 32: + goto tr64 + case 61: + goto tr46 + case 92: + goto tr65 + case 95: + goto tr66 + } + switch { + case data[(p)] < 65: + if 48 <= data[(p)] && data[(p)] <= 57 { + goto tr66 + } + case data[(p)] > 90: + if 97 <= data[(p)] && data[(p)] <= 122 { + goto tr66 + } + default: + goto tr66 + } + goto tr47 + case 39: + switch data[(p)] { + case 32: + goto tr64 + case 44: + goto tr67 + case 46: + goto tr67 + case 61: + goto tr60 + case 92: + goto tr65 + case 95: + goto tr67 + } + switch { + case data[(p)] < 65: + if 48 <= data[(p)] && data[(p)] <= 57 { + goto tr67 + } + case data[(p)] > 93: + if 97 <= data[(p)] && data[(p)] <= 122 { + goto tr67 + } + default: + goto tr67 + } + goto tr47 + case 26: + switch data[(p)] { + case 61: + goto tr47 + case 92: + goto tr47 + } + goto tr46 + case 27: + switch data[(p)] { + case 61: + goto tr48 + case 92: + goto tr48 + } + goto tr46 + case 28: + if data[(p)] == 32 { + goto tr50 + } + goto tr49 + case 29: + switch data[(p)] { + case 32: + goto tr50 + case 95: + goto tr51 + } + switch { + case data[(p)] < 65: + if 48 <= data[(p)] && data[(p)] <= 57 { + goto tr51 + } + case data[(p)] > 90: + if 97 <= data[(p)] && data[(p)] <= 122 { + goto tr51 + } + default: + goto tr51 + } + goto tr49 + case 30: + switch data[(p)] { + case 32: + goto tr50 + case 44: + goto tr51 + case 46: + goto tr51 + case 61: + goto tr52 + case 93: + goto tr51 + case 95: + goto tr51 + } + switch { + case data[(p)] < 65: + if 48 <= data[(p)] && data[(p)] <= 57 { + goto tr51 + } + case data[(p)] > 91: + if 97 <= data[(p)] && data[(p)] <= 122 { + goto tr51 + } + default: + goto tr51 + } + goto tr49 + case 40: + if data[(p)] == 32 { + goto tr50 + } + goto tr49 + } + + tr1: + cs = 0 + goto _again + tr46: + cs = 0 + goto f15 + tr0: + cs = 2 + goto _again + tr2: + cs = 3 + goto _again + tr3: + cs = 4 + goto _again + tr4: + cs = 5 + goto _again + tr6: + cs = 6 + goto _again + tr5: + cs = 6 + goto f0 + tr7: + cs = 7 + goto f1 + tr11: + cs = 8 + goto _again + tr8: + cs = 8 + goto f0 + tr12: + cs = 9 + goto _again + tr9: + cs = 9 + goto f0 + tr10: + cs = 10 + goto f2 + tr13: + cs = 10 + goto f3 + tr17: + cs = 11 + goto _again + tr14: + cs = 11 + goto f0 + tr18: + cs = 12 + goto _again + tr15: + cs = 12 + goto f0 + tr16: + cs = 13 + goto f4 + tr19: + cs = 13 + goto f5 + tr23: + cs = 14 + goto _again + tr20: + cs = 14 + goto f0 + tr24: + cs = 15 + goto _again + tr21: + cs = 15 + goto f0 + tr22: + cs = 16 + goto f6 + tr25: + cs = 16 + goto f7 + tr29: + cs = 17 + goto _again + tr26: + cs = 17 + goto f0 + tr30: + cs = 18 + goto _again + tr27: + cs = 18 + goto f0 + tr28: + cs = 19 + goto f8 + tr31: + cs = 19 + goto f9 + tr35: + cs = 20 + goto _again + tr32: + cs = 20 + goto f0 + tr36: + cs = 21 + goto _again + tr33: + cs = 21 + goto f0 + tr34: + cs = 22 + goto f10 + tr37: + cs = 22 + goto f11 + tr40: + cs = 23 + goto _again + tr38: + cs = 23 + goto f0 + tr42: + cs = 24 + goto _again + tr44: + cs = 25 + goto _again + tr43: + cs = 25 + goto f0 + tr65: + cs = 26 + goto _again + tr63: + cs = 26 + goto f20 + tr57: + cs = 27 + goto _again + tr55: + cs = 27 + goto f20 + tr49: + cs = 28 + goto _again + tr50: + cs = 29 + goto f0 + tr51: + cs = 30 + goto _again + tr39: + cs = 31 + goto f12 + tr41: + cs = 31 + goto f13 + tr45: + cs = 32 + goto f14 + tr48: + cs = 33 + goto f16 + tr53: + cs = 33 + goto f19 + tr56: + cs = 34 + goto f16 + tr54: + cs = 34 + goto f19 + tr59: + cs = 35 + goto f16 + tr58: + cs = 35 + goto f22 + tr60: + cs = 36 + goto f14 + tr47: + cs = 37 + goto f16 + tr61: + cs = 37 + goto f19 + tr64: + cs = 38 + goto f16 + tr62: + cs = 38 + goto f19 + tr67: + cs = 39 + goto f16 + tr66: + cs = 39 + goto f23 + tr52: + cs = 40 + goto f17 + + f0: +//line cef.rl:37 + + mark = p + + goto _again + f1: +//line cef.rl:40 + + e.Version, _ = strconv.Atoi(string(data[mark:p])) + + goto _again + f3: +//line cef.rl:43 + + e.DeviceVendor = string(replaceHeaderEscapes(data[mark:p])) + + goto _again + f5: +//line cef.rl:46 + + e.DeviceProduct = string(replaceHeaderEscapes(data[mark:p])) + + goto _again + f7: +//line cef.rl:49 + + e.DeviceVersion = string(replaceHeaderEscapes(data[mark:p])) + + goto _again + f9: +//line cef.rl:52 + + e.DeviceEventClassID = string(replaceHeaderEscapes(data[mark:p])) + + goto _again + f11: +//line cef.rl:55 + + e.Name = string(replaceHeaderEscapes(data[mark:p])) + + goto _again + f13: +//line cef.rl:58 + + e.Severity = string(data[mark:p]) + + goto _again + f14: +//line cef.rl:61 + + // A new extension key marks the end of the last extension value. + if len(extKey) > 0 && extValueStart <= mark-1 { + e.pushExtension(extKey, replaceExtensionEscapes(data[extValueStart:mark-1])) + extKey, extValueStart, extValueEnd = nil, 0, 0 + } + extKey = data[mark:p] + + goto _again + f20: +//line cef.rl:69 + + extValueStart = p + extValueEnd = p + + goto _again + f16: +//line cef.rl:73 + + extValueEnd = p + 1 + + goto _again + f15: +//line cef.rl:83 + + recoveredErrs = append(recoveredErrs, fmt.Errorf("malformed value for %s at pos %d", extKey, p+1)) + (p)-- + cs = 28 + + goto _again + f17: +//line cef.rl:87 + + extKey, extValueStart, extValueEnd = nil, 0, 0 + // Resume processing at p, the start of the next extension key. + p = mark + cs = 24 + + goto _again + f2: +//line cef.rl:37 + + mark = p + +//line cef.rl:43 + + e.DeviceVendor = string(replaceHeaderEscapes(data[mark:p])) + + goto _again + f4: +//line cef.rl:37 + + mark = p + +//line cef.rl:46 + + e.DeviceProduct = string(replaceHeaderEscapes(data[mark:p])) + + goto _again + f6: +//line cef.rl:37 + + mark = p + +//line cef.rl:49 + + e.DeviceVersion = string(replaceHeaderEscapes(data[mark:p])) + + goto _again + f8: +//line cef.rl:37 + + mark = p + +//line cef.rl:52 + + e.DeviceEventClassID = string(replaceHeaderEscapes(data[mark:p])) + + goto _again + f10: +//line cef.rl:37 + + mark = p + +//line cef.rl:55 + + e.Name = string(replaceHeaderEscapes(data[mark:p])) + + goto _again + f12: +//line cef.rl:37 + + mark = p + +//line cef.rl:58 + + e.Severity = string(data[mark:p]) + + goto _again + f23: +//line cef.rl:37 + + mark = p + +//line cef.rl:73 + + extValueEnd = p + 1 + + goto _again + f19: +//line cef.rl:69 + + extValueStart = p + extValueEnd = p + +//line cef.rl:73 + + extValueEnd = p + 1 + + goto _again + f22: +//line cef.rl:73 + + extValueEnd = p + 1 + +//line cef.rl:37 + + mark = p + + goto _again + + _again: + if cs == 0 { + goto _out + } + if (p)++; (p) != (pe) { + goto _resume + } + _test_eof: + { + } + if (p) == eof { + switch _cef_eof_actions[cs] { + case 22: +//line cef.rl:76 + + // Reaching the EOF marks the end of the final extension value. + if len(extKey) > 0 && extValueStart <= extValueEnd { + e.pushExtension(extKey, replaceExtensionEscapes(data[extValueStart:extValueEnd])) + extKey, extValueStart, extValueEnd = nil, 0, 0 + } + + case 16: +//line cef.rl:83 + + recoveredErrs = append(recoveredErrs, fmt.Errorf("malformed value for %s at pos %d", extKey, p+1)) + (p)-- + cs = 28 + goto _again + + case 19: +//line cef.rl:69 + + extValueStart = p + extValueEnd = p + +//line cef.rl:76 + + // Reaching the EOF marks the end of the final extension value. + if len(extKey) > 0 && extValueStart <= extValueEnd { + e.pushExtension(extKey, replaceExtensionEscapes(data[extValueStart:extValueEnd])) + extKey, extValueStart, extValueEnd = nil, 0, 0 + } + +//line parser.go:847 + } + } + + _out: + { + } + } + +//line cef.rl:145 + + // Check if state machine completed. + if cs < cef_first_final { + // Reached an early end. + if p == pe { + return multierr.Append(multierr.Combine(recoveredErrs...), fmt.Errorf("unexpected end of CEF event")) + } + + // Encountered invalid input. + return multierr.Append(multierr.Combine(recoveredErrs...), fmt.Errorf("error in CEF event at pos %d", p+1)) + } + + return multierr.Combine(recoveredErrs...) +} diff --git a/x-pack/filebeat/processors/decode_cef/config.go b/x-pack/filebeat/processors/decode_cef/config.go new file mode 100644 index 000000000000..361a66672dab --- /dev/null +++ b/x-pack/filebeat/processors/decode_cef/config.go @@ -0,0 +1,22 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package decode_cef + +type config struct { + Field string `config:"field"` // Source field containing the CEF message. + TargetField string `config:"target_field"` // Target field for the CEF object. + IgnoreMissing bool `config:"ignore_missing"` // Ignore missing source field. + IgnoreFailure bool `config:"ignore_failure"` // Ignore failures when the source field does not contain a CEF message. Parse errors do not cause failures, but are added to error.message. + ID string `config:"id"` // Instance ID for debugging purposes. + ECS bool `config:"ecs"` // Generate ECS fields. +} + +func defaultConfig() config { + return config{ + Field: "message", + TargetField: "cef", + ECS: true, + } +} diff --git a/x-pack/filebeat/processors/decode_cef/decode_cef.go b/x-pack/filebeat/processors/decode_cef/decode_cef.go new file mode 100644 index 000000000000..b7554b2efe13 --- /dev/null +++ b/x-pack/filebeat/processors/decode_cef/decode_cef.go @@ -0,0 +1,253 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package decode_cef + +import ( + "encoding/json" + "strconv" + "strings" + + "github.com/pkg/errors" + "go.uber.org/multierr" + + "github.com/elastic/beats/libbeat/beat" + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/libbeat/processors" + "github.com/elastic/beats/x-pack/filebeat/processors/decode_cef/cef" +) + +const ( + procName = "decode_cef" + logName = "processor." + procName +) + +func init() { + processors.RegisterPlugin(procName, New) +} + +type processor struct { + config + log *logp.Logger +} + +// New constructs a new processor built from ucfg config. +func New(cfg *common.Config) (processors.Processor, error) { + c := defaultConfig() + if err := cfg.Unpack(&c); err != nil { + return nil, errors.Wrap(err, "fail to unpack the "+procName+" processor configuration") + } + + return newDecodeCEF(c) +} + +func newDecodeCEF(c config) (*processor, error) { + log := logp.NewLogger(logName) + if c.ID != "" { + log = log.With("instance_id", c.ID) + } + + return &processor{config: c, log: log}, nil +} + +func (p *processor) String() string { + json, _ := json.Marshal(p.config) + return procName + "=" + string(json) +} + +func (p *processor) Run(event *beat.Event) (*beat.Event, error) { + v, err := event.GetValue(p.Field) + if err != nil { + if p.IgnoreMissing { + return event, nil + } + return event, errors.Wrapf(err, "decode_cef field [%v] not found", p.Field) + } + + cefData, ok := v.(string) + if !ok { + if p.IgnoreFailure { + return event, nil + } + return event, errors.Wrapf(err, "decode_cef field [%v] is not a string", p.Field) + } + + // Ignore any leading data before the CEF header. + idx := strings.Index(cefData, "CEF:") + if idx == -1 { + if p.IgnoreFailure { + return event, nil + } + return event, errors.Errorf("decode_cef field [%v] does not contain a CEF header", p.Field) + } + cefData = cefData[idx:] + + // If the version < 0 after parsing then none of the data is valid so return here. + var ce cef.Event + if err = ce.Unpack([]byte(cefData), cef.WithFullExtensionNames()); ce.Version < 0 && err != nil { + if p.IgnoreFailure { + return event, nil + } + return event, errors.Wrap(err, "decode_cef failed to parse message") + } + + cefErrors := multierr.Errors(err) + cefObject := toCEFObject(&ce) + event.PutValue(p.TargetField, cefObject) + + // Map CEF extension fields to ECS fields. + if p.ECS { + writeCEFHeaderToECS(&ce, event) + + for key, v := range ce.Extensions { + mapping, found := ecsExtensionMapping[key] + if !found { + continue + } + + // Apply translation function or use a standard type translation (e.g. string to long). + if mapping.Translate != nil { + translatedValue, err := mapping.Translate(v) + if err != nil { + cefErrors = append(cefErrors, errors.Wrap(err, key)) + continue + } + if translatedValue != nil { + event.PutValue(mapping.Target, translatedValue) + } + } else if mapping.Type != unset { + translatedValue, err := toType(v, mapping.Type) + if err != nil { + cefErrors = append(cefErrors, errors.Wrap(err, key)) + continue + } + event.PutValue(mapping.Target, translatedValue) + } + } + } + + // Add all parsing/conversion errors to error.message. + for _, cefError := range cefErrors { + if err := appendErrorMessage(event.Fields, cefError.Error()); err != nil { + p.log.Warn("Failed adding CEF errors to event.", "error", err) + break + } + } + + return event, nil +} + +func toCEFObject(cefEvent *cef.Event) common.MapStr { + // Add CEF header fields. + cefObject := common.MapStr{"version": strconv.Itoa(cefEvent.Version)} + if cefEvent.DeviceVendor != "" { + cefObject.Put("device.vendor", cefEvent.DeviceVendor) + } + if cefEvent.DeviceProduct != "" { + cefObject.Put("device.product", cefEvent.DeviceProduct) + } + if cefEvent.DeviceVersion != "" { + cefObject.Put("device.version", cefEvent.DeviceVersion) + } + if cefEvent.DeviceEventClassID != "" { + cefObject.Put("device.event_class_id", cefEvent.DeviceEventClassID) + } + if cefEvent.Name != "" { + cefObject.Put("name", cefEvent.Name) + } + if cefEvent.Severity != "" { + cefObject.Put("severity", cefEvent.Severity) + } + + // Add CEF extensions (key-value pairs). + if len(cefEvent.Extensions) > 0 { + extensions := make(common.MapStr, len(cefEvent.Extensions)) + cefObject.Put("extensions", extensions) + for k, v := range cefEvent.Extensions { + extensions.Put(k, v) + } + } + + return cefObject +} + +func writeCEFHeaderToECS(cefEvent *cef.Event, event *beat.Event) { + if cefEvent.DeviceVendor != "" { + event.PutValue("observer.vendor", cefEvent.DeviceVendor) + } + if cefEvent.DeviceProduct != "" { + // TODO: observer.product is not officially part of ECS. + event.PutValue("observer.product", cefEvent.DeviceProduct) + } + if cefEvent.DeviceVersion != "" { + event.PutValue("observer.version", cefEvent.DeviceVersion) + } + if cefEvent.DeviceEventClassID != "" { + event.PutValue("event.code", cefEvent.DeviceEventClassID) + } + if cefEvent.Name != "" { + event.PutValue("message", cefEvent.Name) + } + if cefEvent.Severity != "" { + if sev, ok := cefSeverityToNumber(cefEvent.Severity); ok { + event.PutValue("event.severity", sev) + } + } +} + +func appendErrorMessage(m common.MapStr, msg string) error { + const field = "error.message" + list, _ := m.GetValue(field) + + switch v := list.(type) { + case nil: + m.Put(field, msg) + case string: + if msg != v { + m.Put(field, []string{v, msg}) + } + case []string: + for _, existingTag := range v { + if msg == existingTag { + // Duplicate + return nil + } + } + m.Put(field, append(v, msg)) + case []interface{}: + for _, existingTag := range v { + if msg == existingTag { + // Duplicate + return nil + } + } + m.Put(field, append(v, msg)) + default: + return errors.Errorf("unexpected type %T found for %v field", list, field) + } + return nil +} + +// cefSeverityToNumber converts the CEF severity string to a numeric value. The +// returned boolean indicates if the conversion was successful. +func cefSeverityToNumber(severity string) (int, bool) { + // From CEF spec: + // Severity is a string or integer and reflects the importance of the event. + // The valid string values are Unknown, Low, Medium, High, and Very-High. + // The valid integer values are 0-3=Low, 4-6=Medium, 7- 8=High, and 9-10=Very-High. + switch strings.ToLower(severity) { + case "low": + return 0, true + case "medium": + return 4, true + case "high": + return 7, true + case "very-high": + return 9, true + default: + s, err := strconv.Atoi(severity) + return s, err == nil + } +} diff --git a/x-pack/filebeat/processors/decode_cef/decode_cef_test.go b/x-pack/filebeat/processors/decode_cef/decode_cef_test.go new file mode 100644 index 000000000000..df81740d6949 --- /dev/null +++ b/x-pack/filebeat/processors/decode_cef/decode_cef_test.go @@ -0,0 +1,302 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package decode_cef + +import ( + "bufio" + "encoding/json" + "flag" + "os" + "reflect" + "testing" + + "github.com/pmezard/go-difflib/difflib" + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/libbeat/beat" + "github.com/elastic/beats/libbeat/common" +) + +var updateGolden = flag.Bool("update", false, "update golden test files") + +func TestProcessorRun(t *testing.T) { + type testCase struct { + config func() config + message string + fields common.MapStr + } + + var testCases = map[string]testCase{ + "custom_target_root": { + config: func() config { + c := defaultConfig() + c.TargetField = "" + return c + }, + message: "CEF:1|Trend Micro|Deep Security Manager|1.2.3|600|User Signed In|3|src=10.52.116.160 suser=admin target=admin msg=User signed in from 2001:db8::5", + fields: common.MapStr{ + "version": "1", + "device.event_class_id": "600", + "device.product": "Deep Security Manager", + "device.vendor": "Trend Micro", + "device.version": "1.2.3", + "name": "User Signed In", + "severity": "3", + "event.severity": 3, + "extensions.message": "User signed in from 2001:db8::5", + "extensions.sourceAddress": "10.52.116.160", + "extensions.sourceUserName": "admin", + "extensions.target": "admin", + // ECS + "event.code": "600", + "message": "User signed in from 2001:db8::5", + "observer.product": "Deep Security Manager", + "observer.vendor": "Trend Micro", + "observer.version": "1.2.3", + "source.ip": "10.52.116.160", + "source.user.name": "admin", + }, + }, + "parse_errors": { + message: "CEF:0|Trend Micro|Deep Security Manager|1.2.3|600|User Signed In|Low|msg=User signed in with =xyz", + fields: common.MapStr{ + "cef.version": "0", + "cef.device.event_class_id": "600", + "cef.device.product": "Deep Security Manager", + "cef.device.vendor": "Trend Micro", + "cef.device.version": "1.2.3", + "cef.name": "User Signed In", + "cef.severity": "Low", + // ECS + "event.code": "600", + "event.severity": 0, + "observer.product": "Deep Security Manager", + "observer.vendor": "Trend Micro", + "observer.version": "1.2.3", + "message": "User Signed In", + "error.message": []string{ + "malformed value for msg at pos 94", + "unexpected end of CEF event", + }, + }, + }, + "ecs_disabled": { + config: func() config { + c := defaultConfig() + c.ECS = false + return c + }, + message: "CEF:0|Trend Micro|Deep Security Manager|1.2.3|600|User Signed In|3|src=10.52.116.160 suser=admin target=admin msg=User signed in from 2001:db8::5", + fields: common.MapStr{ + "cef.version": "0", + "cef.device.event_class_id": "600", + "cef.device.product": "Deep Security Manager", + "cef.device.vendor": "Trend Micro", + "cef.device.version": "1.2.3", + "cef.name": "User Signed In", + "cef.severity": "3", + "cef.extensions.message": "User signed in from 2001:db8::5", + "cef.extensions.sourceAddress": "10.52.116.160", + "cef.extensions.sourceUserName": "admin", + "cef.extensions.target": "admin", + "message": "CEF:0|Trend Micro|Deep Security Manager|1.2.3|600|User Signed In|3|src=10.52.116.160 suser=admin target=admin msg=User signed in from 2001:db8::5", + }, + }, + } + + dec, err := newDecodeCEF(defaultConfig()) + if err != nil { + t.Fatal(err) + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + dec := dec + if tc.config != nil { + dec, err = newDecodeCEF(tc.config()) + if err != nil { + t.Fatal(err) + } + } + + evt := &beat.Event{ + Fields: common.MapStr{ + "message": tc.message, + }, + } + + evt, err = dec.Run(evt) + if err != nil { + t.Fatal(err) + } + + assertEqual(t, tc.fields, evt.Fields.Flatten()) + }) + } + + t.Run("not_cef", func(t *testing.T) { + evt := &beat.Event{ + Fields: common.MapStr{ + "message": "hello world!", + }, + } + + evt, err = dec.Run(evt) + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "does not contain a CEF header") + } + }) + + t.Run("leading_garbage", func(t *testing.T) { + tc := testCases["custom_target_root"] + + evt := &beat.Event{ + Fields: common.MapStr{ + "message": "leading garbage" + tc.message, + }, + } + + evt, err = dec.Run(evt) + if err != nil { + t.Fatal(err) + } + + version, _ := evt.GetValue("cef.version") + assert.EqualValues(t, "1", version) + }) +} + +func TestGolden(t *testing.T) { + const source = "testdata/samples.log" + + events := readCEFSamples(t, source) + + if *updateGolden { + writeGoldenJSON(t, source, events) + return + } + + expected := readGoldenJSON(t, source) + if !assert.Len(t, events, len(expected)) { + return + } + for i, e := range events { + assertEqual(t, expected[i], normalize(t, e)) + } +} + +func readCEFSamples(t testing.TB, source string) []common.MapStr { + f, err := os.Open(source) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + conf := defaultConfig() + conf.Field = "event.original" + dec, err := newDecodeCEF(conf) + if err != nil { + t.Fatal(err) + } + + var samples []common.MapStr + s := bufio.NewScanner(f) + for s.Scan() { + data := s.Bytes() + if len(data) == 0 || data[0] == '#' { + continue + } + + evt := &beat.Event{ + Fields: common.MapStr{ + "event": common.MapStr{"original": string(data)}, + }, + } + + evt, err := dec.Run(evt) + if err != nil { + t.Fatalf("Error reading from %v: %v", source, err) + } + + samples = append(samples, evt.Fields) + } + if err = s.Err(); err != nil { + t.Fatal(err) + } + + return samples +} + +func readGoldenJSON(t testing.TB, source string) []common.MapStr { + source = source + ".golden.json" + + f, err := os.Open(source) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + dec := json.NewDecoder(bufio.NewReader(f)) + + var events []common.MapStr + if err = dec.Decode(&events); err != nil { + t.Fatal(err) + } + + return events +} + +func writeGoldenJSON(t testing.TB, source string, events []common.MapStr) { + dest := source + ".golden.json" + + f, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + enc := json.NewEncoder(f) + enc.SetIndent("", " ") + if err = enc.Encode(events); err != nil { + t.Fatal(err) + } +} + +func normalize(t testing.TB, m common.MapStr) common.MapStr { + data, err := json.Marshal(m) + if err != nil { + t.Fatal(err) + } + + var out common.MapStr + if err = json.Unmarshal(data, &out); err != nil { + t.Fatal(err) + } + + return out +} + +// assertEqual asserts that the two objects are deeply equal. If not it will +// error the test and output a diff of the two objects' JSON representation. +func assertEqual(t testing.TB, expected, actual interface{}) bool { + t.Helper() + + if reflect.DeepEqual(expected, actual) { + return true + } + + expJSON, _ := json.MarshalIndent(expected, "", " ") + actJSON, _ := json.MarshalIndent(actual, "", " ") + + diff, _ := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{ + A: difflib.SplitLines(string(expJSON)), + B: difflib.SplitLines(string(actJSON)), + FromFile: "Expected", + ToFile: "Actual", + Context: 1, + }) + t.Errorf("Expected and actual are different:\n%s", diff) + return false +} diff --git a/x-pack/filebeat/processors/decode_cef/fields.go b/x-pack/filebeat/processors/decode_cef/fields.go new file mode 100644 index 000000000000..07b5cb657851 --- /dev/null +++ b/x-pack/filebeat/processors/decode_cef/fields.go @@ -0,0 +1,23 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. + +package decode_cef + +import ( + "github.com/elastic/beats/libbeat/asset" +) + +func init() { + if err := asset.SetFields("filebeat", "decode_cef", asset.ModuleFieldsPri, AssetDecodeCef); err != nil { + panic(err) + } +} + +// AssetDecodeCef returns asset data. +// This is the base64 encoded gzipped contents of processors/decode_cef. +func AssetDecodeCef() string { + return "eJy0lU9v2zwMxu/5FDy+LxAHHTbsT4DssCzBCmzAgK69pqr0OOFiS55EO/W3H2THjttkQNBsOgQIRT38kQTphLaop6SRjoiEJcOUPkM7A5ovllR4pxGC85QyMhNGRAZBey6EnZ3SxxER0dzlubO0qGCFls7nSui/+WL5PxklajKi/etp452QVTm6mPFIXWBKa+/KYm85ESSeTzUZpKrMhGQDujcN6UojvR+g7jwLAqksa+JT6l3e+M8Xy14qRwhqDRJHsuFA942Ie/gJLRO6FtLOimIbupe0gTLoCkHKmnjT6+FRYAM72+cczzDvYe4VfPTt7V0Ntqh3zpuB/Q+ViOeuFSGX9oyhgOaUtYr+VAYYeqib232+k9ERi0HFGpMK1jh/KVHU6IBaYZKNktgdU2qY81hab7kM5nsrcjnNv2jWPsOX4CCO2UpnKoQVm8uobi3/KkFsYIVTRt+7JkijeAIkoIJnqc+IjUeVF3Gp3MHXyRdeb84Du84L50VZjSdEE/qxAVUqY0NBPNt1/FPGafegW7u1bmfH9NXtxk/kvsFwmY8pAoyb2e15hpJsBWv4oeZV8np2JPcmeTvrJN8l9H520P2QvLqaHcSPixd/L2vazcZ5GTo8LdFxyH43haPA7cIbmFvD6gVYc5dl0B3PFnXSlJEKxT6QVt4zYon7ZXVYmc2a3IN30O4hwFfwz5bBKbAhVI/UjX9UeyYdXOk1JlE/DtSgI2er37Rv2/nl9iPRyp5qxmGCg7BtlvNfjy7KryHPov8OAAD//4g/UXk=" +} diff --git a/x-pack/filebeat/processors/decode_cef/keys.ecs.go b/x-pack/filebeat/processors/decode_cef/keys.ecs.go new file mode 100644 index 000000000000..222918170da6 --- /dev/null +++ b/x-pack/filebeat/processors/decode_cef/keys.ecs.go @@ -0,0 +1,447 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package decode_cef + +import ( + "net" + "strconv" + "strings" + "time" + + "github.com/pkg/errors" +) + +type dataType uint8 + +// List of dataTypes. +const ( + unset dataType = iota + Integer + Long + Float + Double + String + Boolean + IP + Timestamp +) + +type ecsMode uint8 + +// List of modes. +const ( + copyMode ecsMode = iota + renameMode +) + +type mappedField struct { + Target string + Type dataType + Translate func(in string) (interface{}, error) +} + +var ecsExtensionMapping = map[string]mappedField{ + "agentAddress": { + Target: "agent.ip", + Type: IP, + }, + "agentDnsDomain": { + Target: "agent.name", + Type: String, + }, + "agentHostName": { + Target: "agent.name", + Type: String, + }, + "agentId": { + Target: "agent.id", + Type: String, + }, + "agentMacAddress": { + Target: "agent.mac", + Type: String, + }, + "agentReceiptTime": { + Target: "event.created", + Type: Timestamp, + }, + "agentType": { + Target: "agent.type", + Type: String, + }, + "agentVersion": { + Target: "agent.version", + Type: String, + }, + "applicationProtocol": { + Target: "network.application", + Type: String, + }, + "bytesIn": { + Target: "source.bytes", + Type: Integer, + }, + "bytesOut": { + Target: "destination.bytes", + Type: Integer, + }, + "customerExternalID": { + Target: "organization.id", + Type: String, + }, + "customerURI": { + Target: "organization.name", + Type: String, + }, + "destinationAddress": { + Target: "destination.ip", + Type: IP, + }, + "destinationDnsDomain": { + Target: "destination.domain", + Type: String, + }, + "destinationGeoLatitude": { + Target: "destination.geo.location.lat", + Type: Double, + }, + "destinationGeoLongitude": { + Target: "destination.geo.location.lon", + Type: Double, + }, + "destinationHostName": { + Target: "destination.domain", + Type: String, + }, + "destinationMacAddress": { + Target: "destination.mac", + Type: String, + }, + "destinationPort": { + Target: "destination.port", + Type: Integer, + }, + "destinationProcessId": { + Target: "destination.process.pid", + Type: Integer, + }, + "destinationProcessName": { + Target: "destination.process.name", + Type: String, + }, + "destinationServiceName": { + Target: "destination.service.name", + Type: String, + }, + "destinationTranslatedAddress": { + Target: "destination.nat.ip", + Type: IP, + }, + "destinationTranslatedPort": { + Target: "destination.nat.port", + Type: Integer, + }, + "destinationUserId": { + Target: "destination.user.id", + Type: String, + }, + "destinationUserName": { + Target: "destination.user.name", + Type: String, + }, + "destinationUserPrivileges": { + Target: "destination.user.group", + Type: String, + }, + "deviceAction": { + Target: "event.action", + Type: String, + }, + "deviceAddress": { + Target: "observer.ip", + Type: IP, + }, + "deviceDirection": { + Target: "network.direction", + Translate: func(in string) (interface{}, error) { + switch in { + case "0": + return "inbound", nil + case "1": + return "outbound", nil + default: + return nil, errors.Errorf("deviceDirection must be 0 or 1") + } + }, + }, + "deviceDnsDomain": { + Target: "observer.hostname", + Type: String, + }, + "deviceHostName": { + Target: "observer.hostname", + Type: String, + }, + "deviceMacAddress": { + Target: "observer.mac", + Type: String, + }, + "devicePayloadId": { + Target: "event.id", + Type: String, + }, + "deviceProcessId": { + Target: "process.pid", + Type: Integer, + }, + "deviceProcessName": { + Target: "process.name", + Type: String, + }, + "deviceReceiptTime": { + Target: "@timestamp", + Type: Timestamp, + }, + "deviceTimeZone": { + Target: "event.timezone", + Type: String, + }, + "endTime": { + Target: "event.end", + Type: Timestamp, + }, + "eventId": { + Target: "event.id", + Type: Long, + }, + "eventOutcome": { + Target: "event.outcome", + Type: String, + }, + "fileCreateTime": { + Target: "file.created", + Type: Timestamp, + }, + "fileId": { + Target: "file.inode", + Type: String, + }, + "fileModificationTime": { + Target: "file.mtime", + Type: Timestamp, + }, + "filename": { + Target: "file.name", + Type: String, + }, + "filePath": { + Target: "file.path", + Type: String, + }, + "filePermission": { + Target: "file.group", + Type: String, + }, + "fileSize": { + Target: "file.size", + Type: Integer, + }, + "fileType": { + Target: "file.type", + Type: String, + }, + "message": { + Target: "message", + Type: String, + }, + "requestClientApplication": { + Target: "user_agent.original", + Type: String, + }, + "requestContext": { + Target: "http.request.referrer", + Translate: func(in string) (interface{}, error) { + // Does the string look like URL? + if strings.HasPrefix(in, "http") { + return in, nil + } + return nil, nil + }, + }, + "requestMethod": { + Target: "http.request.method", + Type: String, + }, + "requestUrl": { + Target: "url.original", + Type: String, + }, + "sourceAddress": { + Target: "source.ip", + Type: IP, + }, + "sourceDnsDomain": { + Target: "source.domain", + Type: String, + }, + "sourceGeoLatitude": { + Target: "source.geo.location.lat", + Type: Double, + }, + "sourceGeoLongitude": { + Target: "source.geo.location.lon", + Type: Double, + }, + "sourceHostName": { + Target: "source.domain", + Type: String, + }, + "sourceMacAddress": { + Target: "source.mac", + Type: String, + }, + "sourcePort": { + Target: "source.port", + Type: Integer, + }, + "sourceProcessId": { + Target: "source.process.pid", + Type: Integer, + }, + "sourceProcessName": { + Target: "source.process.name", + Type: String, + }, + "sourceServiceName": { + Target: "source.service.name", + Type: String, + }, + "sourceTranslatedAddress": { + Target: "source.nat.ip", + Type: IP, + }, + "sourceTranslatedPort": { + Target: "source.nat.port", + Type: Integer, + }, + "sourceUserId": { + Target: "source.user.id", + Type: String, + }, + "sourceUserName": { + Target: "source.user.name", + Type: String, + }, + "sourceUserPrivileges": { + Target: "source.user.group", + Type: String, + }, + "startTime": { + Target: "event.start", + Type: Timestamp, + }, + "transportProtocol": { + Target: "network.transport", + Translate: func(in string) (interface{}, error) { + return strings.ToLower(in), nil + }, + }, + "type": { + Target: "event.kind", + Type: Integer, + }, +} + +func toType(value string, typ dataType) (interface{}, error) { + switch typ { + case String: + return value, nil + case Long: + return toLong(value) + case Integer: + return toInteger(value) + case Float: + return toFloat(value) + case Double: + return toDouble(value) + case Boolean: + return toBoolean(value) + case IP: + return toIP(value) + default: + panic("invalid data type") + } +} + +func toLong(v string) (int64, error) { + return strconv.ParseInt(v, 0, 64) +} + +func toInteger(v string) (int32, error) { + i, err := strconv.ParseInt(v, 0, 32) + return int32(i), err +} + +func toFloat(v string) (float32, error) { + f, err := strconv.ParseFloat(v, 32) + return float32(f), err +} + +func toDouble(v string) (float64, error) { + f, err := strconv.ParseFloat(v, 64) + return f, err +} + +func toBoolean(v string) (bool, error) { + return strconv.ParseBool(v) +} + +func toIP(v string) (string, error) { + // This is validating that the value is an IP. + if net.ParseIP(v) != nil { + return v, nil + } + return "", errors.New("value is not a valid IP address") +} + +var timeLayouts = []string{ + // MMM dd HH:mm:ss.SSS zzz + "Jan _2 15:04:05.000 MST", + // MMM dd HH:mm:sss.SSS + "Jan _2 15:04:05.000", + // MMM dd HH:mm:ss zzz + "Jan _2 15:04:05 MST", + // MMM dd HH:mm:ss + "Jan _2 15:04:05", + // MMM dd yyyy HH:mm:ss.SSS zzz + "Jan _2 2006 15:04:05.000 MST", + // MMM dd yyyy HH:mm:ss.SSS + "Jan _2 2006 15:04:05.000", + // MMM dd yyyy HH:mm:ss zzz + "Jan _2 2006 15:04:05 MST", + // MMM dd yyyy HH:mm:ss + "Jan _2 2006 15:04:05", +} + +func toTimestamp(v string) (time.Time, error) { + if unixMs, err := toLong(v); err == nil { + return time.Unix(0, unixMs*int64(time.Millisecond)), nil + } + + for _, layout := range timeLayouts { + ts, err := time.ParseInLocation(layout, v, time.UTC) + if err == nil { + // Use current year if no year is zero. + if ts.Year() == 0 { + currentYear := time.Now().In(ts.Location()).Year() + ts = ts.AddDate(currentYear, 0, 0) + } + + return ts, nil + } + } + + return time.Time{}, errors.New("value is not a valid timestamp") +} diff --git a/x-pack/filebeat/processors/decode_cef/testdata/samples.log b/x-pack/filebeat/processors/decode_cef/testdata/samples.log new file mode 100644 index 000000000000..821ea30d6bbe --- /dev/null +++ b/x-pack/filebeat/processors/decode_cef/testdata/samples.log @@ -0,0 +1,15 @@ +CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 additional.dotfieldName=new_value ad.Authentification=MICROSOFT_AUTHENTICATION_PACKAGE_V1_0 ad.Error_,Code=3221225578 dst=12.121.122.82 ad.field[0]=field0 ad.foo.name[1]=new_name +CEF:0|Trend Micro|Deep Security Manager|1.2.3|600|User Signed In|3|src=10.52.116.160 suser=admin target=admin msg=User signed in from 2001:db8::5 +CEF:0|Trend Micro|Deep Security Agent|1.2.3|4000000|Eicar_test_file|6|cn1=1 cn1Label=Host ID dvchost=hostname cn2=205 cn2Label=Quarantine File Size cs6=ContainerImageName | ContainerName | ContainerID cs6Label=Container filePath=C:\\Users\\trend\\Desktop\\eicar.exe act=Delete msg=Realtime TrendMicroDsMalwareTarget=N/A TrendMicroDsMalwareTargetType=N/A TrendMicroDsFileMD5=44D88612FEA8A8F36DE82E1278ABB02F TrendMicroDsFileSHA1=3395856CE81F2B7382DEE72602F798B642F14140 TrendMicroDsFileSHA256=275A021BBFB6489E54D471899F7DB9D1663FC695EC2FE2A2C4538AABF651FD0F TrendMicroDsDetectionConfidence=95 TrendMicroDsRelevantDetectionNames=Ransom_CERBER.BZC;Ransom_CERBER.C;Ransom_CRYPNISCA.SM +CEF:0|Trend Micro|Deep Security Agent|10.2.229|6001200|AppControl detectOnly|6|cn1=202 cn1Label=Host ID dvc=192.168.33.128 TrendMicroDsTenant=Primary TrendMicroDsTenantId=0 fileHash=80D4AC182F97D2AB48EE4310AC51DA5974167C596D133D64A83107B9069745E0 suser=root suid=0 act=detectOnly filePath=/home/user1/Desktop/Directory1//heartbeatSync.sh fsize=20 aggregationType=0 repeatCount=1 cs1=notWhitelisted cs1Label=actionReason cs2=0CC9713BA896193A527213D9C94892D41797EB7C cs2Label=sha1 cs3=7EA8EF10BEB2E9876D4D7F7E5A46CF8D cs3Label=md5 +CEF:0|Trend Micro|Deep Security Agent|1.2.3|20|Log for TCP Port 80|0|cn1=1 cn1Label=Host ID dvc=hostname act=Log dmac=00:50:56:F5:7F:47 smac=00:0C:29:EB:35:DE TrendMicroDsFrameType=IP src=192.168.126.150 dst=72.14.204.147 out=1019 cs3=DF MF cs3Label=Fragmentation Bits proto=TCP spt=49617 dpt=80 cs2=0x00 ACK PSH cs2Label=TCP Flags cnt=1 TrendMicroDsPacketData=AFB +CEF:0|Trend Micro|Deep Security Agent|1.2.3|30|New Integrity Monitoring Rule|6|cn1=1 cn1Label=Host ID dvchost=hostname act=updated filePath=c:\\windows\\message.dll suser=admin msg=lastModified,sha1,size +CEF:0|Trend Micro|Deep Security Agent|1.2.3|1001111|Test Intrusion Prevention Rule|3|cn1=1 cn1Label=Host ID dvchost=hostname dmac=00:50:56:F5:7F:47 smac=00:0C:29:EB:35:DE TrendMicroDsFrameType=IP src=192.168.126.150 dst=72.14.204.105 out=1093 cs3=DF MF cs3Label=Fragmentation Bits proto=TCP spt=49786 dpt=80 cs2=0x00 ACK PSH cs2Label=TCP Flags cnt=1 act=IDS:Reset cn3=10 cn3Label=Intrusion Prevention Packet Position cs5=10 cs5Label=Intrusion Prevention Stream Position cs6=8 cs6Label=Intrusion Prevention Flags TrendMicroDsPacketData=R0VUIC9zP3 +CEF:0|Trend Micro|Deep Security Agent|1.2.3|3002795|Microsoft Windows Events|8|cn1=1 cn1Label=Host ID dvchost=hostname cs1Label=LI Description cs1=Multiple Windows Logon Failures fname=Security src=127.0.0.1 duser=(no user) shost=WIN-RM6HM42G65V msg=WinEvtLog Security: AUDIT_FAILURE(4625): Microsoft-Windows-Security-Auditing: (no user): no domain: WIN-RM6HM42G65V: An account failed to log on. Subject: .. +CEF:0|Trend Micro|Deep Security Agent|1.2.3|5000000|WebReputation|5|cn1=1 cn1Label=Host ID dvchost=hostname request=example.com msg=Blocked By Admin +CEF:0|Citrix|NetScaler|NS10.0|APPFW|APPFW_STARTURL|6|src=10.217.253.78 spt=53743 method=GET request=http://vpx247.example.net/FFC/login.html msg=Disallow Illegal URL. cn1=233 cn2=205 cs1=profile1 cs2=PPE0 cs3=AjSZM26h2M+xL809pON6C8joebUA000 cs4=ALERT cs5=2012 act=blocked +CEF:0|Citrix|NetScaler|NS10.0|APPFW|APPFW_STARTURL|6|src=10.217.253.78 spt=54711 method=GET request=http://vpx247.example.net/FFC/login_post.html?abc\=def msg=Disallow Illegal URL. cn1=465 cn2=535 cs1=profile1 cs2=PPE0 cs3=IliG4Dxp1SjOhKVRDVBXmqvAaIcA000 cs4=ALERT cs5=2012 act=not blocked +CEF:0|Citrix|NetScaler|NS10.0|APPFW|APPFW_SAFECOMMERCE_XFORM|6|src=10.217.253.78 spt=56116 method=GET request=http://vpx247.example.net/FFC/CreditCardMind.html msg= Transformed (xout) potential credit card numbers seen in server response cn1=652 cn2=610 cs1=pr_ffc cs2=PPE0 cs3=li8MdGfW49uG8tGdSV85ech41a0A000 cs4=ALERT cs5=2012 act=transformed +CEF:0|Citrix|NetScaler|NS10.0|APPFW|APPFW_SAFECOMMERCE|6|src=10.217.253.78 spt=56116 method=GET request=http://vpx247.example.net/FFC/CreditCardMind.html msg= Maximum no. of potential credit card numbers seen cn1=653 cn2=610 cs1=pr_ffc cs2=PPE0 cs3=li8MdGfW49uG8tGdSV85ech41a0A000 cs4=ALERT cs5=2012 act=transformed +CEF:0|Citrix|NetScaler|NS10.0|APPFW|APPFW_SIGNATURE_MATCH|6|src=10.217.253.78 spt=56687 method=GET request=http://vpx247.example.net/FFC/wwwboard/passwd.txt msg= Signature violation rule ID 807: web-cgi /wwwboard/passwd.txt access cn1=224 cn2=205 cs1=pr_ffc cs2=PPE0 cs3=POousP7CIMW5nwZ5Rs4nq5DND0sA000 cs4=ALERT cs5=2012 act=not blocked +CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Something awesome happened|very-high|eventId=3457 requestMethod=POST dlat=38.915 dlong=-77.511 proto=TCP rawEvent={"x": "y"} sourceServiceName=httpd destinationServiceName=chrome requestContext=application/json diff --git a/x-pack/filebeat/processors/decode_cef/testdata/samples.log.golden.json b/x-pack/filebeat/processors/decode_cef/testdata/samples.log.golden.json new file mode 100644 index 000000000000..293db9b5671c --- /dev/null +++ b/x-pack/filebeat/processors/decode_cef/testdata/samples.log.golden.json @@ -0,0 +1,740 @@ +[ + { + "cef": { + "device": { + "event_class_id": "100", + "product": "threatmanager", + "vendor": "security", + "version": "1.0" + }, + "extensions": { + "ad": { + "Authentification": "MICROSOFT_AUTHENTICATION_PACKAGE_V1_0", + "Error_,Code": "3221225578", + "field[0]": "field0", + "foo": { + "name[1]": "new_name" + } + }, + "additional": { + "dotfieldName": "new_value" + }, + "destinationAddress": "12.121.122.82", + "sourceAddress": "10.0.0.192" + }, + "name": "trojan successfully stopped", + "severity": "10", + "version": "0" + }, + "destination": { + "ip": "12.121.122.82" + }, + "event": { + "code": "100", + "original": "CEF:0|security|threatmanager|1.0|100|trojan successfully stopped|10|src=10.0.0.192 additional.dotfieldName=new_value ad.Authentification=MICROSOFT_AUTHENTICATION_PACKAGE_V1_0 ad.Error_,Code=3221225578 dst=12.121.122.82 ad.field[0]=field0 ad.foo.name[1]=new_name", + "severity": 10 + }, + "message": "trojan successfully stopped", + "observer": { + "product": "threatmanager", + "vendor": "security", + "version": "1.0" + }, + "source": { + "ip": "10.0.0.192" + } + }, + { + "cef": { + "device": { + "event_class_id": "600", + "product": "Deep Security Manager", + "vendor": "Trend Micro", + "version": "1.2.3" + }, + "extensions": { + "message": "User signed in from 2001:db8::5", + "sourceAddress": "10.52.116.160", + "sourceUserName": "admin", + "target": "admin" + }, + "name": "User Signed In", + "severity": "3", + "version": "0" + }, + "event": { + "code": "600", + "original": "CEF:0|Trend Micro|Deep Security Manager|1.2.3|600|User Signed In|3|src=10.52.116.160 suser=admin target=admin msg=User signed in from 2001:db8::5", + "severity": 3 + }, + "message": "User signed in from 2001:db8::5", + "observer": { + "product": "Deep Security Manager", + "vendor": "Trend Micro", + "version": "1.2.3" + }, + "source": { + "ip": "10.52.116.160", + "user": { + "name": "admin" + } + } + }, + { + "cef": { + "device": { + "event_class_id": "4000000", + "product": "Deep Security Agent", + "vendor": "Trend Micro", + "version": "1.2.3" + }, + "extensions": { + "DeviceCustomNumber2": "205", + "TrendMicroDsDetectionConfidence": "95", + "TrendMicroDsFileMD5": "44D88612FEA8A8F36DE82E1278ABB02F", + "TrendMicroDsFileSHA1": "3395856CE81F2B7382DEE72602F798B642F14140", + "TrendMicroDsFileSHA256": "275A021BBFB6489E54D471899F7DB9D1663FC695EC2FE2A2C4538AABF651FD0F", + "TrendMicroDsMalwareTarget": "N/A", + "TrendMicroDsMalwareTargetType": "N/A", + "TrendMicroDsRelevantDetectionNames": "Ransom_CERBER.BZC;Ransom_CERBER.C;Ransom_CRYPNISCA.SM", + "deviceAction": "Delete", + "deviceCustomNumber1": "1", + "deviceCustomNumber1Label": "Host ID", + "deviceCustomNumber2Label": "Quarantine File Size", + "deviceCustomString6": "ContainerImageName | ContainerName | ContainerID", + "deviceCustomString6Label": "Container", + "deviceHostName": "hostname", + "filePath": "C:\\Users\\trend\\Desktop\\eicar.exe", + "message": "Realtime" + }, + "name": "Eicar_test_file", + "severity": "6", + "version": "0" + }, + "event": { + "action": "Delete", + "code": "4000000", + "original": "CEF:0|Trend Micro|Deep Security Agent|1.2.3|4000000|Eicar_test_file|6|cn1=1 cn1Label=Host ID dvchost=hostname cn2=205 cn2Label=Quarantine File Size cs6=ContainerImageName | ContainerName | ContainerID cs6Label=Container filePath=C:\\\\Users\\\\trend\\\\Desktop\\\\eicar.exe act=Delete msg=Realtime TrendMicroDsMalwareTarget=N/A TrendMicroDsMalwareTargetType=N/A TrendMicroDsFileMD5=44D88612FEA8A8F36DE82E1278ABB02F TrendMicroDsFileSHA1=3395856CE81F2B7382DEE72602F798B642F14140 TrendMicroDsFileSHA256=275A021BBFB6489E54D471899F7DB9D1663FC695EC2FE2A2C4538AABF651FD0F TrendMicroDsDetectionConfidence=95 TrendMicroDsRelevantDetectionNames=Ransom_CERBER.BZC;Ransom_CERBER.C;Ransom_CRYPNISCA.SM", + "severity": 6 + }, + "file": { + "path": "C:\\Users\\trend\\Desktop\\eicar.exe" + }, + "message": "Realtime", + "observer": { + "hostname": "hostname", + "product": "Deep Security Agent", + "vendor": "Trend Micro", + "version": "1.2.3" + } + }, + { + "cef": { + "device": { + "event_class_id": "6001200", + "product": "Deep Security Agent", + "vendor": "Trend Micro", + "version": "10.2.229" + }, + "extensions": { + "TrendMicroDsTenant": "Primary", + "TrendMicroDsTenantId": "0", + "aggregationType": "0", + "deviceAction": "detectOnly", + "deviceAddress": "192.168.33.128", + "deviceCustomNumber1": "202", + "deviceCustomNumber1Label": "Host ID", + "deviceCustomString1": "notWhitelisted", + "deviceCustomString1Label": "actionReason", + "deviceCustomString2": "0CC9713BA896193A527213D9C94892D41797EB7C", + "deviceCustomString2Label": "sha1", + "deviceCustomString3": "7EA8EF10BEB2E9876D4D7F7E5A46CF8D", + "deviceCustomString3Label": "md5", + "fileHash": "80D4AC182F97D2AB48EE4310AC51DA5974167C596D133D64A83107B9069745E0", + "filePath": "/home/user1/Desktop/Directory1//heartbeatSync.sh", + "fileSize": "20", + "repeatCount": "1", + "sourceUserId": "0", + "sourceUserName": "root" + }, + "name": "AppControl detectOnly", + "severity": "6", + "version": "0" + }, + "event": { + "action": "detectOnly", + "code": "6001200", + "original": "CEF:0|Trend Micro|Deep Security Agent|10.2.229|6001200|AppControl detectOnly|6|cn1=202 cn1Label=Host ID dvc=192.168.33.128 TrendMicroDsTenant=Primary TrendMicroDsTenantId=0 fileHash=80D4AC182F97D2AB48EE4310AC51DA5974167C596D133D64A83107B9069745E0 suser=root suid=0 act=detectOnly filePath=/home/user1/Desktop/Directory1//heartbeatSync.sh fsize=20 aggregationType=0 repeatCount=1 cs1=notWhitelisted cs1Label=actionReason cs2=0CC9713BA896193A527213D9C94892D41797EB7C cs2Label=sha1 cs3=7EA8EF10BEB2E9876D4D7F7E5A46CF8D cs3Label=md5", + "severity": 6 + }, + "file": { + "path": "/home/user1/Desktop/Directory1//heartbeatSync.sh", + "size": 20 + }, + "message": "AppControl detectOnly", + "observer": { + "ip": "192.168.33.128", + "product": "Deep Security Agent", + "vendor": "Trend Micro", + "version": "10.2.229" + }, + "source": { + "user": { + "id": "0", + "name": "root" + } + } + }, + { + "cef": { + "device": { + "event_class_id": "20", + "product": "Deep Security Agent", + "vendor": "Trend Micro", + "version": "1.2.3" + }, + "extensions": { + "TrendMicroDsFrameType": "IP", + "TrendMicroDsPacketData": "AFB", + "baseEventCount": "1", + "bytesOut": "1019", + "destinationAddress": "72.14.204.147", + "destinationMacAddress": "00:50:56:F5:7F:47", + "destinationPort": "80", + "deviceAction": "Log", + "deviceAddress": "hostname", + "deviceCustomNumber1": "1", + "deviceCustomNumber1Label": "Host ID", + "deviceCustomString2": "0x00 ACK PSH", + "deviceCustomString2Label": "TCP Flags", + "deviceCustomString3": "DF MF", + "deviceCustomString3Label": "Fragmentation Bits", + "sourceAddress": "192.168.126.150", + "sourceMacAddress": "00:0C:29:EB:35:DE", + "sourcePort": "49617", + "transportProtocol": "TCP" + }, + "name": "Log for TCP Port 80", + "severity": "0", + "version": "0" + }, + "destination": { + "bytes": 1019, + "ip": "72.14.204.147", + "mac": "00:50:56:F5:7F:47", + "port": 80 + }, + "error": { + "message": "deviceAddress: value is not a valid IP address" + }, + "event": { + "action": "Log", + "code": "20", + "original": "CEF:0|Trend Micro|Deep Security Agent|1.2.3|20|Log for TCP Port 80|0|cn1=1 cn1Label=Host ID dvc=hostname act=Log dmac=00:50:56:F5:7F:47 smac=00:0C:29:EB:35:DE TrendMicroDsFrameType=IP src=192.168.126.150 dst=72.14.204.147 out=1019 cs3=DF MF cs3Label=Fragmentation Bits proto=TCP spt=49617 dpt=80 cs2=0x00 ACK PSH cs2Label=TCP Flags cnt=1 TrendMicroDsPacketData=AFB", + "severity": 0 + }, + "message": "Log for TCP Port 80", + "network": { + "transport": "tcp" + }, + "observer": { + "product": "Deep Security Agent", + "vendor": "Trend Micro", + "version": "1.2.3" + }, + "source": { + "ip": "192.168.126.150", + "mac": "00:0C:29:EB:35:DE", + "port": 49617 + } + }, + { + "cef": { + "device": { + "event_class_id": "30", + "product": "Deep Security Agent", + "vendor": "Trend Micro", + "version": "1.2.3" + }, + "extensions": { + "deviceAction": "updated", + "deviceCustomNumber1": "1", + "deviceCustomNumber1Label": "Host ID", + "deviceHostName": "hostname", + "filePath": "c:\\windows\\message.dll", + "message": "lastModified,sha1,size", + "sourceUserName": "admin" + }, + "name": "New Integrity Monitoring Rule", + "severity": "6", + "version": "0" + }, + "event": { + "action": "updated", + "code": "30", + "original": "CEF:0|Trend Micro|Deep Security Agent|1.2.3|30|New Integrity Monitoring Rule|6|cn1=1 cn1Label=Host ID dvchost=hostname act=updated filePath=c:\\\\windows\\\\message.dll suser=admin msg=lastModified,sha1,size", + "severity": 6 + }, + "file": { + "path": "c:\\windows\\message.dll" + }, + "message": "lastModified,sha1,size", + "observer": { + "hostname": "hostname", + "product": "Deep Security Agent", + "vendor": "Trend Micro", + "version": "1.2.3" + }, + "source": { + "user": { + "name": "admin" + } + } + }, + { + "cef": { + "device": { + "event_class_id": "1001111", + "product": "Deep Security Agent", + "vendor": "Trend Micro", + "version": "1.2.3" + }, + "extensions": { + "TrendMicroDsFrameType": "IP", + "TrendMicroDsPacketData": "R0VUIC9zP3", + "baseEventCount": "1", + "bytesOut": "1093", + "destinationAddress": "72.14.204.105", + "destinationMacAddress": "00:50:56:F5:7F:47", + "destinationPort": "80", + "deviceAction": "IDS:Reset", + "deviceCustomNumber1": "1", + "deviceCustomNumber1Label": "Host ID", + "deviceCustomNumber3": "10", + "deviceCustomNumber3Label": "Intrusion Prevention Packet Position", + "deviceCustomString2": "0x00 ACK PSH", + "deviceCustomString2Label": "TCP Flags", + "deviceCustomString3": "DF MF", + "deviceCustomString3Label": "Fragmentation Bits", + "deviceCustomString5": "10", + "deviceCustomString5Label": "Intrusion Prevention Stream Position", + "deviceCustomString6": "8", + "deviceCustomString6Label": "Intrusion Prevention Flags", + "deviceHostName": "hostname", + "sourceAddress": "192.168.126.150", + "sourceMacAddress": "00:0C:29:EB:35:DE", + "sourcePort": "49786", + "transportProtocol": "TCP" + }, + "name": "Test Intrusion Prevention Rule", + "severity": "3", + "version": "0" + }, + "destination": { + "bytes": 1093, + "ip": "72.14.204.105", + "mac": "00:50:56:F5:7F:47", + "port": 80 + }, + "event": { + "action": "IDS:Reset", + "code": "1001111", + "original": "CEF:0|Trend Micro|Deep Security Agent|1.2.3|1001111|Test Intrusion Prevention Rule|3|cn1=1 cn1Label=Host ID dvchost=hostname dmac=00:50:56:F5:7F:47 smac=00:0C:29:EB:35:DE TrendMicroDsFrameType=IP src=192.168.126.150 dst=72.14.204.105 out=1093 cs3=DF MF cs3Label=Fragmentation Bits proto=TCP spt=49786 dpt=80 cs2=0x00 ACK PSH cs2Label=TCP Flags cnt=1 act=IDS:Reset cn3=10 cn3Label=Intrusion Prevention Packet Position cs5=10 cs5Label=Intrusion Prevention Stream Position cs6=8 cs6Label=Intrusion Prevention Flags TrendMicroDsPacketData=R0VUIC9zP3", + "severity": 3 + }, + "message": "Test Intrusion Prevention Rule", + "network": { + "transport": "tcp" + }, + "observer": { + "hostname": "hostname", + "product": "Deep Security Agent", + "vendor": "Trend Micro", + "version": "1.2.3" + }, + "source": { + "ip": "192.168.126.150", + "mac": "00:0C:29:EB:35:DE", + "port": 49786 + } + }, + { + "cef": { + "device": { + "event_class_id": "3002795", + "product": "Deep Security Agent", + "vendor": "Trend Micro", + "version": "1.2.3" + }, + "extensions": { + "destinationUserName": "(no user)", + "deviceCustomNumber1": "1", + "deviceCustomNumber1Label": "Host ID", + "deviceCustomString1": "Multiple Windows Logon Failures", + "deviceCustomString1Label": "LI Description", + "deviceHostName": "hostname", + "filename": "Security", + "message": "WinEvtLog Security: AUDIT_FAILURE(4625): Microsoft-Windows-Security-Auditing: (no user): no domain: WIN-RM6HM42G65V: An account failed to log on. Subject: ..", + "sourceAddress": "127.0.0.1", + "sourceHostName": "WIN-RM6HM42G65V" + }, + "name": "Microsoft Windows Events", + "severity": "8", + "version": "0" + }, + "destination": { + "user": { + "name": "(no user)" + } + }, + "event": { + "code": "3002795", + "original": "CEF:0|Trend Micro|Deep Security Agent|1.2.3|3002795|Microsoft Windows Events|8|cn1=1 cn1Label=Host ID dvchost=hostname cs1Label=LI Description cs1=Multiple Windows Logon Failures fname=Security src=127.0.0.1 duser=(no user) shost=WIN-RM6HM42G65V msg=WinEvtLog Security: AUDIT_FAILURE(4625): Microsoft-Windows-Security-Auditing: (no user): no domain: WIN-RM6HM42G65V: An account failed to log on. Subject: ..", + "severity": 8 + }, + "file": { + "name": "Security" + }, + "message": "WinEvtLog Security: AUDIT_FAILURE(4625): Microsoft-Windows-Security-Auditing: (no user): no domain: WIN-RM6HM42G65V: An account failed to log on. Subject: ..", + "observer": { + "hostname": "hostname", + "product": "Deep Security Agent", + "vendor": "Trend Micro", + "version": "1.2.3" + }, + "source": { + "domain": "WIN-RM6HM42G65V", + "ip": "127.0.0.1" + } + }, + { + "cef": { + "device": { + "event_class_id": "5000000", + "product": "Deep Security Agent", + "vendor": "Trend Micro", + "version": "1.2.3" + }, + "extensions": { + "deviceCustomNumber1": "1", + "deviceCustomNumber1Label": "Host ID", + "deviceHostName": "hostname", + "message": "Blocked By Admin", + "requestUrl": "example.com" + }, + "name": "WebReputation", + "severity": "5", + "version": "0" + }, + "event": { + "code": "5000000", + "original": "CEF:0|Trend Micro|Deep Security Agent|1.2.3|5000000|WebReputation|5|cn1=1 cn1Label=Host ID dvchost=hostname request=example.com msg=Blocked By Admin", + "severity": 5 + }, + "message": "Blocked By Admin", + "observer": { + "hostname": "hostname", + "product": "Deep Security Agent", + "vendor": "Trend Micro", + "version": "1.2.3" + }, + "url": { + "original": "example.com" + } + }, + { + "cef": { + "device": { + "event_class_id": "APPFW", + "product": "NetScaler", + "vendor": "Citrix", + "version": "NS10.0" + }, + "extensions": { + "DeviceCustomNumber2": "205", + "deviceAction": "blocked", + "deviceCustomNumber1": "233", + "deviceCustomString1": "profile1", + "deviceCustomString2": "PPE0", + "deviceCustomString3": "AjSZM26h2M+xL809pON6C8joebUA000", + "deviceCustomString4": "ALERT", + "deviceCustomString5": "2012", + "message": "Disallow Illegal URL.", + "method": "GET", + "requestUrl": "http://vpx247.example.net/FFC/login.html", + "sourceAddress": "10.217.253.78", + "sourcePort": "53743" + }, + "name": "APPFW_STARTURL", + "severity": "6", + "version": "0" + }, + "event": { + "action": "blocked", + "code": "APPFW", + "original": "CEF:0|Citrix|NetScaler|NS10.0|APPFW|APPFW_STARTURL|6|src=10.217.253.78 spt=53743 method=GET request=http://vpx247.example.net/FFC/login.html msg=Disallow Illegal URL. cn1=233 cn2=205 cs1=profile1 cs2=PPE0 cs3=AjSZM26h2M+xL809pON6C8joebUA000 cs4=ALERT cs5=2012 act=blocked", + "severity": 6 + }, + "message": "Disallow Illegal URL.", + "observer": { + "product": "NetScaler", + "vendor": "Citrix", + "version": "NS10.0" + }, + "source": { + "ip": "10.217.253.78", + "port": 53743 + }, + "url": { + "original": "http://vpx247.example.net/FFC/login.html" + } + }, + { + "cef": { + "device": { + "event_class_id": "APPFW", + "product": "NetScaler", + "vendor": "Citrix", + "version": "NS10.0" + }, + "extensions": { + "DeviceCustomNumber2": "535", + "deviceAction": "not blocked", + "deviceCustomNumber1": "465", + "deviceCustomString1": "profile1", + "deviceCustomString2": "PPE0", + "deviceCustomString3": "IliG4Dxp1SjOhKVRDVBXmqvAaIcA000", + "deviceCustomString4": "ALERT", + "deviceCustomString5": "2012", + "message": "Disallow Illegal URL.", + "method": "GET", + "requestUrl": "http://vpx247.example.net/FFC/login_post.html?abc=def", + "sourceAddress": "10.217.253.78", + "sourcePort": "54711" + }, + "name": "APPFW_STARTURL", + "severity": "6", + "version": "0" + }, + "event": { + "action": "not blocked", + "code": "APPFW", + "original": "CEF:0|Citrix|NetScaler|NS10.0|APPFW|APPFW_STARTURL|6|src=10.217.253.78 spt=54711 method=GET request=http://vpx247.example.net/FFC/login_post.html?abc\\=def msg=Disallow Illegal URL. cn1=465 cn2=535 cs1=profile1 cs2=PPE0 cs3=IliG4Dxp1SjOhKVRDVBXmqvAaIcA000 cs4=ALERT cs5=2012 act=not blocked", + "severity": 6 + }, + "message": "Disallow Illegal URL.", + "observer": { + "product": "NetScaler", + "vendor": "Citrix", + "version": "NS10.0" + }, + "source": { + "ip": "10.217.253.78", + "port": 54711 + }, + "url": { + "original": "http://vpx247.example.net/FFC/login_post.html?abc=def" + } + }, + { + "cef": { + "device": { + "event_class_id": "APPFW", + "product": "NetScaler", + "vendor": "Citrix", + "version": "NS10.0" + }, + "extensions": { + "DeviceCustomNumber2": "610", + "deviceAction": "transformed", + "deviceCustomNumber1": "652", + "deviceCustomString1": "pr_ffc", + "deviceCustomString2": "PPE0", + "deviceCustomString3": "li8MdGfW49uG8tGdSV85ech41a0A000", + "deviceCustomString4": "ALERT", + "deviceCustomString5": "2012", + "message": " Transformed (xout) potential credit card numbers seen in server response", + "method": "GET", + "requestUrl": "http://vpx247.example.net/FFC/CreditCardMind.html", + "sourceAddress": "10.217.253.78", + "sourcePort": "56116" + }, + "name": "APPFW_SAFECOMMERCE_XFORM", + "severity": "6", + "version": "0" + }, + "event": { + "action": "transformed", + "code": "APPFW", + "original": "CEF:0|Citrix|NetScaler|NS10.0|APPFW|APPFW_SAFECOMMERCE_XFORM|6|src=10.217.253.78 spt=56116 method=GET request=http://vpx247.example.net/FFC/CreditCardMind.html msg= Transformed (xout) potential credit card numbers seen in server response cn1=652 cn2=610 cs1=pr_ffc cs2=PPE0 cs3=li8MdGfW49uG8tGdSV85ech41a0A000 cs4=ALERT cs5=2012 act=transformed", + "severity": 6 + }, + "message": " Transformed (xout) potential credit card numbers seen in server response", + "observer": { + "product": "NetScaler", + "vendor": "Citrix", + "version": "NS10.0" + }, + "source": { + "ip": "10.217.253.78", + "port": 56116 + }, + "url": { + "original": "http://vpx247.example.net/FFC/CreditCardMind.html" + } + }, + { + "cef": { + "device": { + "event_class_id": "APPFW", + "product": "NetScaler", + "vendor": "Citrix", + "version": "NS10.0" + }, + "extensions": { + "DeviceCustomNumber2": "610", + "deviceAction": "transformed", + "deviceCustomNumber1": "653", + "deviceCustomString1": "pr_ffc", + "deviceCustomString2": "PPE0", + "deviceCustomString3": "li8MdGfW49uG8tGdSV85ech41a0A000", + "deviceCustomString4": "ALERT", + "deviceCustomString5": "2012", + "message": " Maximum no. of potential credit card numbers seen", + "method": "GET", + "requestUrl": "http://vpx247.example.net/FFC/CreditCardMind.html", + "sourceAddress": "10.217.253.78", + "sourcePort": "56116" + }, + "name": "APPFW_SAFECOMMERCE", + "severity": "6", + "version": "0" + }, + "event": { + "action": "transformed", + "code": "APPFW", + "original": "CEF:0|Citrix|NetScaler|NS10.0|APPFW|APPFW_SAFECOMMERCE|6|src=10.217.253.78 spt=56116 method=GET request=http://vpx247.example.net/FFC/CreditCardMind.html msg= Maximum no. of potential credit card numbers seen cn1=653 cn2=610 cs1=pr_ffc cs2=PPE0 cs3=li8MdGfW49uG8tGdSV85ech41a0A000 cs4=ALERT cs5=2012 act=transformed", + "severity": 6 + }, + "message": " Maximum no. of potential credit card numbers seen", + "observer": { + "product": "NetScaler", + "vendor": "Citrix", + "version": "NS10.0" + }, + "source": { + "ip": "10.217.253.78", + "port": 56116 + }, + "url": { + "original": "http://vpx247.example.net/FFC/CreditCardMind.html" + } + }, + { + "cef": { + "device": { + "event_class_id": "APPFW", + "product": "NetScaler", + "vendor": "Citrix", + "version": "NS10.0" + }, + "extensions": { + "DeviceCustomNumber2": "205", + "deviceAction": "not blocked", + "deviceCustomNumber1": "224", + "deviceCustomString1": "pr_ffc", + "deviceCustomString2": "PPE0", + "deviceCustomString3": "POousP7CIMW5nwZ5Rs4nq5DND0sA000", + "deviceCustomString4": "ALERT", + "deviceCustomString5": "2012", + "message": " Signature violation rule ID 807: web-cgi /wwwboard/passwd.txt access", + "method": "GET", + "requestUrl": "http://vpx247.example.net/FFC/wwwboard/passwd.txt", + "sourceAddress": "10.217.253.78", + "sourcePort": "56687" + }, + "name": "APPFW_SIGNATURE_MATCH", + "severity": "6", + "version": "0" + }, + "event": { + "action": "not blocked", + "code": "APPFW", + "original": "CEF:0|Citrix|NetScaler|NS10.0|APPFW|APPFW_SIGNATURE_MATCH|6|src=10.217.253.78 spt=56687 method=GET request=http://vpx247.example.net/FFC/wwwboard/passwd.txt msg= Signature violation rule ID 807: web-cgi /wwwboard/passwd.txt access cn1=224 cn2=205 cs1=pr_ffc cs2=PPE0 cs3=POousP7CIMW5nwZ5Rs4nq5DND0sA000 cs4=ALERT cs5=2012 act=not blocked", + "severity": 6 + }, + "message": " Signature violation rule ID 807: web-cgi /wwwboard/passwd.txt access", + "observer": { + "product": "NetScaler", + "vendor": "Citrix", + "version": "NS10.0" + }, + "source": { + "ip": "10.217.253.78", + "port": 56687 + }, + "url": { + "original": "http://vpx247.example.net/FFC/wwwboard/passwd.txt" + } + }, + { + "cef": { + "device": { + "event_class_id": "18", + "product": "Vaporware", + "vendor": "Elastic", + "version": "1.0.0-alpha" + }, + "extensions": { + "destinationGeoLatitude": "38.915", + "destinationGeoLongitude": "-77.511", + "destinationServiceName": "chrome", + "eventId": "3457", + "rawEvent": "{\"x\": \"y\"}", + "requestContext": "application/json", + "requestMethod": "POST", + "sourceServiceName": "httpd", + "transportProtocol": "TCP" + }, + "name": "Something awesome happened", + "severity": "very-high", + "version": "0" + }, + "destination": { + "geo": { + "location": { + "lat": 38.915, + "lon": -77.511 + } + }, + "service": { + "name": "chrome" + } + }, + "event": { + "code": "18", + "id": 3457, + "original": "CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Something awesome happened|very-high|eventId=3457 requestMethod=POST dlat=38.915 dlong=-77.511 proto=TCP rawEvent={\"x\": \"y\"} sourceServiceName=httpd destinationServiceName=chrome requestContext=application/json", + "severity": 9 + }, + "http": { + "request": { + "method": "POST" + } + }, + "message": "Something awesome happened", + "network": { + "transport": "tcp" + }, + "observer": { + "product": "Vaporware", + "vendor": "Elastic", + "version": "1.0.0-alpha" + }, + "source": { + "service": { + "name": "httpd" + } + } + } +]