Skip to content

Commit

Permalink
Add query by event ID, level, provider, and age (time)
Browse files Browse the repository at this point in the history
Use ucfg to unpack eventlog API config
All system tests now require Windows.
Add ignore_older filtering to eventlogging API since it is not provided by Windows
Change expected time.ParseDuration error message due to ucfg change
andrewkroh committed Mar 28, 2016
1 parent 663becb commit e5565f5
Showing 21 changed files with 1,201 additions and 569 deletions.
4 changes: 0 additions & 4 deletions .travis.yml
Original file line number Diff line number Diff line change
@@ -18,7 +18,6 @@ env:
- TARGETS="-C libbeat testsuite"
- TARGETS="-C topbeat testsuite"
- TARGETS="-C filebeat testsuite"
- TARGETS="-C winlogbeat testsuite"
- TARGETS="-C packetbeat testsuite"
- TARGETS="-C metricbeat testsuite"
- TARGETS="-C libbeat crosscompile"
@@ -36,8 +35,6 @@ matrix:
env: TARGETS="-C filebeat crosscompile"
- os: osx
env: TARGETS="-C libbeat crosscompile"
- os: osx
env: TARGETS="-C winlogbeat testsuite"
- os: osx
env: TARGETS="-C winlogbeat crosscompile"
- os: osx
@@ -86,5 +83,4 @@ after_success:
- test -f packetbeat/build/coverage/full.cov && bash <(curl -s https://codecov.io/bash) -f packetbeat/build/coverage/full.cov
- test -f topbeat/build/coverage/full.cov && bash <(curl -s https://codecov.io/bash) -f topbeat/build/coverage/full.cov
- test -f libbeat/build/coverage/full.cov && bash <(curl -s https://codecov.io/bash) -f libbeat/build/coverage/full.cov
- test -f winlogbeat/build/coverage/full.cov && bash <(curl -s https://codecov.io/bash) -f winlogbeat/build/coverage/full.cov
- test -f metricbeat/build/coverage/full.cov && bash <(curl -s https://codecov.io/bash) -f metricbeat/build/coverage/full.cov
3 changes: 3 additions & 0 deletions CHANGELOG.asciidoc
Original file line number Diff line number Diff line change
@@ -65,6 +65,7 @@ https://github.com/elastic/beats/compare/v1.1.2...master[Check the HEAD diff]
- Omit `fields` from Filebeat events when null {issue}899[899]

*Winlogbeat*
- Fix invalid `event_id` on Windows XP and Windows 2003 {pull}1153[1153]


==== Added
@@ -105,6 +106,8 @@ https://github.com/elastic/beats/compare/v1.1.2...master[Check the HEAD diff]
- Add additional data to the events published by Winlogbeat. The new fields are `activity_id`,
`event_data`, `keywords`, `opcode`, `process_id`, `provider_guid`, `related_activity_id`,
`task`, `thread_id`, `user_data`. and `version`. {issue}1053[1053]
- Add `event_id`, `level`, and `provider` configuration options for filtering events {pull}1218[1218]
- Add `include_xml` configuration option for including the raw XML with the event {pull}1218[1218]

==== Deprecated

75 changes: 22 additions & 53 deletions winlogbeat/beater/winlogbeat.go
Original file line number Diff line number Diff line change
@@ -33,23 +33,17 @@ func init() {
// Debug logging functions for this package.
var (
debugf = logp.MakeDebug("winlogbeat")
detailf = logp.MakeDebug("winlogbeat_detail")
memstatsf = logp.MakeDebug("memstats")
)

// Time the application was started.
var startTime = time.Now().UTC()

type log struct {
config.EventLogConfig
eventLog eventlog.EventLog
}

// Winlogbeat is used to conform to the beat interface
type Winlogbeat struct {
beat *beat.Beat // Common beat information.
config *config.Settings // Configuration settings.
eventLogs []log // List of all event logs being monitored.
eventLogs []eventlog.EventLog // List of all event logs being monitored.
done chan struct{} // Channel to initiate shutdown of main event loop.
client publisher.Client // Interface to publish event.
checkpoint *checkpoint.Checkpoint // Persists event log state to disk.
@@ -116,15 +110,27 @@ func (eb *Winlogbeat) Setup(b *beat.Beat) error {
err := http.Serve(sock, nil)
if err != nil {
logp.Warn("Unable to launch HTTP service for metrics. %v", err)
return
}
}()
}

// Create the event logs. This will validate the event log specific
// configuration.
eb.eventLogs = make([]eventlog.EventLog, 0, len(eb.config.Winlogbeat.EventLogs))
for _, config := range eb.config.Winlogbeat.EventLogs {
eventLog, err := eventlog.New(config)
if err != nil {
return fmt.Errorf("Failed to create new event log. %v", err)
}
debugf("Initialized EventLog[%s]", eventLog.Name())

eb.eventLogs = append(eb.eventLogs, eventLog)
}

return nil
}

// Run is used within the beats interface to execute the winlogbeat.
// Run is used within the beats interface to execute the Winlogbeat workers.
func (eb *Winlogbeat) Run(b *beat.Beat) error {
persistedState := eb.checkpoint.States()

@@ -133,40 +139,17 @@ func (eb *Winlogbeat) Run(b *beat.Beat) error {
publishedEvents.Add("failures", 0)
ignoredEvents.Add("total", 0)

// TODO: If no event_logs are specified in the configuration, use the
// Windows registry to discover the available event logs.
eb.eventLogs = make([]log, 0, len(eb.config.Winlogbeat.EventLogs))
for _, eventLogConfig := range eb.config.Winlogbeat.EventLogs {
debugf("Initializing EventLog[%s]", eventLogConfig.Name)

eventLog, err := eventlog.New(eventlog.Config{
Name: eventLogConfig.Name,
API: eventLogConfig.API,
EventMetadata: eventLogConfig.EventMetadata,
})
if err != nil {
return fmt.Errorf("Failed to create new event log for %s. %v",
eventLogConfig.Name, err)
}

// Initialize per event log metrics.
publishedEvents.Add(eventLogConfig.Name, 0)
ignoredEvents.Add(eventLogConfig.Name, 0)

eb.eventLogs = append(eb.eventLogs, log{
EventLogConfig: eventLogConfig,
eventLog: eventLog,
})
}

var wg sync.WaitGroup
for _, log := range eb.eventLogs {
state, _ := persistedState[log.Name]
ignoreOlder, _ := config.IgnoreOlderDuration(log.IgnoreOlder)
state, _ := persistedState[log.Name()]

// Initialize per event log metrics.
publishedEvents.Add(log.Name(), 0)
ignoredEvents.Add(log.Name(), 0)

// Start a goroutine for each event log.
wg.Add(1)
go eb.processEventLog(&wg, log.eventLog, state, ignoreOlder)
go eb.processEventLog(&wg, log, state)
}

wg.Wait()
@@ -201,7 +184,6 @@ func (eb *Winlogbeat) processEventLog(
wg *sync.WaitGroup,
api eventlog.EventLog,
state checkpoint.EventLogState,
ignoreOlder time.Duration,
) {
defer wg.Done()

@@ -243,21 +225,8 @@ loop:
continue
}

// Filter events.
var events []common.MapStr
events := make([]common.MapStr, 0, len(records))
for _, lr := range records {
// TODO: Move filters close to source. Short circuit processing
// of event if it is going to be filtered.
// TODO: Add a severity filter.
// TODO: Check the global IgnoreOlder filter.
if ignoreOlder != 0 && time.Since(lr.TimeCreated.SystemTime) > ignoreOlder {
detailf("EventLog[%s] ignore_older filter dropping event: %+v",
api.Name(), lr)
ignoredEvents.Add("total", 1)
ignoredEvents.Add(api.Name(), 1)
continue
}

events = append(events, lr.ToMapStr())
}

71 changes: 5 additions & 66 deletions winlogbeat/config/config.go
Original file line number Diff line number Diff line change
@@ -7,9 +7,7 @@ import (
"sort"
"strconv"
"strings"
"time"

"github.com/elastic/beats/libbeat/common"
"github.com/joeshaw/multierror"
)

@@ -30,7 +28,7 @@ type Validator interface {
// Settings is the root of the Winlogbeat configuration data hierarchy.
type Settings struct {
Winlogbeat WinlogbeatConfig `config:"winlogbeat"`
All map[string]interface{} `config:",inline"`
Raw map[string]interface{} `config:",inline"`
}

// Validate validates the Settings data and returns an error describing
@@ -41,7 +39,7 @@ func (s Settings) Validate() error {

// Check for invalid top-level keys.
var errs multierror.Errors
for k := range s.All {
for k := range s.Raw {
k = strings.ToLower(k)
i := sort.SearchStrings(validKeys, k)
if i >= len(validKeys) || validKeys[i] != k {
@@ -60,32 +58,21 @@ func (s Settings) Validate() error {

// WinlogbeatConfig contains all of Winlogbeat configuration data.
type WinlogbeatConfig struct {
IgnoreOlder string `config:"ignore_older"`
EventLogs []EventLogConfig `config:"event_logs"`
Metrics MetricsConfig `config:"metrics"`
RegistryFile string `config:"registry_file"`
EventLogs []map[string]interface{} `config:"event_logs"`
Metrics MetricsConfig `config:"metrics"`
RegistryFile string `config:"registry_file"`
}

// Validate validates the WinlogbeatConfig data and returns an error describing
// all problems or nil if there are none.
func (ebc WinlogbeatConfig) Validate() error {
var errs multierror.Errors
if _, err := IgnoreOlderDuration(ebc.IgnoreOlder); err != nil {
errs = append(errs, fmt.Errorf("Invalid top level ignore_older value "+
"'%s' (%v)", ebc.IgnoreOlder, err))
}

if len(ebc.EventLogs) == 0 {
errs = append(errs, fmt.Errorf("At least one event log must be "+
"configured as part of event_logs"))
}

for _, eventLog := range ebc.EventLogs {
if err := eventLog.Validate(); err != nil {
errs = append(errs, err)
}
}

if err := ebc.Metrics.Validate(); err != nil {
errs = append(errs, err)
}
@@ -129,51 +116,3 @@ func (mc MetricsConfig) Validate() error {

return nil
}

// EventLogConfig holds the configuration data that specifies which event logs
// to monitor.
type EventLogConfig struct {
common.EventMetadata `config:",inline"`
Name string
IgnoreOlder string `config:"ignore_older"`
API string
}

// Validate validates the EventLogConfig data and returns an error describing
// any problems or nil.
func (elc EventLogConfig) Validate() error {
var errs multierror.Errors
if elc.Name == "" {
err := fmt.Errorf("event log is missing a 'name'")
errs = append(errs, err)
}

if _, err := IgnoreOlderDuration(elc.IgnoreOlder); err != nil {
err := fmt.Errorf("Invalid ignore_older value ('%s') for event_log "+
"'%s' (%v)", elc.IgnoreOlder, elc.Name, err)
errs = append(errs, err)
}

switch strings.ToLower(elc.API) {
case "", "eventlogging", "wineventlog":
break
default:
err := fmt.Errorf("Invalid api value ('%s') for event_log '%s'",
elc.API, elc.Name)
errs = append(errs, err)
}

return errs.Err()
}

// IgnoreOlderDuration returns the parsed value of the IgnoreOlder string. If
// IgnoreOlder is not set then (0, nil) is returned. If IgnoreOlder is not
// parsable as a duration then an error is returned. See time.ParseDuration.
func IgnoreOlderDuration(ignoreOlder string) (time.Duration, error) {
if ignoreOlder == "" {
return time.Duration(0), nil
}

duration, err := time.ParseDuration(ignoreOlder)
return duration, err
}
57 changes: 10 additions & 47 deletions winlogbeat/config/config_test.go
Original file line number Diff line number Diff line change
@@ -17,7 +17,10 @@ func (v validationTestCase) run(t *testing.T) {
if v.errMsg == "" {
assert.NoError(t, v.config.Validate())
} else {
assert.Contains(t, v.config.Validate().Error(), v.errMsg)
err := v.config.Validate()
if assert.Error(t, err, "expected '%s'", v.errMsg) {
assert.Contains(t, err.Error(), v.errMsg)
}
}
}

@@ -26,17 +29,17 @@ func TestConfigValidate(t *testing.T) {
// Top-level config
{
WinlogbeatConfig{
EventLogs: []EventLogConfig{
{Name: "App"},
EventLogs: []map[string]interface{}{
{"Name": "App"},
},
},
"", // No Error
},
{
Settings{
WinlogbeatConfig{
EventLogs: []EventLogConfig{
{Name: "App"},
EventLogs: []map[string]interface{}{
{"Name": "App"},
},
},
map[string]interface{}{"other": "value"},
@@ -49,29 +52,15 @@ func TestConfigValidate(t *testing.T) {
"1 error: At least one event log must be configured as part of " +
"event_logs",
},
{
WinlogbeatConfig{IgnoreOlder: "1"},
"2 errors: Invalid top level ignore_older value '1' (time: " +
"missing unit in duration 1); At least one event log must be " +
"configured as part of event_logs",
},
{
WinlogbeatConfig{
EventLogs: []EventLogConfig{
{Name: "App"},
EventLogs: []map[string]interface{}{
{"Name": "App"},
},
Metrics: MetricsConfig{BindAddress: "example.com"},
},
"1 error: bind_address",
},
{
WinlogbeatConfig{
EventLogs: []EventLogConfig{
{},
},
},
"1 error: event log is missing a 'name'",
},
// MetricsConfig
{
MetricsConfig{},
@@ -103,32 +92,6 @@ func TestConfigValidate(t *testing.T) {
MetricsConfig{BindAddress: "example.com:65536"},
"bind_address port must be within [1-65535] but was '65536'",
},
// EventLogConfig
{
EventLogConfig{Name: "System"},
"",
},
{
EventLogConfig{},
"event log is missing a 'name'",
},
{
EventLogConfig{Name: "System", IgnoreOlder: "24"},
"Invalid ignore_older value ('24') for event_log 'System' " +
"(time: missing unit in duration 24)",
},
{
EventLogConfig{Name: "System", API: "eventlogging"},
"",
},
{
EventLogConfig{Name: "System", API: "wineventlog"},
"",
},
{
EventLogConfig{Name: "System", API: "invalid"},
"Invalid api value ('invalid') for event_log 'System'",
},
}

for _, test := range testCases {
Loading

0 comments on commit e5565f5

Please sign in to comment.