Skip to content

Commit

Permalink
Support Go execd plugins with shim (influxdata#7283)
Browse files Browse the repository at this point in the history
  • Loading branch information
ssoroka authored May 4, 2020
1 parent 7a5690c commit b73a232
Show file tree
Hide file tree
Showing 58 changed files with 915 additions and 65 deletions.
8 changes: 8 additions & 0 deletions EXTERNAL_PLUGINS.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
# External Plugins

This is a list of plugins that can be compiled outside of Telegraf and used via the execd input.

Pull requests welcome.

## Inputs
- [rand](https://github.com/ssoroka/rand) - Generate random numbers
2 changes: 1 addition & 1 deletion agent/accumulator_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (
"time"

"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/internal/models"
"github.com/influxdata/telegraf/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
Expand Down
4 changes: 2 additions & 2 deletions agent/agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,9 @@ import (
"time"

"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/config"
"github.com/influxdata/telegraf/internal"
"github.com/influxdata/telegraf/internal/config"
"github.com/influxdata/telegraf/internal/models"
"github.com/influxdata/telegraf/models"
"github.com/influxdata/telegraf/plugins/serializers/influx"
)

Expand Down
26 changes: 13 additions & 13 deletions agent/agent_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import (
"testing"
"time"

"github.com/influxdata/telegraf/internal/config"
"github.com/influxdata/telegraf/config"
_ "github.com/influxdata/telegraf/plugins/inputs/all"
_ "github.com/influxdata/telegraf/plugins/outputs/all"
"github.com/stretchr/testify/assert"
Expand All @@ -22,35 +22,35 @@ func TestAgent_OmitHostname(t *testing.T) {
func TestAgent_LoadPlugin(t *testing.T) {
c := config.NewConfig()
c.InputFilters = []string{"mysql"}
err := c.LoadConfig("../internal/config/testdata/telegraf-agent.toml")
err := c.LoadConfig("../config/testdata/telegraf-agent.toml")
assert.NoError(t, err)
a, _ := NewAgent(c)
assert.Equal(t, 1, len(a.Config.Inputs))

c = config.NewConfig()
c.InputFilters = []string{"foo"}
err = c.LoadConfig("../internal/config/testdata/telegraf-agent.toml")
err = c.LoadConfig("../config/testdata/telegraf-agent.toml")
assert.NoError(t, err)
a, _ = NewAgent(c)
assert.Equal(t, 0, len(a.Config.Inputs))

c = config.NewConfig()
c.InputFilters = []string{"mysql", "foo"}
err = c.LoadConfig("../internal/config/testdata/telegraf-agent.toml")
err = c.LoadConfig("../config/testdata/telegraf-agent.toml")
assert.NoError(t, err)
a, _ = NewAgent(c)
assert.Equal(t, 1, len(a.Config.Inputs))

c = config.NewConfig()
c.InputFilters = []string{"mysql", "redis"}
err = c.LoadConfig("../internal/config/testdata/telegraf-agent.toml")
err = c.LoadConfig("../config/testdata/telegraf-agent.toml")
assert.NoError(t, err)
a, _ = NewAgent(c)
assert.Equal(t, 2, len(a.Config.Inputs))

c = config.NewConfig()
c.InputFilters = []string{"mysql", "foo", "redis", "bar"}
err = c.LoadConfig("../internal/config/testdata/telegraf-agent.toml")
err = c.LoadConfig("../config/testdata/telegraf-agent.toml")
assert.NoError(t, err)
a, _ = NewAgent(c)
assert.Equal(t, 2, len(a.Config.Inputs))
Expand All @@ -59,50 +59,50 @@ func TestAgent_LoadPlugin(t *testing.T) {
func TestAgent_LoadOutput(t *testing.T) {
c := config.NewConfig()
c.OutputFilters = []string{"influxdb"}
err := c.LoadConfig("../internal/config/testdata/telegraf-agent.toml")
err := c.LoadConfig("../config/testdata/telegraf-agent.toml")
assert.NoError(t, err)
a, _ := NewAgent(c)
assert.Equal(t, 2, len(a.Config.Outputs))

c = config.NewConfig()
c.OutputFilters = []string{"kafka"}
err = c.LoadConfig("../internal/config/testdata/telegraf-agent.toml")
err = c.LoadConfig("../config/testdata/telegraf-agent.toml")
assert.NoError(t, err)
a, _ = NewAgent(c)
assert.Equal(t, 1, len(a.Config.Outputs))

c = config.NewConfig()
c.OutputFilters = []string{}
err = c.LoadConfig("../internal/config/testdata/telegraf-agent.toml")
err = c.LoadConfig("../config/testdata/telegraf-agent.toml")
assert.NoError(t, err)
a, _ = NewAgent(c)
assert.Equal(t, 3, len(a.Config.Outputs))

c = config.NewConfig()
c.OutputFilters = []string{"foo"}
err = c.LoadConfig("../internal/config/testdata/telegraf-agent.toml")
err = c.LoadConfig("../config/testdata/telegraf-agent.toml")
assert.NoError(t, err)
a, _ = NewAgent(c)
assert.Equal(t, 0, len(a.Config.Outputs))

c = config.NewConfig()
c.OutputFilters = []string{"influxdb", "foo"}
err = c.LoadConfig("../internal/config/testdata/telegraf-agent.toml")
err = c.LoadConfig("../config/testdata/telegraf-agent.toml")
assert.NoError(t, err)
a, _ = NewAgent(c)
assert.Equal(t, 2, len(a.Config.Outputs))

c = config.NewConfig()
c.OutputFilters = []string{"influxdb", "kafka"}
err = c.LoadConfig("../internal/config/testdata/telegraf-agent.toml")
err = c.LoadConfig("../config/testdata/telegraf-agent.toml")
assert.NoError(t, err)
assert.Equal(t, 3, len(c.Outputs))
a, _ = NewAgent(c)
assert.Equal(t, 3, len(a.Config.Outputs))

c = config.NewConfig()
c.OutputFilters = []string{"influxdb", "foo", "kafka", "bar"}
err = c.LoadConfig("../internal/config/testdata/telegraf-agent.toml")
err = c.LoadConfig("../config/testdata/telegraf-agent.toml")
assert.NoError(t, err)
a, _ = NewAgent(c)
assert.Equal(t, 3, len(a.Config.Outputs))
Expand Down
2 changes: 1 addition & 1 deletion cmd/telegraf/telegraf.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ import (
"time"

"github.com/influxdata/telegraf/agent"
"github.com/influxdata/telegraf/config"
"github.com/influxdata/telegraf/internal"
"github.com/influxdata/telegraf/internal/config"
"github.com/influxdata/telegraf/internal/goplugin"
"github.com/influxdata/telegraf/logger"
_ "github.com/influxdata/telegraf/plugins/aggregators/all"
Expand Down
File renamed without changes.
2 changes: 1 addition & 1 deletion internal/config/config.go → config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import (

"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/internal"
"github.com/influxdata/telegraf/internal/models"
"github.com/influxdata/telegraf/models"
"github.com/influxdata/telegraf/plugins/aggregators"
"github.com/influxdata/telegraf/plugins/inputs"
"github.com/influxdata/telegraf/plugins/outputs"
Expand Down
2 changes: 1 addition & 1 deletion internal/config/config_test.go → config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import (
"time"

"github.com/influxdata/telegraf/internal"
"github.com/influxdata/telegraf/internal/models"
"github.com/influxdata/telegraf/models"
"github.com/influxdata/telegraf/plugins/inputs"
"github.com/influxdata/telegraf/plugins/inputs/exec"
"github.com/influxdata/telegraf/plugins/inputs/http_listener_v2"
Expand Down
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
88 changes: 88 additions & 0 deletions config/types.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
package config

import (
"bytes"
"strconv"
"time"

"github.com/alecthomas/units"
)

// Duration is a time.Duration
type Duration time.Duration

// Size is an int64
type Size int64

// Number is a float
type Number float64

// UnmarshalTOML parses the duration from the TOML config file
func (d Duration) UnmarshalTOML(b []byte) error {
var err error
b = bytes.Trim(b, `'`)

// see if we can directly convert it
dur, err := time.ParseDuration(string(b))
if err == nil {
d = Duration(dur)
return nil
}

// Parse string duration, ie, "1s"
if uq, err := strconv.Unquote(string(b)); err == nil && len(uq) > 0 {
dur, err := time.ParseDuration(uq)
if err == nil {
d = Duration(dur)
return nil
}
}

// First try parsing as integer seconds
sI, err := strconv.ParseInt(string(b), 10, 64)
if err == nil {
dur := time.Second * time.Duration(sI)
d = Duration(dur)
return nil
}
// Second try parsing as float seconds
sF, err := strconv.ParseFloat(string(b), 64)
if err == nil {
dur := time.Second * time.Duration(sF)
d = Duration(dur)
return nil
}

return nil
}

func (s Size) UnmarshalTOML(b []byte) error {
var err error
b = bytes.Trim(b, `'`)

val, err := strconv.ParseInt(string(b), 10, 64)
if err == nil {
s = Size(val)
return nil
}
uq, err := strconv.Unquote(string(b))
if err != nil {
return err
}
val, err = units.ParseStrictBytes(uq)
if err != nil {
return err
}
s = Size(val)
return nil
}

func (n Number) UnmarshalTOML(b []byte) error {
value, err := strconv.ParseFloat(string(b), 64)
if err != nil {
return err
}

n = Number(value)
return nil
}
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ require (
github.com/Azure/azure-storage-queue-go v0.0.0-20181215014128-6ed74e755687
github.com/Azure/go-autorest/autorest v0.9.3
github.com/Azure/go-autorest/autorest/azure/auth v0.4.2
github.com/BurntSushi/toml v0.3.1
github.com/Mellanox/rdmamap v0.0.0-20191106181932-7c3c4763a6ee
github.com/Microsoft/ApplicationInsights-Go v0.4.2
github.com/Microsoft/go-winio v0.4.9 // indirect
Expand Down
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
2 changes: 1 addition & 1 deletion plugins/inputs/cloud_pubsub_push/pubsub_push_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import (
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/agent"
"github.com/influxdata/telegraf/internal"
"github.com/influxdata/telegraf/internal/models"
"github.com/influxdata/telegraf/models"
"github.com/influxdata/telegraf/plugins/parsers"
"github.com/influxdata/telegraf/testutil"
)
Expand Down
2 changes: 1 addition & 1 deletion plugins/inputs/cloudwatch/cloudwatch.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,9 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/cloudwatch"
"github.com/influxdata/telegraf"
internalaws "github.com/influxdata/telegraf/config/aws"
"github.com/influxdata/telegraf/filter"
"github.com/influxdata/telegraf/internal"
internalaws "github.com/influxdata/telegraf/internal/config/aws"
"github.com/influxdata/telegraf/internal/limiter"
"github.com/influxdata/telegraf/metric"
"github.com/influxdata/telegraf/plugins/inputs"
Expand Down
10 changes: 9 additions & 1 deletion plugins/inputs/execd/examples/count.rb
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,16 @@

counter = 0

def time_ns_str(t)
ns = t.nsec.to_s
(9 - ns.size).times do
ns = "0" + ns # left pad
end
t.to_i.to_s + ns
end

loop do
puts "counter_ruby count=#{counter}"
puts "counter_ruby count=#{counter} #{time_ns_str(Time.now)}"
STDOUT.flush
counter += 1

Expand Down
8 changes: 4 additions & 4 deletions plugins/inputs/execd/examples/count.sh
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
#!/bin/bash
#!/bin/sh

## Example in bash using STDIN signaling

counter=0

while read; do
while read LINE; do
echo "counter_bash count=${counter}"
let counter=counter+1
counter=$((counter+1))
done

(>&2 echo "terminate")
trap "echo terminate 1>&2" EXIT
Loading

0 comments on commit b73a232

Please sign in to comment.