Skip to content

Commit

Permalink
sFlow対応
Browse files Browse the repository at this point in the history
  • Loading branch information
@twsnmp
twsnmp committed Jun 25, 2024
1 parent 5afe121 commit ed9a6ea
Show file tree
Hide file tree
Showing 14 changed files with 459 additions and 33 deletions.
4 changes: 3 additions & 1 deletion datastore/datastore.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -277,7 +277,9 @@ func openDB(path string) error {

func initDB() error {
buckets := []string{"config", "nodes", "items", "lines", "pollings", "logs", "pollingLogs",
"syslog", "trap", "netflow", "ipfix", "arplog", "arp", "ai", "report", "grok", "images"}
"syslog", "trap", "netflow", "ipfix", "arplog", "arp", "ai", "report", "grok", "images",
"sflow", "sflowCounter",
}
reports := []string{"devices", "users", "flows", "servers", "ips",
"ether", "dns", "radius", "tls", "cert",
"sensor",
Expand Down
21 changes: 18 additions & 3 deletions datastore/log.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,12 @@ type LogFilterEnt struct {
LogType string
}

type SFlowCounterEnt struct {
Remote string
Type string
Data string
}

func AddEventLog(e *EventLogEnt) {
e.Time = time.Now().UnixNano()
if e.NodeID != "" && e.NodeName == "" {
Expand Down Expand Up @@ -224,7 +230,7 @@ func deleteOldLogs() {
log.Printf("deleteOldLog err=%v", err)
return
}
buckets := []string{"logs", "pollingLogs", "syslog", "trap", "netflow", "ipfix", "arplog"}
buckets := []string{"logs", "pollingLogs", "syslog", "trap", "netflow", "ipfix", "arplog", "sflow", "sflowCounter"}
doneMap := make(map[string]bool)
doneCount := 0
lt := time.Now().Unix() + 50
Expand Down Expand Up @@ -257,7 +263,7 @@ func deleteOldLogs() {

func DeleteAllLogs() {
st := time.Now()
buckets := []string{"logs", "pollingLogs", "syslog", "trap", "netflow", "ipfix"}
buckets := []string{"logs", "pollingLogs", "syslog", "trap", "netflow", "ipfix", "sflow", "sflowCounter"}
for _, b := range buckets {
db.Batch(func(tx *bbolt.Tx) error {
if err := tx.DeleteBucket([]byte(b)); err != nil {
Expand Down Expand Up @@ -395,6 +401,8 @@ func SaveLogBuffer(logBuffer []*LogEnt) {
}
syslog := tx.Bucket([]byte("syslog"))
netflow := tx.Bucket([]byte("netflow"))
sflow := tx.Bucket([]byte("sflow"))
sflowCounter := tx.Bucket([]byte("sflowCounter"))
ipfix := tx.Bucket([]byte("ipfix"))
trap := tx.Bucket([]byte("trap"))
arp := tx.Bucket([]byte("arplog"))
Expand All @@ -403,6 +411,7 @@ func SaveLogBuffer(logBuffer []*LogEnt) {
tc := 0
ac := 0
oc := 0
sf := 0
for i, l := range logBuffer {
k := fmt.Sprintf("%016x", l.Time+int64(i))
s, err := json.Marshal(l)
Expand Down Expand Up @@ -430,11 +439,17 @@ func SaveLogBuffer(logBuffer []*LogEnt) {
case "arplog":
ac++
arp.Put([]byte(k), []byte(s))
case "sflow":
sf++
sflow.Put([]byte(k), []byte(s))
case "sflowCounter":
sf++
sflowCounter.Put([]byte(k), []byte(s))
default:
oc++
}
}
log.Printf("syslog=%d,netflow=%d,trap=%d,arplog=%d,other=%d,dur=%v", sc, nfc, tc, ac, oc, time.Since(st))
log.Printf("syslog=%d,netflow=%d,trap=%d,arplog=%d,sflow=%d,other=%d,dur=%v", sc, nfc, tc, ac, sf, oc, time.Since(st))
return nil
})
}
Expand Down
1 change: 1 addition & 0 deletions datastore/map.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ type MapConfEnt struct {
EnableNetflowd bool
EnableArpWatch bool
EnableSshd bool
EnableSflowd bool
EnableMobileAPI bool
AILevel string
AIThreshold int
Expand Down
2 changes: 2 additions & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ module github.com/twsnmp/twsnmpfc
go 1.22

require (
github.com/Cistern/sflow v0.0.0-20240622235316-ed105e3cf9fb
github.com/PuerkitoBio/goquery v1.8.1
github.com/Songmu/timeout v0.4.0
github.com/beevik/ntp v0.3.0
Expand All @@ -12,6 +13,7 @@ require (
github.com/gliderlabs/ssh v0.3.6
github.com/go-playground/validator v9.31.0+incompatible
github.com/golang-jwt/jwt/v5 v5.0.0
github.com/google/gopacket v1.1.19
github.com/gosnmp/gosnmp v1.35.0
github.com/influxdata/influxdb1-client v0.0.0-20200827194710-b269163b24ab
github.com/labstack/echo-jwt/v4 v4.2.0
Expand Down
20 changes: 8 additions & 12 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7
gioui.org v0.0.0-20200628203458-851255f7a67b/go.mod h1:jiUwifN9cRl/zmco43aAqh0aV+s9GbhG13KcD+gEpkU=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/Cistern/sflow v0.0.0-20240622235316-ed105e3cf9fb h1:qyIMnqUY0n8L6vSPUONI87wG1CtA/AUUybMTmgWTHv0=
github.com/Cistern/sflow v0.0.0-20240622235316-ed105e3cf9fb/go.mod h1:l+o2vDh/2ArP1c4VtYBQeoNbRIdXd78j/aBnfFmKnQE=
github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM=
github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ=
github.com/Songmu/timeout v0.4.0 h1:7qUlKeO2neby/Htk9bYYd9w6VSj5MDkE6jnwGZV5zmU=
Expand Down Expand Up @@ -88,7 +90,6 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.5.0 h1:LUVKkCeviFUMKqHa4tXIIij/lbhnMbP7Fn5wKdKkRh4=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
Expand All @@ -104,6 +105,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
github.com/google/uuid v0.0.0-20170306145142-6a5e28554805 h1:skl44gU1qEIcRpwKjb9bhlRwjvr96wLdvpTogCBBJe8=
github.com/google/uuid v0.0.0-20170306145142-6a5e28554805/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gosnmp/gosnmp v1.35.0 h1:EuWWNPxTCdAUx2/NbQcSa3WdNxjzpy4Phv57b4MWpJM=
Expand Down Expand Up @@ -224,8 +227,6 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
Expand All @@ -244,8 +245,10 @@ golang.org/x/image v0.0.0-20200618115811-c13761719519/go.mod h1:FeLwcggjj3mMvU+o
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
Expand All @@ -263,8 +266,6 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo=
golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
Expand Down Expand Up @@ -298,24 +299,19 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA=
golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
Expand All @@ -330,6 +326,7 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
Expand Down Expand Up @@ -369,7 +366,6 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
Expand Down
17 changes: 16 additions & 1 deletion logger/logger.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,13 @@ var logCh = make(chan *datastore.LogEnt, 5000)
var trapListen = "0.0.0.0:162"
var netflowListen = ":2055"
var syslogListen = "0.0.0.0:514"
var sflowListen = ":6343"

func Start(ctx context.Context, wg *sync.WaitGroup, trap, netflow, syslog int) error {
func Start(ctx context.Context, wg *sync.WaitGroup, trap, netflow, syslog, sflow int) error {
trapListen = fmt.Sprintf("0.0.0.0:%d", trap)
netflowListen = fmt.Sprintf(":%d", netflow)
syslogListen = fmt.Sprintf("0.0.0.0:%d", syslog)
sflowListen = fmt.Sprintf(":%d", sflow)
logCh = make(chan *datastore.LogEnt, 100)
wg.Add(1)
go logger(ctx, wg)
Expand All @@ -39,11 +41,13 @@ func logger(ctx context.Context, wg *sync.WaitGroup) {
var netflowdRunning = false
var arpWatchRunning = false
var sshdRunning = false
var sflowdRunning = false
var stopSyslogd chan bool
var stopTrapd chan bool
var stopNetflowd chan bool
var stopArpWatch chan bool
var stopSshd chan bool
var stopSflowd chan bool
log.Println("start logger")
timer1 := time.NewTicker(time.Second * 10)
timer2 := time.NewTicker(time.Second * 1)
Expand All @@ -69,6 +73,9 @@ func logger(ctx context.Context, wg *sync.WaitGroup) {
if sshdRunning {
close(stopSshd)
}
if sflowdRunning {
close(stopSflowd)
}
if len(logBuffer) > 0 {
datastore.SaveLogBuffer(logBuffer)
}
Expand Down Expand Up @@ -123,6 +130,14 @@ func logger(ctx context.Context, wg *sync.WaitGroup) {
close(stopSshd)
sshdRunning = false
}
if datastore.MapConf.EnableSflowd && !sflowdRunning {
stopSflowd = make(chan bool)
sflowdRunning = true
go sflowd(stopSflowd)
} else if !datastore.MapConf.EnableSflowd && sflowdRunning {
close(stopSflowd)
sflowdRunning = false
}
if datastore.RestartSnmpTrapd && trapdRunning {
close(stopTrapd)
datastore.RestartSnmpTrapd = false
Expand Down
Loading

0 comments on commit ed9a6ea

Please sign in to comment.