Skip to content

Commit

Permalink
Gather FQDN IPs using DNS lookups from previous chunks (activecm#676)
Browse files Browse the repository at this point in the history 
* Determine which hostnames need their fqdn beacon entries updated by checking which hostnames are associated with the external IPs we saw in the current import run.

Move fqdnInput out of hostnames package and into beaconfqdn package.

Add ./.vscode to gitignore (vscode dlv debugger creates launch profiles here)

* linting fixes

* Simplify affectedHostnameIPsChunked, fix a preallocation issue

* added comment on preallocation of externalHosts in the affectedHostnames methods

* comment formatting on reverseDNSQueryWithIPs

* Allow disk use for large reverseDNSQuery aggregation

* Add requested comments, use short variable assignments

* Add println after stopping the first fqdn spinner

Co-authored-by: Logan L <[email protected]>
  • Loading branch information
@Zalgo2462
Zalgo2462 and Logan L authored Aug 23, 2021
1 parent afd9a3e commit 8baf8e5
Show file tree
Hide file tree
Showing 10 changed files with 303 additions and 67 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
*.swp
*.swo
*.exe
/.vscode/
/rita
/vendor/
/rita-html-report/
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ require (
github.com/activecm/mgosec v0.1.1
github.com/activecm/rita-bl v0.0.0-20200806232046-0db4a39fcf49
github.com/blang/semver v3.5.1+incompatible
github.com/briandowns/spinner v1.16.0
github.com/creasty/defaults v1.3.0
github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8
github.com/golang/protobuf v1.3.3 // indirect
Expand All @@ -20,7 +21,6 @@ require (
github.com/google/safebrowsing v0.0.0-20190214191829-0feabcc2960b // indirect
github.com/google/uuid v1.1.2
github.com/json-iterator/go v1.1.11
github.com/mattn/go-isatty v0.0.4 // indirect
github.com/mattn/go-runewidth v0.0.4 // indirect
github.com/olekukonko/tablewriter v0.0.2-0.20190214164707-93462a5dfaa6
github.com/pbnjay/memory v0.0.0-20201129165224-b12e5d931931
Expand Down
9 changes: 9 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,15 @@ github.com/activecm/rita-bl v0.0.0-20200806232046-0db4a39fcf49 h1:xvOWFArOpp9poD
github.com/activecm/rita-bl v0.0.0-20200806232046-0db4a39fcf49/go.mod h1:5a489AThTs93aEzhBN2toFq+Hb0sbvXrdxn34ai259o=
github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
github.com/briandowns/spinner v1.16.0 h1:DFmp6hEaIx2QXXuqSJmtfSBSAjRmpGiKG6ip2Wm/yOs=
github.com/briandowns/spinner v1.16.0/go.mod h1:QOuQk7x+EaDASo80FEXwlwiA+j/PPIcX3FScO+3/ZPQ=
github.com/creasty/defaults v1.3.0 h1:uG+RAxYbJgOPCOdKEcec9ZJXeva7Y6mj/8egdzwmLtw=
github.com/creasty/defaults v1.3.0/go.mod h1:CIEEvs7oIVZm30R8VxtFJs+4k201gReYyuYHJxZc68I=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/globalsign/mgo v0.0.0-20180615134936-113d3961e731/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 h1:DujepqpGd1hyOd7aW59XpK7Qymp8iy83xq74fLr21is=
github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
Expand All @@ -36,8 +40,12 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs=
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y=
github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
Expand Down Expand Up @@ -80,6 +88,7 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
Expand Down
18 changes: 10 additions & 8 deletions parser/fsimporter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -192,7 +192,7 @@ func (fs *FSImporter) Run(indexedFiles []*files.IndexedFile, threads int) {
fs.buildBeacons(retVals.UniqueConnMap, minTimestamp, maxTimestamp)

// build or update the FQDN Beacons Table
fs.buildFQDNBeacons(retVals.HostnameMap, minTimestamp, maxTimestamp)
fs.buildFQDNBeacons(retVals.HostMap, minTimestamp, maxTimestamp)

// build or update the Proxy Beacons Table
fs.buildProxyBeacons(retVals.ProxyUniqueConnMap, minTimestamp, maxTimestamp)
Expand Down Expand Up @@ -521,7 +521,7 @@ func (fs *FSImporter) buildHosts(hostMap map[string]*host.Input) {
fs.log.Error(err)
}

// send uconns to host analysis
// add the hosts to the database
hostRepo.Upsert(hostMap)
} else {
fmt.Println("\t[!] No Host data to analyze")
Expand All @@ -540,7 +540,7 @@ func (fs *FSImporter) markBlacklistedPeers(hostMap map[string]*host.Input) {
fs.log.Error(err)
}

// send uconns to host analysis
// send the hosts out for threat intel analysis
blacklistRepo.Upsert()
}
}
Expand All @@ -564,18 +564,20 @@ func (fs *FSImporter) buildBeacons(uconnMap map[string]*uconn.Input, minTimestam

}

func (fs *FSImporter) buildFQDNBeacons(hostnameMap map[string]*hostname.Input, minTimestamp, maxTimestamp int64) {
func (fs *FSImporter) buildFQDNBeacons(hostMap map[string]*host.Input, minTimestamp, maxTimestamp int64) {
if fs.config.S.BeaconFQDN.Enabled {
if len(hostnameMap) > 0 {
if len(hostMap) > 0 {
beaconFQDNRepo := beaconfqdn.NewMongoRepository(fs.database, fs.config, fs.log)

err := beaconFQDNRepo.CreateIndexes()
if err != nil {
fs.log.Error(err)
}

// send uconns to beacon analysis
beaconFQDNRepo.Upsert(hostnameMap, minTimestamp, maxTimestamp)
// Send the list of hosts out to the FQDN beacon analysis pkg.
// The list of external hosts seen in the current set of logs determines
// which FQDN beacons need to be updated.
beaconFQDNRepo.Upsert(hostMap, minTimestamp, maxTimestamp)
} else {
fmt.Println("\t[!] No FQDN Beacon data to analyze")
}
Expand All @@ -593,7 +595,7 @@ func (fs *FSImporter) buildProxyBeacons(uconnProxyMap map[string]*uconnproxy.Inp
fs.log.Error(err)
}

// send uconns to beacon analysis
// send proxy uconns to beacon analysis
beaconProxyRepo.Upsert(uconnProxyMap, minTimestamp, maxTimestamp)
} else {
fmt.Println("\t[!] No Proxy Beacon data to analyze")
Expand Down
29 changes: 14 additions & 15 deletions pkg/beaconfqdn/analyzer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,23 +9,22 @@ import (
"github.com/activecm/rita/config"
"github.com/activecm/rita/database"
"github.com/activecm/rita/pkg/data"
"github.com/activecm/rita/pkg/hostname"
"github.com/activecm/rita/util"
"github.com/globalsign/mgo/bson"
)

type (
analyzer struct {
tsMin int64 // min timestamp for the whole dataset
tsMax int64 // max timestamp for the whole dataset
chunk int //current chunk (0 if not on rolling analysis)
chunkStr string //current chunk (0 if not on rolling analysis)
db *database.DB // provides access to MongoDB
conf *config.Config // contains details needed to access MongoDB
analyzedCallback func(*update) // called on each analyzed result
closedCallback func() // called when .close() is called and no more calls to analyzedCallback will be made
analysisChannel chan *hostname.FqdnInput // holds unanalyzed data
analysisWg sync.WaitGroup // wait for analysis to finish
tsMin int64 // min timestamp for the whole dataset
tsMax int64 // max timestamp for the whole dataset
chunk int //current chunk (0 if not on rolling analysis)
chunkStr string //current chunk (0 if not on rolling analysis)
db *database.DB // provides access to MongoDB
conf *config.Config // contains details needed to access MongoDB
analyzedCallback func(*update) // called on each analyzed result
closedCallback func() // called when .close() is called and no more calls to analyzedCallback will be made
analysisChannel chan *fqdnInput // holds unanalyzed data
analysisWg sync.WaitGroup // wait for analysis to finish
}
)

Expand All @@ -40,12 +39,12 @@ func newAnalyzer(min int64, max int64, chunk int, db *database.DB, conf *config.
conf: conf,
analyzedCallback: analyzedCallback,
closedCallback: closedCallback,
analysisChannel: make(chan *hostname.FqdnInput),
analysisChannel: make(chan *fqdnInput),
}
}

//collect sends a chunk of data to be analyzed
func (a *analyzer) collect(data *hostname.FqdnInput) {
func (a *analyzer) collect(data *fqdnInput) {
a.analysisChannel <- data
}

Expand Down Expand Up @@ -85,7 +84,7 @@ func (a *analyzer) start() {
"avg_bytes": entry.TotalBytes / entry.ConnectionCount,
"connection_count": entry.ConnectionCount,
"src_network_name": entry.Src.SrcNetworkName,
"resolved_ips": entry.ResolvedIPs.Items(),
"resolved_ips": entry.ResolvedIPs,
"cid": a.chunk,
}

Expand Down Expand Up @@ -241,7 +240,7 @@ func (a *analyzer) start() {
"score": score,
"cid": a.chunk,
"src_network_name": entry.Src.SrcNetworkName,
"resolved_ips": entry.ResolvedIPs.Items(),
"resolved_ips": entry.ResolvedIPs,
"strobeFQDN": false,
}

Expand Down
23 changes: 11 additions & 12 deletions pkg/beaconfqdn/dissector.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,36 +6,35 @@ import (
"github.com/activecm/rita/config"
"github.com/activecm/rita/database"
"github.com/activecm/rita/pkg/data"
"github.com/activecm/rita/pkg/hostname"
"github.com/globalsign/mgo/bson"
)

type (
dissector struct {
connLimit int64 // limit for strobe classification
db *database.DB // provides access to MongoDB
conf *config.Config // contains details needed to access MongoDB
dissectedCallback func(*hostname.FqdnInput) // called on each analyzed result
closedCallback func() // called when .close() is called and no more calls to analyzedCallback will be made
dissectChannel chan *hostname.FqdnInput // holds unanalyzed data
dissectWg sync.WaitGroup // wait for analysis to finish
connLimit int64 // limit for strobe classification
db *database.DB // provides access to MongoDB
conf *config.Config // contains details needed to access MongoDB
dissectedCallback func(*fqdnInput) // called on each analyzed result
closedCallback func() // called when .close() is called and no more calls to analyzedCallback will be made
dissectChannel chan *fqdnInput // holds unanalyzed data
dissectWg sync.WaitGroup // wait for analysis to finish
}
)

//newdissector creates a new collector for gathering data
func newDissector(connLimit int64, db *database.DB, conf *config.Config, dissectedCallback func(*hostname.FqdnInput), closedCallback func()) *dissector {
func newDissector(connLimit int64, db *database.DB, conf *config.Config, dissectedCallback func(*fqdnInput), closedCallback func()) *dissector {
return &dissector{
connLimit: connLimit,
db: db,
conf: conf,
dissectedCallback: dissectedCallback,
closedCallback: closedCallback,
dissectChannel: make(chan *hostname.FqdnInput),
dissectChannel: make(chan *fqdnInput),
}
}

//collect sends a chunk of data to be analyzed
func (d *dissector) collect(entry *hostname.FqdnInput) {
func (d *dissector) collect(entry *fqdnInput) {
d.dissectChannel <- entry
}

Expand Down Expand Up @@ -267,7 +266,7 @@ func (d *dissector) start() {
for _, res := range allResults {

srcCurr := data.UniqueSrcIP{SrcIP: res.Src, SrcNetworkUUID: res.SrcNetworkUUID, SrcNetworkName: res.SrcNetworkName}
analysisInput := &hostname.FqdnInput{
analysisInput := &fqdnInput{
FQDN: entry.FQDN,
Src: srcCurr,
ConnectionCount: res.Count,
Expand Down
Loading

0 comments on commit 8baf8e5

Please sign in to comment.