Update to bimodal portion of the histogram score (activecm#794)

* updated bimodal portion of the histogram score to be jitter tolerant

* linter issue

* requested changes, added hardcoded values as yaml file

* division by zero contingency

---------

Co-authored-by: Logan Lembke <[email protected]>

config/static.go

@@ -101,31 +101,46 @@ type (
 
 	//BeaconStaticCfg is used to control the beaconing analysis module
 	BeaconStaticCfg struct {
-		Enabled                 bool    `yaml:"Enabled" default:"true"`
-		DefaultConnectionThresh int     `yaml:"DefaultConnectionThresh" default:"23"`
-		TsWeight                float64 `yaml:"TimestampScoreWeight" default:"0.25"`
-		DsWeight                float64 `yaml:"DatasizeScoreWeight" default:"0.25"`
-		DurWeight               float64 `yaml:"DurationScoreWeight" default:"0.25"`
-		HistWeight              float64 `yaml:"HistogramScoreWeight" default:"0.25"`
+		Enabled                      bool    `yaml:"Enabled" default:"true"`
+		DefaultConnectionThresh      int     `yaml:"DefaultConnectionThresh" default:"23"`
+		TsWeight                     float64 `yaml:"TimestampScoreWeight" default:"0.25"`
+		DsWeight                     float64 `yaml:"DatasizeScoreWeight" default:"0.25"`
+		DurWeight                    float64 `yaml:"DurationScoreWeight" default:"0.25"`
+		HistWeight                   float64 `yaml:"HistogramScoreWeight" default:"0.25"`
+		DurMinHoursSeen              int     `yaml:"DurationMinHoursSeen" default:"6"`
+		DurConsistencyIdealHoursSeen int     `yaml:"DurationConsistencyIdealHoursSeen" default:"12"`
+		HistBimodalBucketSize        float64 `yaml:"HistogramBimodalBucketSize" default:"0.05"`
+		HistBimodalOutlierRemoval    int     `yaml:"HistogramBimodalOutlierRemoval" default:"1"`
+		HistBimodalMinHoursSeen      int     `yaml:"HistogramBimodalMinHoursSeen" default:"11"`
 	}
 
 	//BeaconProxyStaticCfg is used to control the proxy beaconing analysis module
 	BeaconProxyStaticCfg struct {
-		Enabled                 bool    `yaml:"Enabled" default:"true"`
-		DefaultConnectionThresh int     `yaml:"DefaultConnectionThresh" default:"23"`
-		TsWeight                float64 `yaml:"TimestampScoreWeight" default:"0.333"`
-		DurWeight               float64 `yaml:"DurationScoreWeight" default:"0.333"`
-		HistWeight              float64 `yaml:"HistogramScoreWeight" default:"0.333"`
+		Enabled                      bool    `yaml:"Enabled" default:"true"`
+		DefaultConnectionThresh      int     `yaml:"DefaultConnectionThresh" default:"23"`
+		TsWeight                     float64 `yaml:"TimestampScoreWeight" default:"0.333"`
+		DurWeight                    float64 `yaml:"DurationScoreWeight" default:"0.333"`
+		HistWeight                   float64 `yaml:"HistogramScoreWeight" default:"0.333"`
+		DurMinHoursSeen              int     `yaml:"DurationMinHoursSeen" default:"6"`
+		DurConsistencyIdealHoursSeen int     `yaml:"DurationConsistencyIdealHoursSeen" default:"12"`
+		HistBimodalBucketSize        float64 `yaml:"HistogramBimodalBucketSize" default:"0.05"`
+		HistBimodalOutlierRemoval    int     `yaml:"HistogramBimodalOutlierRemoval" default:"1"`
+		HistBimodalMinHoursSeen      int     `yaml:"HistogramBimodalMinHoursSeen" default:"11"`
 	}
 
 	//BeaconSNIStaticCfg is used to control the SNI beaconing analysis module
 	BeaconSNIStaticCfg struct {
-		Enabled                 bool    `yaml:"Enabled" default:"true"`
-		DefaultConnectionThresh int     `yaml:"DefaultConnectionThresh" default:"23"`
-		TsWeight                float64 `yaml:"TimestampScoreWeight" default:"0.25"`
-		DsWeight                float64 `yaml:"DatasizeScoreWeight" default:"0.25"`
-		DurWeight               float64 `yaml:"DurationScoreWeight" default:"0.25"`
-		HistWeight              float64 `yaml:"HistogramScoreWeight" default:"0.25"`
+		Enabled                      bool    `yaml:"Enabled" default:"true"`
+		DefaultConnectionThresh      int     `yaml:"DefaultConnectionThresh" default:"23"`
+		TsWeight                     float64 `yaml:"TimestampScoreWeight" default:"0.25"`
+		DsWeight                     float64 `yaml:"DatasizeScoreWeight" default:"0.25"`
+		DurWeight                    float64 `yaml:"DurationScoreWeight" default:"0.25"`
+		HistWeight                   float64 `yaml:"HistogramScoreWeight" default:"0.25"`
+		DurMinHoursSeen              int     `yaml:"DurationMinHoursSeen" default:"6"`
+		DurConsistencyIdealHoursSeen int     `yaml:"DurationConsistencyIdealHoursSeen" default:"12"`
+		HistBimodalBucketSize        float64 `yaml:"HistogramBimodalBucketSize" default:"0.05"`
+		HistBimodalOutlierRemoval    int     `yaml:"HistogramBimodalOutlierRemoval" default:"1"`
+		HistBimodalMinHoursSeen      int     `yaml:"HistogramBimodalMinHoursSeen" default:"11"`
 	}
 
 	//DNSStaticCfg is used to control the DNS analysis module
@@ -206,6 +221,17 @@ func parseStaticConfig(cfgFile []byte, config *StaticCfg) error {
 		config.BeaconProxy.DefaultConnectionThresh = minBeaconConnectionThreshLimit
 	}
 
+	// make sure value is above zero to avoid division by zero
+	if config.Beacon.DurConsistencyIdealHoursSeen < 1 {
+		config.Beacon.DurConsistencyIdealHoursSeen = 1
+	}
+	if config.BeaconProxy.DurConsistencyIdealHoursSeen < 1 {
+		config.BeaconProxy.DurConsistencyIdealHoursSeen = 1
+	}
+	if config.BeaconSNI.DurConsistencyIdealHoursSeen < 1 {
+		config.BeaconSNI.DurConsistencyIdealHoursSeen = 1
+	}
+
 	// expand env variables, config is a pointer
 	// so we have to call elem on the reflect value
 	expandConfig(reflect.ValueOf(config).Elem())

config/static_test.go

@@ -38,19 +38,34 @@ Beacon:
     DatasizeScoreWeight: 0.25
     DurationScoreWeight: 0.25
     HistogramScoreWeight: 0.25
+    DurationMinHoursSeen: 6
+    DurationConsistencyIdealHoursSeen: 12
+    HistogramBimodalBucketSize: 0.05
+    HistogramBimodalOutlierRemoval: 1
+    HistogramBimodalMinHoursSeen: 11
 BeaconSNI:
     Enabled: true
     DefaultConnectionThresh: 5
     TimestampScoreWeight: 0.25
     DatasizeScoreWeight: 0.25
     DurationScoreWeight: 0.25
     HistogramScoreWeight: 0.25
+    DurationMinHoursSeen: 6
+    DurationConsistencyIdealHoursSeen: 12
+    HistogramBimodalBucketSize: 0.05
+    HistogramBimodalOutlierRemoval: 1
+    HistogramBimodalMinHoursSeen: 11
 BeaconProxy:
     Enabled: true
     DefaultConnectionThresh: 5
     TimestampScoreWeight: 0.333
     DurationScoreWeight: 0.333
     HistogramScoreWeight: 0.333
+    DurationMinHoursSeen: 6
+    DurationConsistencyIdealHoursSeen: 12
+    HistogramBimodalBucketSize: 0.05
+    HistogramBimodalOutlierRemoval: 1
+    HistogramBimodalMinHoursSeen: 11
 Strobe:
     ConnectionLimit: 250000
 Filtering:
@@ -93,27 +108,42 @@ var testConfigFullExp = StaticCfg{
 		Enabled: true,
 	},
 	Beacon: BeaconStaticCfg{
-		Enabled:                 true,
-		DefaultConnectionThresh: minBeaconConnectionThreshLimit,
-		TsWeight:                0.25,
-		DsWeight:                0.25,
-		DurWeight:               0.25,
-		HistWeight:              0.25,
+		Enabled:                      true,
+		DefaultConnectionThresh:      minBeaconConnectionThreshLimit,
+		TsWeight:                     0.25,
+		DsWeight:                     0.25,
+		DurWeight:                    0.25,
+		HistWeight:                   0.25,
+		DurMinHoursSeen:              6,
+		DurConsistencyIdealHoursSeen: 12,
+		HistBimodalBucketSize:        0.05,
+		HistBimodalOutlierRemoval:    1,
+		HistBimodalMinHoursSeen:      11,
 	},
 	BeaconSNI: BeaconSNIStaticCfg{
-		Enabled:                 true,
-		DefaultConnectionThresh: minBeaconConnectionThreshLimit,
-		TsWeight:                0.25,
-		DsWeight:                0.25,
-		DurWeight:               0.25,
-		HistWeight:              0.25,
+		Enabled:                      true,
+		DefaultConnectionThresh:      minBeaconConnectionThreshLimit,
+		TsWeight:                     0.25,
+		DsWeight:                     0.25,
+		DurWeight:                    0.25,
+		HistWeight:                   0.25,
+		DurMinHoursSeen:              6,
+		DurConsistencyIdealHoursSeen: 12,
+		HistBimodalBucketSize:        0.05,
+		HistBimodalOutlierRemoval:    1,
+		HistBimodalMinHoursSeen:      11,
 	},
 	BeaconProxy: BeaconProxyStaticCfg{
-		Enabled:                 true,
-		DefaultConnectionThresh: minBeaconConnectionThreshLimit,
-		TsWeight:                0.333,
-		DurWeight:               0.333,
-		HistWeight:              0.333,
+		Enabled:                      true,
+		DefaultConnectionThresh:      minBeaconConnectionThreshLimit,
+		TsWeight:                     0.333,
+		DurWeight:                    0.333,
+		HistWeight:                   0.333,
+		DurMinHoursSeen:              6,
+		DurConsistencyIdealHoursSeen: 12,
+		HistBimodalBucketSize:        0.05,
+		HistBimodalOutlierRemoval:    1,
+		HistBimodalMinHoursSeen:      11,
 	},
 	Strobe: StrobeStaticCfg{
 		ConnectionLimit: maxStrobeConnectionLimit,

config/testing.go

@@ -36,19 +36,34 @@ Beacon:
     DatasizeScoreWeight: 0.25
     DurationScoreWeight: 0.25
     HistogramScoreWeight: 0.25
+    DurationMinHoursSeen: 6
+    DurationConsistencyIdealHoursSeen: 12
+    HistogramBimodalBucketSize: 0.05
+    HistogramBimodalOutlierRemoval: 1
+    HistogramBimodalMinHoursSeen: 11
 BeaconSNI:
     Enabled: true
     DefaultConnectionThresh: 23
     TimestampScoreWeight: 0.25
     DatasizeScoreWeight: 0.25
     DurationScoreWeight: 0.25
     HistogramScoreWeight: 0.25
+    DurationMinHoursSeen: 6
+    DurationConsistencyIdealHoursSeen: 12
+    HistogramBimodalBucketSize: 0.05
+    HistogramBimodalOutlierRemoval: 1
+    HistogramBimodalMinHoursSeen: 11
 BeaconProxy:
     Enabled: true
     DefaultConnectionThresh: 23
     TimestampScoreWeight: 0.333
     DurationScoreWeight: 0.333
     HistogramScoreWeight: 0.333
+    DurationMinHoursSeen: 6
+    DurationConsistencyIdealHoursSeen: 12
+    HistogramBimodalBucketSize: 0.05
+    HistogramBimodalOutlierRemoval: 1
+    HistogramBimodalMinHoursSeen: 11
 Strobe:
     ConnectionLimit: 250000
 Filtering:

etc/rita.yaml

@@ -149,6 +149,33 @@ Beacon:
   DurationScoreWeight: 0.25
   HistogramScoreWeight: 0.25
 
+  # The number of hours seen in a connection graph representation of a beacon must
+  # be greater than this threshold for an overall duration score to be calculated.
+  # Default value: 6
+  DurationMinHoursSeen: 6
+  # This is the minimum number of hours seen in a connection graph representation
+  # of a beacon for the consistency subscore of duration to score at 100%
+  # Default value: 12 (half the day)
+  DurationConsistencyIdealHoursSeen: 12
+
+  # The histogram score has a subscore that attempts to detect multiple 
+  # flat sections in a connection graph representation of a beacon. The 
+  # variable below controls the bucket size for grouping connections. This
+  # is expressed as a percentage of the largest connection count. For example, 
+  # if the max connection count is 400 and this variable is set to 0.05 (5%), 
+  # the bucket size will be 20 (400*0.05=20). As you make this variable 
+  # larger, the algorithm becomes more forgiving to variation. 
+  # Default value 0.05
+  HistogramBimodalBucketSize: 0.05
+  # This is the number of buckets that can be considered outliers and dropped
+  # from the calculation.
+  # Default value: 1
+  HistogramBimodalOutlierRemoval: 1
+  # This is the minimum number of hours seen in a connection graph representation
+  # of a beacon before the bimodal subscore score is used.
+  # Default value: 11 (sets the minimum coverage to just below half of the day)
+  HistogramBimodalMinHoursSeen: 11
+
 BeaconSNI:
   Enabled: true
   # The default minimum number of connections used for beacons SNI analysis.
@@ -170,6 +197,33 @@ BeaconSNI:
   DurationScoreWeight: 0.25
   HistogramScoreWeight: 0.25
 
+  # The number of hours seen in a connection graph representation of a beacon must
+  # be greater than this threshold for an overall duration score to be calculated.
+  # Default value: 6
+  DurationMinHoursSeen: 6
+  # This is the minimum number of hours seen in a connection graph representation
+  # of a beacon for the consistency subscore of duration to score at 100%
+  # Default value: 12 (half the day)
+  DurationConsistencyIdealHoursSeen: 12
+
+  # The histogram score has a subscore that attempts to detect multiple 
+  # flat sections in a connection graph representation of a beacon. The 
+  # variable below controls the bucket size for grouping connections. This
+  # is expressed as a percentage of the largest connection count. For example, 
+  # if the max connection count is 400 and this variable is set to 0.05 (5%), 
+  # the bucket size will be 20 (400*0.05=20). As you make this variable 
+  # larger, the algorithm becomes more forgiving to variation. 
+  # Default value 0.05
+  HistogramBimodalBucketSize: 0.05
+  # This is the number of buckets that can be considered outliers and dropped
+  # from the calculation.
+  # Default value: 1
+  HistogramBimodalOutlierRemoval: 1
+  # This is the minimum number of hours seen in a connection graph representation
+  # of a beacon before the subscore score is used.
+  # Default value: 11 (sets the minimum coverage to just below half of the day)
+  HistogramBimodalMinHoursSeen: 11
+
 BeaconProxy:
   Enabled: true
   # The default minimum number of connections used for beacons proxy analysis.
@@ -189,6 +243,33 @@ BeaconProxy:
   TimestampScoreWeight: 0.333
   DurationScoreWeight: 0.333
   HistogramScoreWeight: 0.333
+
+  # The number of hours seen in a connection graph representation of a beacon must
+  # be greater than this threshold for an overall duration score to be calculated.
+  # Default value: 6
+  DurationMinHoursSeen: 6
+  # This is the minimum number of hours seen in a connection graph representation
+  # of a beacon for the consistency subscore of duration to score at 100%
+  # Default value: 12 (half the day)
+  DurationConsistencyIdealHoursSeen: 12
+
+  # The histogram score has a subscore that attempts to detect multiple 
+  # flat sections in a connection graph representation of a beacon. The 
+  # variable below controls the bucket size for grouping connections. This
+  # is expressed as a percentage of the largest connection count. For example, 
+  # if the max connection count is 400 and this variable is set to 0.05 (5%), 
+  # the bucket size will be 20 (400*0.05=20). As you make this variable 
+  # larger, the algorithm becomes more forgiving to variation. 
+  # Default value 0.05
+  HistogramBimodalBucketSize: 0.05
+  # This is the number of buckets that can be considered outliers and dropped
+  # from the calculation.
+  # Default value: 1
+  HistogramBimodalOutlierRemoval: 1
+  # This is the minimum number of hours seen in a connection graph representation
+  # of a beacon before the subscore score is used.
+  # Default value: 11 (sets the minimum coverage to just below half of the day)
+  HistogramBimodalMinHoursSeen: 11
 
 DNS:
   Enabled: true