feat(alerts): Allow 'fake' anomalies for testing (getsentry#76249)

In order to do end to end testing in production (and because I am not
able to get ngrok working to validate notification content changes)
we'll allow for a 'fake' anomaly to come through if the flag is turned
on so we can see these rules trigger.

src/sentry/features/temporary.py

@@ -55,6 +55,8 @@ def register_temporary_features(manager: FeatureManager):
     manager.add("organizations:alerts-migration-enabled", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=False)
     # Enable anomaly detection alerts
     manager.add("organizations:anomaly-detection-alerts", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=True)
+    # Enable anomaly detection alerts
+    manager.add("organizations:fake-anomaly-detection", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=False)
     # Enable anr frame analysis
     manager.add("organizations:anr-analyze-frames", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=True)
     # Enable auth provider configuration through api

src/sentry/incidents/subscription_processor.py

@@ -514,6 +514,9 @@ def process_update(self, subscription_update: QuerySubscriptionUpdate) -> None:
         self.has_anomaly_detection = features.has(
             "organizations:anomaly-detection-alerts", self.subscription.project.organization
         )
+        has_fake_anomalies = features.has(
+            "organizations:fake-anomaly-detection", self.subscription.project.organization
+        )
 
         if self.has_anomaly_detection:
             potential_anomalies = self.get_anomaly_data_from_seer(aggregation_value)
@@ -558,7 +561,7 @@ def process_update(self, subscription_update: QuerySubscriptionUpdate) -> None:
                                 continue
 
                         if self.has_anomaly(
-                            potential_anomaly, trigger.label
+                            potential_anomaly, trigger.label, has_fake_anomalies
                         ) and not self.check_trigger_matches_status(trigger, TriggerStatus.ACTIVE):
                             metrics.incr(
                                 "incidents.alert_rules.threshold.alert",
@@ -573,7 +576,9 @@ def process_update(self, subscription_update: QuerySubscriptionUpdate) -> None:
                             self.trigger_alert_counts[trigger.id] = 0
 
                         if (
-                            not self.has_anomaly(potential_anomaly, trigger.label)
+                            not self.has_anomaly(
+                                potential_anomaly, trigger.label, has_fake_anomalies
+                            )
                             and self.active_incident
                             and self.check_trigger_matches_status(trigger, TriggerStatus.ACTIVE)
                         ):
@@ -640,11 +645,14 @@ def process_update(self, subscription_update: QuerySubscriptionUpdate) -> None:
         # before the next one then we might alert twice.
         self.update_alert_rule_stats()
 
-    def has_anomaly(self, anomaly, label: str) -> bool:
+    def has_anomaly(self, anomaly, label: str, has_fake_anomalies: bool) -> bool:
         """
         Helper function to determine whether we care about an anomaly based on the
         anomaly type and trigger type.
         """
+        if has_fake_anomalies:
+            return True
+
         anomaly_type = anomaly.get("anomaly", {}).get("anomaly_type")
 
         if anomaly_type == AnomalyType.HIGH_CONFIDENCE.value or (

tests/sentry/incidents/test_subscription_processor.py

@@ -625,12 +625,23 @@ def test_has_anomaly(self):
         label = self.trigger.label
 
         processor = SubscriptionProcessor(self.sub)
-        assert processor.has_anomaly(anomaly1, label)
-        assert processor.has_anomaly(anomaly1, warning_label)
-        assert not processor.has_anomaly(anomaly2, label)
-        assert processor.has_anomaly(anomaly2, warning_label)
-        assert not processor.has_anomaly(not_anomaly, label)
-        assert not processor.has_anomaly(not_anomaly, warning_label)
+        assert processor.has_anomaly(anomaly1, label, False)
+        assert processor.has_anomaly(anomaly1, warning_label, False)
+        assert not processor.has_anomaly(anomaly2, label, False)
+        assert processor.has_anomaly(anomaly2, warning_label, False)
+        assert not processor.has_anomaly(not_anomaly, label, False)
+        assert not processor.has_anomaly(not_anomaly, warning_label, False)
+
+    def test_fake_anomaly(self):
+        anomaly = {
+            "anomaly": {"anomaly_score": 0.2, "anomaly_type": AnomalyType.NONE.value},
+            "timestamp": 1,
+            "value": 10,
+        }
+        label = self.trigger.label
+        processor = SubscriptionProcessor(self.sub)
+
+        assert processor.has_anomaly(anomaly, label, True)
 
     @with_feature("organizations:anomaly-detection-alerts")
     @mock.patch(
@@ -764,6 +775,44 @@ def test_enable_dynamic_alert_rule_and_fire(self, mock_seer_request):
             [(10, IncidentStatus.CRITICAL, mock.ANY)],
         )
 
+    @with_feature("organizations:anomaly-detection-alerts")
+    @with_feature("organizations:fake-anomaly-detection")
+    @mock.patch(
+        "sentry.incidents.subscription_processor.SubscriptionProcessor.seer_anomaly_detection_connection_pool.urlopen"
+    )
+    def test_fire_dynamic_alert_rule_fake_anomaly(self, mock_seer_request):
+        """
+        Test that we can fire a dynamic alert with a 'fake' anomaly for testing
+        """
+        rule = self.dynamic_rule
+        value = 10
+        seer_return_value = {
+            "anomalies": [
+                {
+                    "anomaly": {
+                        "anomaly_score": 0.0,
+                        "anomaly_type": AnomalyType.LOW_CONFIDENCE.value,
+                    },
+                    "timestamp": 1,
+                    "value": value,
+                }
+            ]
+        }
+        mock_seer_request.return_value = HTTPResponse(orjson.dumps(seer_return_value), status=200)
+        processor = self.send_update(rule, value)
+
+        rule.refresh_from_db()
+
+        assert mock_seer_request.call_count == 1
+        self.assert_trigger_counts(processor, self.trigger, 0, 0)
+        incident = self.assert_active_incident(rule)
+        self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
+        self.assert_actions_fired_for_incident(
+            incident,
+            [self.action],
+            [(value, IncidentStatus.CRITICAL, mock.ANY)],
+        )
+
     @with_feature("organizations:anomaly-detection-alerts")
     @mock.patch(
         "sentry.incidents.subscription_processor.SubscriptionProcessor.seer_anomaly_detection_connection_pool.urlopen"