prom-rules-tests.yaml

---
rule_files:
  - /tmp/rules.verify

group_eval_order:
  - kubevirt.rules
#information about this format can be found in: https://prometheus.io/docs/prometheus/latest/configuration/unit_testing_rules/
tests:
  # Pod is using more CPU than expected
  - interval: 1m
    input_series:
      - series: 'node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{namespace="ci",pod="virt-controller-8546c99968-x9jgg",node="node1"}'
        values: '2+0x10'
      - series: 'kube_pod_container_resource_requests{namespace="ci",container="virt-controller",resource="cpu",pod="virt-controller-8546c99968-x9jgg",node="node1"}'
        values: '0+0x6'

    alert_rule_test:
      - eval_time: 5m
        alertname: KubeVirtComponentExceedsRequestedCPU
        exp_alerts:
          - exp_annotations:
              description: "Container virt-controller in pod virt-controller-8546c99968-x9jgg cpu usage exceeds the CPU requested"
              summary: "The container is using more CPU than what is defined in the containers resource requests"
              runbook_url: "https://kubevirt.io/monitoring/runbooks/KubeVirtComponentExceedsRequestedCPU"
            exp_labels:
              severity: "warning"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"
              pod: "virt-controller-8546c99968-x9jgg"
              container: "virt-controller"
              namespace: ci
              node: node1
              resource: cpu

  # Pod is using more memory than expected
  - interval: 1m
    input_series:
      - series: 'container_memory_working_set_bytes{namespace="ci",container="",pod="virt-controller-8546c99968-x9jgg",node="node1"}'
        values: "157286400 157286400 157286400 157286400 157286400 157286400 157286400 157286400"
      - series: 'kube_pod_container_resource_requests{namespace="ci",container="virt-controller",resource="memory",pod="virt-controller-8546c99968-x9jgg",node="node1"}'
        values: "118325248 118325248 118325248 118325248 118325248 118325248 118325248 118325248"

    alert_rule_test:
      - eval_time: 5m
        alertname: KubeVirtComponentExceedsRequestedMemory
        exp_alerts:
          - exp_annotations:
              description: "Container virt-controller in pod virt-controller-8546c99968-x9jgg memory usage exceeds the memory requested"
              summary: "The container is using more memory than what is defined in the containers resource requests"
              runbook_url: "https://kubevirt.io/monitoring/runbooks/KubeVirtComponentExceedsRequestedMemory"
            exp_labels:
              severity: "warning"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"
              namespace: ci
              node: "node1"
              pod: "virt-controller-8546c99968-x9jgg"
              resource: "memory"
              container: virt-controller

  # Alerts to test whether our operators are up or not
  - interval: 1m
    input_series:
      - series: 'up{namespace="ci", pod="virt-api-1"}'
        values: "_ _ _ _ _ _ _ _ _ _ _ 0 0 0 0 0 0 1"
      - series: 'up{namespace="ci", pod="virt-controller-1"}'
        values: "_ _ _ _ _ _ _ _ _ _ _ 0 0 0 0 0 0 1"
      - series: 'up{namespace="ci", pod="virt-operator-1"}'
        values: "_ _ _ _ _ _ _ _ _ _ _ 0 0 0 0 0 0 1"

    alert_rule_test:
      # it must not trigger before 10m
      - eval_time: 8m
        alertname: VirtAPIDown
        exp_alerts: []
      - eval_time: 8m
        alertname: VirtControllerDown
        exp_alerts: [ ]
      - eval_time: 8m
        alertname: VirtOperatorDown
        exp_alerts: [ ]
      # it must trigger when there is no data
      - eval_time: 10m
        alertname: VirtAPIDown
        exp_alerts:
          - exp_annotations:
              summary: "All virt-api servers are down."
              runbook_url: "https://kubevirt.io/monitoring/runbooks/VirtAPIDown"
            exp_labels:
              severity: "critical"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"
      - eval_time: 10m
        alertname: VirtControllerDown
        exp_alerts:
          - exp_annotations:
              summary: "No running virt-controller was detected for the last 10 min."
              runbook_url: "https://kubevirt.io/monitoring/runbooks/VirtControllerDown"
            exp_labels:
              severity: "critical"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"
      - eval_time: 10m
        alertname: VirtOperatorDown
        exp_alerts:
          - exp_annotations:
              summary: "All virt-operator servers are down."
              runbook_url: "https://kubevirt.io/monitoring/runbooks/VirtOperatorDown"
            exp_labels:
              severity: "critical"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"
      # it must trigger when operators are not healthy
      - eval_time: 16m
        alertname: VirtAPIDown
        exp_alerts:
          - exp_annotations:
              summary: "All virt-api servers are down."
              runbook_url: "https://kubevirt.io/monitoring/runbooks/VirtAPIDown"
            exp_labels:
              severity: "critical"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"
      - eval_time: 16m
        alertname: VirtControllerDown
        exp_alerts:
          - exp_annotations:
              summary: "No running virt-controller was detected for the last 10 min."
              runbook_url: "https://kubevirt.io/monitoring/runbooks/VirtControllerDown"
            exp_labels:
              severity: "critical"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"
      - eval_time: 16m
        alertname: VirtOperatorDown
        exp_alerts:
          - exp_annotations:
              summary: "All virt-operator servers are down."
              runbook_url: "https://kubevirt.io/monitoring/runbooks/VirtOperatorDown"
            exp_labels:
              severity: "critical"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"
      # it must not trigger when operators are healthy
      - eval_time: 17m
        alertname: VirtAPIDown
        exp_alerts: []
      - eval_time: 17m
        alertname: VirtControllerDown
        exp_alerts: [ ]
      - eval_time: 17m
        alertname: VirtOperatorDown
        exp_alerts: [ ]


    # vmi running on a node with an unready virt-handler pod
  - interval: 1m
    input_series:
      - series: 'kube_pod_info{pod="virt-launcher-testvm-123", node="node01"}'
        values: '1 1 1 1 1 1 1 1 1 1 1'
      - series: 'kube_pod_status_ready{pod="virt-handler-asdf", condition="true"}'
        values: '_ _ _ _ _ _ _ _ _ _ _'
      - series: 'kube_pod_status_ready{pod="virt-handler-asdf", condition="false"}'
        values: '1 1 1 1 1 1 1 1 1 1 1'
      - series: 'kube_pod_info{pod="virt-handler-asdf", node="node01"}'
        values: '1 1 1 1 1 1 1 1 1 1 1'
      - series: 'kube_pod_info{pod="virt-launcher-vmi", node="node02"}'
        values: '1 1 1 1 1 1 1 1 1 1 1'
      - series: 'kube_pod_status_ready{pod="virt-handler-asdfg", condition="true"}'
        values: '1 1 1 1 1 1 1 1 1 1 1'
      - series: 'kube_pod_info{pod="virt-handler-asdfg", node="node02"}'
        values: '1 1 1 1 1 1 1 1 1 1 1'


    alert_rule_test:
      # no alert before 10 minutes
      - eval_time: 9m
        alertname: OrphanedVirtualMachineInstances
        exp_alerts: [ ]
      - eval_time: 10m
        alertname: OrphanedVirtualMachineInstances
        exp_alerts:
          - exp_annotations:
              summary: "No ready virt-handler pod detected on node node01 with running vmis for more than 10 minutes"
              runbook_url: "https://kubevirt.io/monitoring/runbooks/OrphanedVirtualMachineInstances"
            exp_labels:
              node: "node01"
              severity: "warning"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"


    # vmi running on a node without a virt-handler pod
  - interval: 1m
    input_series:
      - series: 'kube_pod_info{pod="virt-launcher-testvm-123", node="node01"}'
        values: '1 1 1 1 1 1 1 1 1 1 1'
      - series: 'kube_pod_status_ready{pod="virt-handler-asdf", condition="true"}'
        values: '_ _ _ _ _ _ _ _ _ _ _'
      - series: 'kube_pod_info{pod="virt-handler-asdf", node="node01"}'
        values: '_ _ _ _ _ _ _ _ _ _ _'
      - series: 'kube_pod_info{pod="virt-launcher-vmi", node="node02"}'
        values: '1 1 1 1 1 1 1 1 1 1 1'
      - series: 'kube_pod_status_ready{pod="virt-handler-asdfg", condition="true"}'
        values: '1 1 1 1 1 1 1 1 1 1 1'
      - series: 'kube_pod_info{pod="virt-handler-asdfg", node="node02"}'
        values: '1 1 1 1 1 1 1 1 1 1 1'

    alert_rule_test:
      # no alert before 10 minutes
      - eval_time: 9m
        alertname: OrphanedVirtualMachineInstances
        exp_alerts: [ ]
      - eval_time: 10m
        alertname: OrphanedVirtualMachineInstances
        exp_alerts:
          - exp_annotations:
              summary: "No ready virt-handler pod detected on node node01 with running vmis for more than 10 minutes"
              runbook_url: "https://kubevirt.io/monitoring/runbooks/OrphanedVirtualMachineInstances"
            exp_labels:
              node: "node01"
              severity: "warning"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"


  # Some virt controllers are not ready
  - interval: 1m
    input_series:
      - series: 'kubevirt_virt_controller_ready{namespace="ci", pod="virt-controller-1"}'
        values: '1+0x11'
      - series: 'kubevirt_virt_controller_ready{namespace="ci", pod="virt-controller-2"}'
        values: '0+0x11'
      - series: 'up{namespace="ci", pod="virt-controller-1"}'
        values: '1+0x11'
      - series: 'up{namespace="ci", pod="virt-controller-2"}'
        values: '1+0x11'

    alert_rule_test:
      - eval_time: 10m
        alertname: LowReadyVirtControllersCount
        exp_alerts:
          - exp_annotations:
              summary: "Some virt controllers are running but not ready."
              runbook_url: "https://kubevirt.io/monitoring/runbooks/LowReadyVirtControllersCount"
            exp_labels:
              severity: "warning"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"

  # All virt controllers are not ready
  - interval: 1m
    input_series:
      - series: 'kubevirt_virt_controller_ready{namespace="ci", pod="virt-controller-1"}'
        values: "0 0 0 0 0 0 0 0 0 0 0"

    alert_rule_test:
      # no alert before 10 minutes
      - eval_time: 9m
        alertname: NoReadyVirtController
        exp_alerts: [ ]
      - eval_time: 10m
        alertname: NoReadyVirtController
        exp_alerts:
          - exp_annotations:
              summary: "No ready virt-controller was detected for the last 10 min."
              runbook_url: "https://kubevirt.io/monitoring/runbooks/NoReadyVirtController"
            exp_labels:
              severity: "critical"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"
  # All virt controllers are not ready (ImagePullBackOff)
  - interval: 1m
    input_series:
      - series: 'kubevirt_virt_controller_ready{namespace="ci", pod="virt-controller-1"}'
        values: "stale stale stale stale stale stale stale stale stale stale"

    alert_rule_test:
      # no alert before 10 minutes
      - eval_time: 9m
        alertname: NoReadyVirtController
        exp_alerts: [ ]
      - eval_time: 10m
        alertname: NoReadyVirtController
        exp_alerts:
          - exp_annotations:
              summary: "No ready virt-controller was detected for the last 10 min."
              runbook_url: "https://kubevirt.io/monitoring/runbooks/NoReadyVirtController"
            exp_labels:
              severity: "critical"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"

  # All virt operators are not ready (ImagePullBackOff)
  - interval: 1m
    input_series:
      - series: 'kubevirt_virt_operator_ready{namespace="ci", pod="virt-operator-1"}'
        values: "stale stale stale stale stale stale stale stale stale stale"

    alert_rule_test:
      # no alert before 10 minutes
      - eval_time: 9m
        alertname: NoReadyVirtOperator
        exp_alerts: [ ]
      - eval_time: 10m
        alertname: NoReadyVirtOperator
        exp_alerts:
          - exp_annotations:
              summary: "No ready virt-operator was detected for the last 10 min."
              runbook_url: "https://kubevirt.io/monitoring/runbooks/NoReadyVirtOperator"
            exp_labels:
              severity: "critical"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"

  # All virt operators are not ready
  - interval: 1m
    input_series:
      - series: 'kubevirt_virt_operator_ready{namespace="ci", pod="virt-operator-1"}'
        values: "0 0 0 0 0 0 0 0 0 0 0"

    alert_rule_test:
      # no alert before 10 minutes
      - eval_time: 9m
        alertname: NoReadyVirtOperator
        exp_alerts: [ ]
      - eval_time: 10m
        alertname: NoReadyVirtOperator
        exp_alerts:
          - exp_annotations:
              summary: "No ready virt-operator was detected for the last 10 min."
              runbook_url: "https://kubevirt.io/monitoring/runbooks/NoReadyVirtOperator"
            exp_labels:
              severity: "critical"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"

  # Burst REST errors
  # values: '0+10x20' == values: "0 10 20 30 40 ... 190"
  # values: '0+100x20' == values :"0 100 200 .... 1900"
  # so now for EACH POD the total requests should be 10+20+30+..+190+100+200+300+...+1900
  # and the number of requests with error code should be 190+100+200+300+...+1900 which is more than 80% of the total requests
  # in each 5 minutes interval of the test and the error condition(more than 80% of the requests has error code) is true for
  # more than 5 minutes(because each test run for 20 minutes) which should fire an alert for EACH POD
  # values : 0+100x15  0+100x5  ==  :"0 100 200 .... 1400  0 100 200 300 400"  we should treat values : `0+100x20` and
  # values : `0+100x15  0+100x5`  the same way because prometheus counters might reset
  - interval: 1m
    input_series:
      - series: 'rest_client_requests_total{namespace="ci", pod="virt-controller-1", code="200"}'
        values: '0+10x20'
      - series: 'rest_client_requests_total{namespace="ci", pod="virt-controller-1", code="400"}'
        values: '0+100x15  0+100x5'
      - series: 'rest_client_requests_total{namespace="ci", pod="virt-operator-1", code="200"}'
        values: '0+10x20'
      - series: 'rest_client_requests_total{namespace="ci", pod="virt-operator-1", code="400"}'
        values: '0+100x15  0+100x5'
      - series: 'rest_client_requests_total{namespace="ci", pod="virt-handler-1", code="200"}'
        values: '0+10x20'
      - series: 'rest_client_requests_total{namespace="ci", pod="virt-handler-1", code="500"}'
        values: '0+100x15  0+100x5'
      - series: 'rest_client_requests_total{namespace="ci", pod="virt-api-1", code="200"}'
        values: '0+10x20'
      - series: 'rest_client_requests_total{namespace="ci", pod="virt-api-1", code="500"}'
        values: '0+100x15  0+100x5'

    alert_rule_test:
      - eval_time: 20m
        alertname: VirtControllerRESTErrorsBurst
        exp_alerts:
          - exp_annotations:
              summary: "More than 80% of the rest calls failed in virt-controller for the last 5 minutes"
              runbook_url: "https://kubevirt.io/monitoring/runbooks/VirtControllerRESTErrorsBurst"
            exp_labels:
              severity: "critical"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"
      - eval_time: 20m
        alertname: VirtOperatorRESTErrorsBurst
        exp_alerts:
          - exp_annotations:
              summary: "More than 80% of the rest calls failed in virt-operator for the last 5 minutes"
              runbook_url: "https://kubevirt.io/monitoring/runbooks/VirtOperatorRESTErrorsBurst"
            exp_labels:
              severity: "critical"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"
      - eval_time: 20m
        alertname: VirtHandlerRESTErrorsBurst
        exp_alerts:
          - exp_annotations:
              summary: "More than 80% of the rest calls failed in virt-handler for the last 5 minutes"
              runbook_url: "https://kubevirt.io/monitoring/runbooks/VirtHandlerRESTErrorsBurst"
            exp_labels:
              severity: "critical"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"
      - eval_time: 20m
        alertname: VirtApiRESTErrorsBurst
        exp_alerts:
          - exp_annotations:
              summary: "More than 80% of the rest calls failed in virt-api for the last 5 minutes"
              runbook_url: "https://kubevirt.io/monitoring/runbooks/VirtApiRESTErrorsBurst"
            exp_labels:
              severity: "critical"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"

  # values: '0+10x100' == values: "0 10 20 30 40 ... 990"
  # values: '0+5x100' == values :"0 5 10 .... 495"
  # so now for EACH POD the total requests should be 10+20+30+..+990+5+10+15+...+495
  # and the number of requests with error code should be 5+10+15+...+495 which is more than 5% of the total requests
  # the error condition(more than 5% of the requests has error code) is true which should fire an alert for EACH POD
  # High REST errors
  # values : '0+5x90 0+5x10'  ==  :"0 5 10 .... 445  0 5 10 ... 45"  we should treat values : '0+5x100' and
  # values : '0+5x90 0+5x10'  the same way because prometheus counters might reset
  - interval: 1m
    input_series:
      - series: 'rest_client_requests_total{namespace="ci", pod="virt-controller-1", code="200"}'
        values: '0+10x100'
      - series: 'rest_client_requests_total{namespace="ci", pod="virt-controller-1", code="400"}'
        values: '0+5x90 0+5x10'
      - series: 'rest_client_requests_total{namespace="ci", pod="virt-operator-1", code="200"}'
        values: '0+10x100'
      - series: 'rest_client_requests_total{namespace="ci", pod="virt-operator-1", code="400"}'
        values: '0+5x90 0+5x10'
      - series: 'rest_client_requests_total{namespace="ci", pod="virt-handler-1", code="200"}'
        values: '0+10x100'
      - series: 'rest_client_requests_total{namespace="ci", pod="virt-handler-1", code="500"}'
        values: '0+5x90 0+5x10'
      - series: 'rest_client_requests_total{namespace="ci", pod="virt-api-1", code="200"}'
        values: '0+10x100'
      - series: 'rest_client_requests_total{namespace="ci", pod="virt-api-1", code="500"}'
        values: '0+5x90 0+5x10'

    alert_rule_test:
      - eval_time: 100m
        alertname: VirtControllerRESTErrorsHigh
        exp_alerts:
          - exp_annotations:
              summary: "More than 5% of the rest calls failed in virt-controller for the last hour"
              runbook_url: "https://kubevirt.io/monitoring/runbooks/VirtControllerRESTErrorsHigh"
            exp_labels:
              severity: "warning"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"
      - eval_time: 100m
        alertname: VirtOperatorRESTErrorsHigh
        exp_alerts:
          - exp_annotations:
              summary: "More than 5% of the rest calls failed in virt-operator for the last hour"
              runbook_url: "https://kubevirt.io/monitoring/runbooks/VirtOperatorRESTErrorsHigh"
            exp_labels:
              severity: "warning"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"
      - eval_time: 100m
        alertname: VirtHandlerRESTErrorsHigh
        exp_alerts:
          - exp_annotations:
              summary: "More than 5% of the rest calls failed in virt-handler for the last hour"
              runbook_url: "https://kubevirt.io/monitoring/runbooks/VirtHandlerRESTErrorsHigh"
            exp_labels:
              severity: "warning"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"
      - eval_time: 100m
        alertname: VirtApiRESTErrorsHigh
        exp_alerts:
          - exp_annotations:
              summary: "More than 5% of the rest calls failed in virt-api for the last hour"
              runbook_url: "https://kubevirt.io/monitoring/runbooks/VirtApiRESTErrorsHigh"
            exp_labels:
              severity: "warning"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"

  # Some nodes without KVM resources
  - interval: 1m
    input_series:
      - series: 'kube_node_status_allocatable{resource="devices_kubevirt_io_kvm", node ="node1"}'
        values: "110 110 110 110 110 110"
      - series: 'kube_node_status_allocatable{resource="devices_kubevirt_io_kvm", node ="node2 "}'
        values: "0 0 0 0 0 0"

    alert_rule_test:
      - eval_time: 5m
        alertname: LowKVMNodesCount
        exp_alerts:
          - exp_annotations:
              description: "Low number of nodes with KVM resource available."
              summary: "At least two nodes with kvm resource required for VM live migration."
              runbook_url: "https://kubevirt.io/monitoring/runbooks/LowKVMNodesCount"
            exp_labels:
              severity: "warning"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"

  # All nodes without KVM resources
  - interval: 1m
    input_series:
      - series: 'kube_node_status_allocatable{resource="devices_kubevirt_io_kvm", node ="node1"}'
        values: "0 0 0 0 0 0"
      - series: 'kube_node_status_allocatable{resource="devices_kubevirt_io_kvm", node ="node2 "}'
        values: "0 0 0 0 0 0"

    alert_rule_test:
      - eval_time: 5m
        alertname: LowKVMNodesCount
        exp_alerts:
          - exp_annotations:
              description: "Low number of nodes with KVM resource available."
              summary: "At least two nodes with kvm resource required for VM live migration."
              runbook_url: "https://kubevirt.io/monitoring/runbooks/LowKVMNodesCount"
            exp_labels:
              severity: "warning"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"

  # Two nodes with KVM resources
  - interval: 1m
    input_series:
      - series: 'kube_node_status_allocatable{resource="devices_kubevirt_io_kvm", node ="node1"}'
        values: "110 110 110 110 110 110"
      - series: 'kube_node_status_allocatable{resource="devices_kubevirt_io_kvm", node ="node2 "}'
        values: "110 110 110 110 110 110"

    alert_rule_test:
      - eval_time: 5m
        alertname: LowKVMNodesCount
        exp_alerts: []

  # Memory utilization less than 20MB close to requested memory - based on memory working set
  - interval: 1m
    input_series:
      - series: 'kube_pod_container_resource_requests{pod="virt-launcher-testvm-123", container="compute", resource="memory", namespace="ns-test"}'
        values: "67108864 67108864 67108864 67108864"
      - series: 'container_memory_working_set_bytes{pod="virt-launcher-testvm-123", container="compute", namespace="ns-test"}'
        values: "47185920 48234496 48234496 49283072"
      - series: 'container_memory_rss{pod="virt-launcher-testvm-123", container="compute", namespace="ns-test"}'
        values: "19922944 18874368 18874368 17825792"

    alert_rule_test:
      - eval_time: 1m
        alertname: KubevirtVmHighMemoryUsage
        exp_alerts:
          - exp_annotations:
              description: "Container compute in pod virt-launcher-testvm-123 in namespace ns-test free memory is less than 20 MB and it is close to requested memory"
              summary: "VM is at risk of being evicted and in serious cases of memory exhaustion being terminated by the runtime."
              runbook_url: "https://kubevirt.io/monitoring/runbooks/KubevirtVmHighMemoryUsage"
            exp_labels:
              severity: "warning"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"
              pod: "virt-launcher-testvm-123"
              container: "compute"
              namespace: "ns-test"

  # Memory utilization less than 20MB close to requested memory - based on memory RSS
  - interval: 1m
    input_series:
      - series: 'kube_pod_container_resource_requests{pod="virt-launcher-testvm-123", container="compute", resource="memory", namespace="ns-test"}'
        values: "67108864 67108864 67108864 67108864"
      - series: 'container_memory_working_set_bytes{pod="virt-launcher-testvm-123", container="compute", namespace="ns-test"}'
        values: "19922944 18874368 18874368 17825792"
      - series: 'container_memory_rss{pod="virt-launcher-testvm-123", container="compute", namespace="ns-test"}'
        values: "47185920 48234496 48234496 49283072"

    alert_rule_test:
      - eval_time: 1m
        alertname: KubevirtVmHighMemoryUsage
        exp_alerts:
          - exp_annotations:
              description: "Container compute in pod virt-launcher-testvm-123 in namespace ns-test free memory is less than 20 MB and it is close to requested memory"
              summary: "VM is at risk of being evicted and in serious cases of memory exhaustion being terminated by the runtime."
              runbook_url: "https://kubevirt.io/monitoring/runbooks/KubevirtVmHighMemoryUsage"
            exp_labels:
              severity: "warning"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"
              pod: "virt-launcher-testvm-123"
              container: "compute"
              namespace: "ns-test"

  # Memory utilization less than 20MB close to requested memory - based on memory RSS and memory working set
  - interval: 1m
    input_series:
      - series: 'kube_pod_container_resource_requests{pod="virt-launcher-testvm-123", container="compute", resource="memory", namespace="ns-test"}'
        values: "67108864 67108864 67108864 67108864"
      - series: 'container_memory_working_set_bytes{pod="virt-launcher-testvm-123", container="compute", namespace="ns-test"}'
        values: "47185920 48234496 48234496 49283072"
      - series: 'container_memory_rss{pod="virt-launcher-testvm-123", container="compute", namespace="ns-test"}'
        values: "47185920 48234496 48234496 49283072"

    alert_rule_test:
      - eval_time: 1m
        alertname: KubevirtVmHighMemoryUsage
        exp_alerts:
          - exp_annotations:
              description: "Container compute in pod virt-launcher-testvm-123 in namespace ns-test free memory is less than 20 MB and it is close to requested memory"
              summary: "VM is at risk of being evicted and in serious cases of memory exhaustion being terminated by the runtime."
              runbook_url: "https://kubevirt.io/monitoring/runbooks/KubevirtVmHighMemoryUsage"
            exp_labels:
              severity: "warning"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"
              pod: "virt-launcher-testvm-123"
              container: "compute"
              namespace: "ns-test"

  # Memory utilization more than 20MB close to requested memory
  - interval: 30s
    input_series:
      - series: 'kube_pod_container_resource_requests{pod="virt-launcher-testvm-123", container="compute", resource="memory", namespace="ns-test"}'
        values: "67108864 67108864 67108864 67108864"
      - series: 'container_memory_working_set_bytes{pod="virt-launcher-testvm-123", container="compute", namespace="ns-test"}'
        values: "19922944 18874368 18874368 17825792"
      - series: 'container_memory_rss{pod="virt-launcher-testvm-123", container="compute", namespace="ns-test"}'
        values: "19922944 18874368 18874368 17825792"

    alert_rule_test:
      - eval_time: 1m
        alertname: KubevirtVmHighMemoryUsage
        exp_alerts: []

  # VM eviction strategy is set but vm is not migratable
  - interval: 1m
    input_series:
      - series: 'kubevirt_vmi_non_evictable{node="node1", namespace="ns-test", name="vm-evict-nonmigratable"}'
        values: "1 1 1 1 1 1 1 1"

    alert_rule_test:
      - eval_time: 1m
        alertname: VMCannotBeEvicted
        exp_alerts:
          - exp_annotations:
              description: "Eviction policy for vm-evict-nonmigratable (on node node1) is set to Live Migration but the VM is not migratable"
              summary: "The VM's eviction strategy is set to Live Migration but the VM is not migratable"
              runbook_url: "https://kubevirt.io/monitoring/runbooks/VMCannotBeEvicted"
            exp_labels:
              severity: "warning"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"
              name: "vm-evict-nonmigratable"
              namespace: "ns-test"
              node: "node1"

  # VM eviction strategy is set and vm is migratable
  - interval: 1m
    input_series:
      - series: 'kubevirt_vmi_non_evictable{node="node1", namespace="ns-test", name="vm-evict-migratable"}'
        values: "0 0 0 0 0 0 0 0 "

    alert_rule_test:
      - eval_time: 1m
        alertname: VMCannotBeEvicted
        exp_alerts: []

  # Test recording rule
  - interval: 1m
    input_series:
      # In reality there are many labels on these metrics
      # they are the same except the ones containing vm name like "name" in the example below
      - series: 'kubevirt_vmi_memory_available_bytes{container="virt-handler", name="vm-example-1", namespace="default", node="node-1"}'
        # time:  0          1          2          3
        values: "1376882688 1376882688 1376882688 1376882688"
      - series: 'kubevirt_vmi_memory_available_bytes{container="virt-handler", name="vm-example-2", namespace="default", node="node-1"}'
        # time:  0          1          2          3
        values: "2893266944 2893266944 2893266944 2893266944"
      - series: 'kubevirt_vmi_memory_usable_bytes{container="virt-handler", name="vm-example-1", namespace="default", node="node-1"}'
        # time:  0          1          2          3
        values: "1073176576 1073176576 1073176576 1273176576"
      - series: 'kubevirt_vmi_memory_usable_bytes{container="virt-handler", name="vm-example-2", namespace="default", node="node-1"}'
        # time:  0          1          2          3
        values: "2448936960 2448936960 2448936960 2658936964"
    promql_expr_test:
      - expr: 'kubevirt_vmi_memory_used_bytes'
        eval_time: 1m
        exp_samples:
          - labels: 'kubevirt_vmi_memory_used_bytes{container="virt-handler", name="vm-example-1", namespace="default", node="node-1"}'
            value: 303706112
          - labels: 'kubevirt_vmi_memory_used_bytes{container="virt-handler", name="vm-example-2", namespace="default", node="node-1"}'
            value: 444329984
      - expr: 'kubevirt_vmi_memory_used_bytes'
        eval_time: 3m
        exp_samples:
          - labels: 'kubevirt_vmi_memory_used_bytes{container="virt-handler", name="vm-example-1", namespace="default", node="node-1"}'
            value: 103706112
          - labels: 'kubevirt_vmi_memory_used_bytes{container="virt-handler", name="vm-example-2", namespace="default", node="node-1"}'
            value: 234329980

  # Excessive VMI Migrations in a period of time
  # Simulating metric resets at time slots 5 and 9
  - interval: 1h
    input_series:
      - series: 'kubevirt_migrate_vmi_succeeded_total{vmi="vmi-example-1", source="node-1", target="node-2"}'
        # time:  0 1 2 3 4 5 6 7 8 9 10 11 12 13
        values: "_ 1 2 2 3 _ 1 2 2 _ _  1  2  3+0x13"        # 7 increases, in samples: 2,4,6,7,11,12,13
      - series: 'kubevirt_migrate_vmi_succeeded_total{vmi="vmi-example-1", source="node-2", target="node-1"}'
        # time:  0 1 2 3 4 5 6 7 8 9 10 11 12 13
        values: "_ _ 1 2 2 _ _ 1 2 _ 1  2  2  2+0x13"        # 6 increases, in samples: 2,3,7,8,10,11

    alert_rule_test:
      # at 11h, there are total of 11 migrations made on a single VMI, so the alert should not be fired.
      - eval_time: 11h
        alertname: KubeVirtVMIExcessiveMigrations
        exp_alerts: []
      # at 16h, there are total of 13 migrations made on a single VMI, thus the alert is expected to be fired.
      - eval_time: 16h
        alertname: KubeVirtVMIExcessiveMigrations
        exp_alerts:
          - exp_annotations:
              description: "VirtualMachineInstance vmi-example-1 has been migrated more than 12 times during the last 24 hours"
              summary: "An excessive amount of migrations have been detected on a VirtualMachineInstance in the last 24 hours."
              runbook_url: "https://kubevirt.io/monitoring/runbooks/KubeVirtVMIExcessiveMigrations"
            exp_labels:
              severity: "warning"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"
              vmi: vmi-example-1
      - eval_time: 24h
        alertname: KubeVirtVMIExcessiveMigrations
        exp_alerts:
          - exp_annotations:
              description: "VirtualMachineInstance vmi-example-1 has been migrated more than 12 times during the last 24 hours"
              summary: "An excessive amount of migrations have been detected on a VirtualMachineInstance in the last 24 hours."
              runbook_url: "https://kubevirt.io/monitoring/runbooks/KubeVirtVMIExcessiveMigrations"
            exp_labels:
              severity: "warning"
              kubernetes_operator_part_of: "kubevirt"
              kubernetes_operator_component: "kubevirt"
              vmi: vmi-example-1
      # since the first increase occurred in 2h, will need to evaluate at 26h to disregard the first increase and clear the alert.
      - eval_time: 26h
        alertname: KubeVirtVMIExcessiveMigrations
        exp_alerts: []