Merge pull request kubevirt#10107 from PiotrProkop/add-delay-vcpu-metric

Expose kubevirt_vmi_vcpu_delay_seconds_total metric.
zyjj1 · Jul 27, 2023 · 2e23712 · 2e23712
2 parents 17e766a + bc56933
commit 2e23712
Show file tree

Hide file tree

Showing 6 changed files with 67 additions and 8 deletions.
diff --git a/docs/metrics.md b/docs/metrics.md
@@ -231,6 +231,9 @@ Total time (ms) spent on write operations. Type: Counter.
 ### kubevirt_vmi_storage_write_traffic_bytes_total
 Total number of written bytes. Type: Counter.
 
+### kubevirt_vmi_vcpu_delay_seconds_total
+Amount of time spent by each vcpu waiting in the queue instead of running. Type: Counter.
+
 ### kubevirt_vmi_vcpu_seconds
 Total amount of time spent in each state by each vcpu (cpu_time excluding hypervisor time). Where `id` is the vcpu identifier and `state` can be one of the following: [`OFFLINE`, `RUNNING`, `BLOCKED`]. Type: Counter.
 

diff --git a/pkg/monitoring/domainstats/prometheus/prometheus.go b/pkg/monitoring/domainstats/prometheus/prometheus.go
@@ -300,6 +300,16 @@ func (metrics *vmiMetrics) updateVcpu(vcpuStats []stats.DomainStatsVcpu) {
 				[]string{stringVcpuIdx},
 			)
 		}
+		if vcpu.DelaySet {
+			metrics.pushCustomMetric(
+				"kubevirt_vmi_vcpu_delay_seconds_total",
+				"Amount of time spent by each vcpu waiting in the queue instead of running.",
+				prometheus.CounterValue,
+				float64(vcpu.Delay)/float64(1000000000),
+				[]string{"id"},
+				[]string{stringVcpuIdx},
+			)
+		}
 	}
 }
 

diff --git a/pkg/monitoring/domainstats/prometheus/prometheus_test.go b/pkg/monitoring/domainstats/prometheus/prometheus_test.go
@@ -1154,6 +1154,32 @@ var _ = Describe("Prometheus", func() {
 			Expect(result.Desc().String()).To(ContainSubstring("kubevirt_vmi_vcpu_wait_seconds"))
 		})
 
+		It("should expose vcpu delay metric", func() {
+			ch := make(chan prometheus.Metric, 1)
+			defer close(ch)
+
+			ps := prometheusScraper{ch: ch}
+
+			domainStats := &stats.DomainStats{
+				Cpu:    &stats.DomainStatsCPU{},
+				Memory: &stats.DomainStatsMemory{},
+				Net:    []stats.DomainStatsNet{},
+				Vcpu: []stats.DomainStatsVcpu{
+					{
+						DelaySet: true,
+						Delay:    800000000,
+					},
+				},
+			}
+
+			vmi := k6tv1.VirtualMachineInstance{}
+			ps.Report("test", &vmi, newVmStats(domainStats, nil))
+
+			result := <-ch
+			Expect(result).ToNot(BeNil())
+			Expect(result.Desc().String()).To(ContainSubstring("kubevirt_vmi_vcpu_delay_seconds_total"))
+		})
+
 		It("should expose vcpu to cpu pinning metric", func() {
 			ch := make(chan prometheus.Metric, 1)
 			defer close(ch)

diff --git a/pkg/virt-launcher/virtwrap/stats/types.go b/pkg/virt-launcher/virtwrap/stats/types.go
@@ -89,6 +89,8 @@ type DomainStatsVcpu struct {
 	Time     uint64
 	WaitSet  bool
 	Wait     uint64
+	DelaySet bool
+	Delay    uint64
 }
 
 type DomainStatsNet struct {

diff --git a/pkg/virt-launcher/virtwrap/statsconv/converter.go b/pkg/virt-launcher/virtwrap/statsconv/converter.go
@@ -130,6 +130,8 @@ func Convert_libvirt_DomainStatsVcpu_To_stats_DomainStatsVcpu(in []libvirt.Domai
 			Time:     inItem.Time,
 			WaitSet:  inItem.WaitSet,
 			Wait:     inItem.Wait,
+			Delay:    inItem.Delay,
+			DelaySet: inItem.DelaySet,
 		})
 	}
 	return ret

diff --git a/pkg/virt-launcher/virtwrap/statsconv/util/domstats_utils.go b/pkg/virt-launcher/virtwrap/statsconv/util/domstats_utils.go
@@ -16,7 +16,9 @@ var Testdata = `[
             "State" : 1,
             "Time" : 23810000000,
             "WaitSet": false,
-            "Wait": 0
+            "Wait": 0,
+            "DelaySet": false,
+            "Delay": 0
             
          },
          {
@@ -25,23 +27,29 @@ var Testdata = `[
             "TimeSet" : true,
             "Time" : 17800000000,
             "WaitSet": false,
-            "Wait": 0
+            "Wait": 0,
+            "DelaySet": false,
+            "Delay": 0
          },
          {
             "State" : 1,
             "StateSet" : true,
             "TimeSet" : true,
             "Time" : 23310000000,
             "WaitSet": false,
-            "Wait": 0
+            "Wait": 0,
+            "DelaySet": false,
+            "Delay": 0
          },
          {
             "State" : 1,
             "StateSet" : true,
             "TimeSet" : true,
             "Time" : 17360000000,
             "WaitSet": true,
-            "Wait": 1500
+            "Wait": 1500,
+            "DelaySet": true,
+            "Delay": 100
          }
       ],
       "Perf" : null,
@@ -228,15 +236,19 @@ var Testdataexpected = `{
        "Time": 23810000000, 
        "TimeSet": true,
        "WaitSet": false,
-       "Wait": 0
+       "Wait": 0,
+       "DelaySet": false,
+       "Delay": 0
      }, 
      {
        "State": 1, 
        "StateSet": true, 
        "Time": 17800000000, 
        "TimeSet": true,
        "WaitSet": false,
-       "Wait": 0
+       "Wait": 0,
+       "DelaySet": false,
+       "Delay": 0
        
      }, 
      {
@@ -245,15 +257,19 @@ var Testdataexpected = `{
        "Time": 23310000000, 
        "TimeSet": true,
        "WaitSet": false,
-       "Wait": 0
+       "Wait": 0,
+       "DelaySet": false,
+       "Delay": 0
      }, 
      {
        "State": 1, 
        "StateSet": true, 
        "Time": 17360000000, 
        "TimeSet": true,
        "WaitSet": true,
-       "Wait": 1500
+       "Wait": 1500,
+       "DelaySet": true,
+       "Delay": 100
      }
    ],
    "CPUMapSet": false,