Improving live migration metrics

Add and fix metrics for live migration. Fix MemoryDirtyRate Add DiskBps Add MemoryBps Signed-off-by: borod108 <[email protected]>
dominikholler · Jun 20, 2022 · 7389d84 · 7389d84
1 parent 970d68a
commit 7389d84
Show file tree

Hide file tree

Showing 8 changed files with 69 additions and 14 deletions.
diff --git a/docs/metrics.md b/docs/metrics.md
@@ -21,6 +21,12 @@ The remaining guest OS data to be migrated to the new VM. Type: Gauge.
 ### kubevirt_migrate_vmi_dirty_memory_rate_bytes
 The rate of memory being dirty in the Guest OS. Type: Gauge.
 
+### kubevirt_migrate_vmi_disk_transfer_rate_bytes
+The rate at which the disk is being transferred. Type: Gauge.
+
+### kubevirt_migrate_vmi_memory_transfer_rate_bytes
+The rate at which the memory is being transferred. Type: Gauge.
+
 ### kubevirt_virt_controller_leading
 Indication for an operating virt-controller. Type: Gauge.
 

diff --git a/pkg/monitoring/domainstats/prometheus/prometheus.go b/pkg/monitoring/domainstats/prometheus/prometheus.go
@@ -41,10 +41,12 @@ import (
 )
 
 const (
-	PrometheusCollectionTimeout         = vms.CollectionTimeout
-	MigrateVmiDataRemainingMetricName   = "kubevirt_migrate_vmi_data_remaining_bytes"
-	MigrateVmiDataProcessedMetricName   = "kubevirt_migrate_vmi_data_processed_bytes"
-	MigrateVmiDirtyMemoryRateMetricName = "kubevirt_migrate_vmi_dirty_memory_rate_bytes"
+	PrometheusCollectionTimeout            = vms.CollectionTimeout
+	MigrateVmiDataRemainingMetricName      = "kubevirt_migrate_vmi_data_remaining_bytes"
+	MigrateVmiDataProcessedMetricName      = "kubevirt_migrate_vmi_data_processed_bytes"
+	MigrateVmiDirtyMemoryRateMetricName    = "kubevirt_migrate_vmi_dirty_memory_rate_bytes"
+	MigrateVmiMemoryTransferRateMetricName = "kubevirt_migrate_vmi_memory_transfer_rate_bytes"
+	MigrateVmiDiskTransferRateMetricName   = "kubevirt_migrate_vmi_disk_transfer_rate_bytes"
 )
 
 var (
@@ -100,6 +102,23 @@ func (metrics *vmiMetrics) updateMigrateInfo(jobInfo *stats.DomainJobInfo) {
 		)
 	}
 
+	if jobInfo.MemoryBpsSet {
+		metrics.pushCommonMetric(
+			MigrateVmiMemoryTransferRateMetricName,
+			"The total VM memory processed and migrated.",
+			prometheus.GaugeValue,
+			float64(jobInfo.MemoryBps),
+		)
+	}
+
+	if jobInfo.DiskBpsSet {
+		metrics.pushCommonMetric(
+			MigrateVmiDiskTransferRateMetricName,
+			"The total VM data processed and migrated.",
+			prometheus.GaugeValue,
+			float64(jobInfo.DiskBps),
+		)
+	}
 }
 
 func (metrics *vmiMetrics) updateMemory(mem *stats.DomainStatsMemory) {

diff --git a/pkg/monitoring/domainstats/prometheus/prometheus_test.go b/pkg/monitoring/domainstats/prometheus/prometheus_test.go
@@ -351,6 +351,18 @@ var _ = Describe("Prometheus", func() {
 					MemDirtyRateSet: true,
 					MemDirtyRate:    1,
 				}),
+			Entry("should handle MemoryBps metrics for VMs",
+				MigrateVmiMemoryTransferRateMetricName,
+				&stats.DomainJobInfo{
+					MemoryBpsSet: true,
+					MemoryBps:    1,
+				}),
+			Entry("should handle DiskBps metrics for VMs",
+				MigrateVmiDiskTransferRateMetricName,
+				&stats.DomainJobInfo{
+					DiskBpsSet: true,
+					DiskBps:    1,
+				}),
 		)
 
 		It("should handle vcpu metrics", func() {

diff --git a/pkg/virt-launcher/virtwrap/live-migration-source.go b/pkg/virt-launcher/virtwrap/live-migration-source.go
@@ -788,11 +788,7 @@ func (m *migrationMonitor) startMonitor() {
 
 	logger := log.Log.Object(vmi)
 	defer func() {
-		m.l.migrateInfoStats = &stats.DomainJobInfo{
-			DataProcessed: 0,
-			DataRemaining: 0,
-			MemDirtyRate:  0,
-		}
+		m.l.migrateInfoStats = &stats.DomainJobInfo{}
 	}()
 
 	domName := api.VMINamespaceKeyFunc(vmi)
@@ -811,10 +807,10 @@ func (m *migrationMonitor) startMonitor() {
 
 		err := m.hasMigrationErr()
 		if err != nil && m.migrationFailedWithError == nil {
-			logger.Reason(err).Error("Recevied a live migration error. Will check the latest migration status.")
+			logger.Reason(err).Error("Received a live migration error. Will check the latest migration status.")
 			m.migrationFailedWithError = err
 		} else if m.migrationFailedWithError != nil {
-			logger.Info("Didn't manage to get a job status. Post the received error and finilize.")
+			logger.Info("Didn't manage to get a job status. Post the received error and finalize.")
 			logger.Reason(m.migrationFailedWithError).Error(liveMigrationFailed)
 			var abortStatus v1.MigrationAbortStatus
 			if strings.Contains(m.migrationFailedWithError.Error(), "canceled by client") {

diff --git a/pkg/virt-launcher/virtwrap/stats/types.go b/pkg/virt-launcher/virtwrap/stats/types.go
@@ -177,6 +177,10 @@ type DomainStatsMemory struct {
 type DomainJobInfo struct {
 	DataProcessedSet bool
 	DataProcessed    uint64
+	MemoryBpsSet     bool
+	MemoryBps        uint64
+	DiskBpsSet       bool
+	DiskBps          uint64
 	DataRemainingSet bool
 	DataRemaining    uint64
 	MemDirtyRateSet  bool

diff --git a/pkg/virt-launcher/virtwrap/statsconv/converter.go b/pkg/virt-launcher/virtwrap/statsconv/converter.go
@@ -214,9 +214,13 @@ func Convert_libvirt_DomainJobInfo_To_stats_DomainJobInfo(info *libvirt.DomainJo
 	return &stats.DomainJobInfo{
 		DataProcessedSet: info.DataProcessedSet,
 		DataProcessed:    info.DataProcessed,
+		MemoryBpsSet:     info.MemBpsSet,
+		MemoryBps:        info.MemBps,
+		DiskBpsSet:       info.DiskBpsSet,
+		DiskBps:          info.DiskBps,
 		DataRemainingSet: info.DataRemainingSet,
 		DataRemaining:    info.DataRemaining,
-		MemDirtyRateSet:  info.MemDirtyRateSet,
-		MemDirtyRate:     info.MemDirtyRate,
+		MemDirtyRateSet:  info.MemDirtyRateSet && info.MemPageSizeSet,
+		MemDirtyRate:     info.MemDirtyRate * info.MemPageSize,
 	}
 }
diff --git a/pkg/virt-launcher/virtwrap/statsconv/util/domstats_utils.go b/pkg/virt-launcher/virtwrap/statsconv/util/domstats_utils.go
@@ -187,7 +187,11 @@ var Testdataexpected = `{
      "DataRemaining": 0,
      "DataRemainingSet": false,
      "MemDirtyRate": 0,
-     "MemDirtyRateSet": false
+     "MemDirtyRateSet": false,
+     "MemoryBpsSet": false,
+     "MemoryBps": 0,
+     "DiskBpsSet": false,
+     "DiskBps": 0
    },
    "Name": "testName", 
    "Net": [

diff --git a/tools/doc-generator/doc-generator.go b/tools/doc-generator/doc-generator.go
@@ -158,6 +158,16 @@ var (
 			description: "The rate of memory being dirty in the Guest OS.",
 			mType:       "Gauge",
 		},
+		{
+			name:        domainstats.MigrateVmiMemoryTransferRateMetricName,
+			description: "The rate at which the memory is being transferred.",
+			mType:       "Gauge",
+		},
+		{
+			name:        domainstats.MigrateVmiDiskTransferRateMetricName,
+			description: "The rate at which the disk is being transferred.",
+			mType:       "Gauge",
+		},
 	}
 )