add numa and aligned cpu to devices metadata

Signed-off-by: Vladik Romanovsky <[email protected]>

pkg/cloud-init/cloud-init.go

@@ -93,14 +93,14 @@ type ConfigDriveMetadata struct {
 }
 
 type DeviceData struct {
-	Type     	DeviceMetadataType `json:"type"`
-	Bus      	string             `json:"bus"`
-	Address 	string             `json:"address"`
-	MAC      	string             `json:"mac,omitempty"`
-	Serial   	string             `json:"serial,omitempty"`
-	NumaNode 	uint32             `json:"numaNode,omitempty"`
-	AlignedCPUs     []uint32           `json:"alignedCPUs,omitempty"`
-	Tags     	[]string           `json:"tags"`
+	Type        DeviceMetadataType `json:"type"`
+	Bus         string             `json:"bus"`
+	Address     string             `json:"address"`
+	MAC         string             `json:"mac,omitempty"`
+	Serial      string             `json:"serial,omitempty"`
+	NumaNode    uint32             `json:"numaNode,omitempty"`
+	AlignedCPUs []uint32           `json:"alignedCPUs,omitempty"`
+	Tags        []string           `json:"tags"`
 }
 
 // IsValidCloudInitData checks if the given CloudInitData object is valid in the sense that GenerateLocalData can be called with it.

pkg/cloud-init/cloud-init_test.go

@@ -162,13 +162,13 @@ var _ = Describe("CloudInit", func() {
 }`
 				devices := []DeviceData{
 					{
-						Type:     HostDevMetadataType,
-						Bus:      "pci",
-						Address:  "0000:65:10:0",
-						MAC:      "",
-						NumaNode: uint32(1),
+						Type:        HostDevMetadataType,
+						Bus:         "pci",
+						Address:     "0000:65:10:0",
+						MAC:         "",
+						NumaNode:    uint32(1),
 						AlignedCPUs: []uint32{0, 1},
-						Tags:     []string{"testtag1"},
+						Tags:        []string{"testtag1"},
 					},
 				}
 

pkg/util/hardware/BUILD.bazel

@@ -6,8 +6,8 @@ go_library(
     importpath = "kubevirt.io/kubevirt/pkg/util/hardware",
     visibility = ["//visibility:public"],
     deps = [
-        "//staging/src/kubevirt.io/api/core/v1:go_default_library",
         "//pkg/util:go_default_library",
+        "//staging/src/kubevirt.io/api/core/v1:go_default_library",
     ],
 )
 

pkg/util/hardware/hw_utils.go

@@ -113,7 +113,6 @@ func ParsePciAddress(pciAddress string) ([]string, error) {
 	return res[1:], nil
 }
 
-
 func GetDeviceNumaNode(pciAddress string) (*uint32, error) {
 	pciBasePath := "/sys/bus/pci/devices"
 	numaNodePath := filepath.Join(pciBasePath, pciAddress, "numa_node")

pkg/virt-launcher/virtwrap/BUILD.bazel

@@ -25,6 +25,7 @@ go_library(
         "//pkg/network/cache:go_default_library",
         "//pkg/network/setup:go_default_library",
         "//pkg/util:go_default_library",
+        "//pkg/util/hardware:go_default_library",
         "//pkg/util/net/ip:go_default_library",
         "//pkg/virt-handler/cmd-client:go_default_library",
         "//pkg/virt-handler/migration-proxy:go_default_library",

pkg/virt-launcher/virtwrap/manager.go

@@ -43,6 +43,7 @@ import (
 
 	"kubevirt.io/kubevirt/pkg/downwardmetrics"
 	"kubevirt.io/kubevirt/pkg/network/cache"
+	"kubevirt.io/kubevirt/pkg/util/hardware"
 	cmdclient "kubevirt.io/kubevirt/pkg/virt-handler/cmd-client"
 	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/agent"
 	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/converter"
@@ -73,8 +74,6 @@ import (
 	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/cli"
 	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/device/hostdevice"
 	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/device/hostdevice/legacy"
-	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/device/hostdevice/generic"
-	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/device/hostdevice/gpu"
 	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/device/hostdevice/sriov"
 	domainerrors "kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/errors"
 	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/stats"
@@ -203,18 +202,26 @@ func newLibvirtDomainManager(connection cli.Connection, virtShareDir string, age
 	return &manager, nil
 }
 
-func getAllDomainDevices(dom cli.VirDomain) (api.Devices, error) {
+func getDomainSpec(dom cli.VirDomain) (*api.DomainSpec, error) {
+	var newSpec api.DomainSpec
 	xmlstr, err := dom.GetXMLDesc(0)
 	if err != nil {
-		return api.Devices{}, err
+		return &newSpec, err
 	}
-	var newSpec api.DomainSpec
 	err = xml.Unmarshal([]byte(xmlstr), &newSpec)
 	if err != nil {
-		return api.Devices{}, err
+		return &newSpec, err
 	}
 
-	return newSpec.Devices, nil
+	return &newSpec, nil
+}
+
+func getAllDomainDevices(dom cli.VirDomain) (api.Devices, error) {
+	domSpec, err := getDomainSpec(dom)
+	if err != nil {
+		return domSpec.Devices, err
+	}
+	return domSpec.Devices, nil
 }
 
 func getAllDomainDisks(dom cli.VirDomain) ([]api.Disk, error) {
@@ -1483,23 +1490,71 @@ func (l *LibvirtDomainManager) GetDomainStats() ([]*stats.DomainStats, error) {
 	return l.virConn.GetDomainStats(statsTypes, flags)
 }
 
-func addToDeviceMetadata(metadataType cloudinit.DeviceMetadataType, address *api.Address, mac string, tag string, devicesMetadata []cloudinit.DeviceData) []cloudinit.DeviceData {
-	pciAddrStr := fmt.Sprintf("%s:%s:%s:%s", address.Domain[2:], address.Bus[2:], address.Slot[2:], address.Function[2:])
+func lookupDeviceCPUAffinity(pciAddress string, domainSpec *api.DomainSpec) ([]uint32, error) {
+	alignedVCPUList := []uint32{}
+	p2vCPUMap := make(map[string]uint32)
+	alignedPhysicalCPUs, err := hardware.GetDeviceAlignedCPUs(pciAddress)
+	if err != nil {
+		return nil, err
+	}
+
+	// make sure that the VMI has cpus from this numa node.
+	cpuTune := domainSpec.CPUTune.VCPUPin
+	for _, vcpuPin := range cpuTune {
+		p2vCPUMap[vcpuPin.CPUSet] = vcpuPin.VCPU
+	}
+
+	for _, pcpu := range alignedPhysicalCPUs {
+		if vCPU, exist := p2vCPUMap[strconv.Itoa(int(pcpu))]; exist {
+			alignedVCPUList = append(alignedVCPUList, uint32(vCPU))
+		}
+	}
+	return alignedVCPUList, nil
+
+}
+
+func formatPCIAddressStr(address *api.Address) string {
+	return fmt.Sprintf("%s:%s:%s.%s", address.Domain[2:], address.Bus[2:], address.Slot[2:], address.Function[2:])
+}
+
+func addToDeviceMetadata(metadataType cloudinit.DeviceMetadataType, address *api.Address, mac string, tag string, devicesMetadata []cloudinit.DeviceData, numa *uint32, numaAlignedCPUs []uint32) []cloudinit.DeviceData {
+	pciAddrStr := formatPCIAddressStr(address)
 	deviceData := cloudinit.DeviceData{
 		Type:    metadataType,
 		Bus:     address.Type,
 		Address: pciAddrStr,
 		MAC:     mac,
 		Tags:    []string{tag},
 	}
+	if numa != nil {
+		deviceData.NumaNode = *numa
+	}
+	if len(numaAlignedCPUs) > 0 {
+		deviceData.AlignedCPUs = numaAlignedCPUs
+	}
 	devicesMetadata = append(devicesMetadata, deviceData)
 	return devicesMetadata
 }
 
+func getDeviceNUMACPUAffinity(pciAddress string, vmi *v1.VirtualMachineInstance, domainSpec *api.DomainSpec) (numaNodePtr *uint32, cpuList []uint32) {
+	if vmi.Spec.Domain.CPU.NUMA != nil && vmi.Spec.Domain.CPU.NUMA.GuestMappingPassthrough != nil {
+		if numa, err := hardware.GetDeviceNumaNode(pciAddress); err == nil {
+			numaNodePtr = numa
+			return
+		}
+	} else if vmi.IsCPUDedicated() {
+		if vCPUList, err := lookupDeviceCPUAffinity(pciAddress, domainSpec); err == nil {
+			cpuList = vCPUList
+			return
+		}
+	}
+	return
+}
+
 func (l *LibvirtDomainManager) buildDevicesMetadata(vmi *v1.VirtualMachineInstance, dom cli.VirDomain) ([]cloudinit.DeviceData, error) {
 	taggedInterfaces := make(map[string]v1.Interface)
-	taggedHostDevices := make(map[string]v1.HostDevices)
-	taggedGPUs := make(map[string]v1.GPUs)
+	taggedHostDevices := make(map[string]v1.HostDevice)
+	taggedGPUs := make(map[string]v1.GPU)
 	var devicesMetadata []cloudinit.DeviceData
 
 	// Get all tagged interfaces for lookup
@@ -1523,55 +1578,67 @@ func (l *LibvirtDomainManager) buildDevicesMetadata(vmi *v1.VirtualMachineInstan
 		}
 	}
 
-	devices, err := getAllDomainDevices(dom)
+	domainSpec, err := getDomainSpec(dom)
 	if err != nil {
 		return nil, err
 	}
-
+	devices := domainSpec.Devices
 	interfaces := devices.Interfaces
 	for _, nic := range interfaces {
 		if data, exist := taggedInterfaces[nic.Alias.GetName()]; exist {
 			var mac string
 			if nic.MAC != nil {
 				mac = nic.MAC.MAC
 			}
+			deviceNumaNode, deviceAlignedCPUs := getDeviceNUMACPUAffinity(formatPCIAddressStr(nic.Source.Address), vmi, domainSpec)
 			devicesMetadata = addToDeviceMetadata(cloudinit.NICMetadataType,
 				nic.Address,
 				mac,
 				data.Tag,
-				devicesMetadata)
+				devicesMetadata,
+				deviceNumaNode,
+				deviceAlignedCPUs,
+			)
 		}
 	}
 
 	hostDevices := devices.HostDevices
 	for _, dev := range hostDevices {
 		devAliasNoPrefix := strings.Replace(dev.Alias.GetName(), sriov.AliasPrefix, "", -1)
 		hostDevAliasNoPrefix := strings.Replace(dev.Alias.GetName(), generic.AliasPrefix, "", -1)
+		gpuDevAliasNoPrefix := strings.Replace(dev.Alias.GetName(), gpu.AliasPrefix, "", -1)
 		if data, exist := taggedInterfaces[devAliasNoPrefix]; exist {
+			deviceNumaNode, deviceAlignedCPUs := getDeviceNUMACPUAffinity(formatPCIAddressStr(dev.Source.Address), vmi, domainSpec)
 			devicesMetadata = addToDeviceMetadata(cloudinit.NICMetadataType,
 				dev.Address,
 				"",
 				data.Tag,
-				devicesMetadata)
+				devicesMetadata,
+				deviceNumaNode,
+				deviceAlignedCPUs,
+			)
 		}
 		if data, exist := taggedHostDevices[hostDevAliasNoPrefix]; exist {
+			deviceNumaNode, deviceAlignedCPUs := getDeviceNUMACPUAffinity(formatPCIAddressStr(dev.Source.Address), vmi, domainSpec)
 			devicesMetadata = addToDeviceMetadata(cloudinit.HostDevMetadataType,
 				dev.Address,
 				"",
 				data.Tag,
-				devicesMetadata)
+				devicesMetadata,
+				deviceNumaNode,
+				deviceAlignedCPUs,
+			)
 		}
-	}
-
-	GPUs := devices.GPUs
-	for _, dev := range GPUs {
-		devAliasNoPrefix := strings.Replace(dev.Alias.GetName(), gpu.AliasPrefix, "", -1)
-		if data, exist := taggedGPUs[devAliasNoPrefix]; exist {
-			devicesMetadata = addToDeviceMetadata(cloudinit.NICMetadataType,
+		if data, exist := taggedHostDevices[gpuDevAliasNoPrefix]; exist {
+			deviceNumaNode, deviceAlignedCPUs := getDeviceNUMACPUAffinity(formatPCIAddressStr(dev.Source.Address), vmi, domainSpec)
+			devicesMetadata = addToDeviceMetadata(cloudinit.HostDevMetadataType,
 				dev.Address,
 				"",
 				data.Tag,
-				devicesMetadata)
+				devicesMetadata,
+				deviceNumaNode,
+				deviceAlignedCPUs,
+			)
 		}
 	}
 	return devicesMetadata, nil

pkg/virt-operator/resource/generate/components/validations_generated.go

@@ -14464,6 +14464,11 @@ var CRDsValidation map[string]string = map[string]string{
                                         description: Name of the GPU device as exposed
                                           by a device plugin
                                         type: string
+                                      tag:
+                                        description: If specified, the virtual network
+                                          interface address and its tag will be provided
+                                          to the guest via config drive
+                                        type: string
                                       virtualGPUOptions:
                                         properties:
                                           display:
@@ -14505,6 +14510,11 @@ var CRDsValidation map[string]string = map[string]string{
                                         type: string
                                       name:
                                         type: string
+                                      tag:
+                                        description: If specified, the virtual network
+                                          interface address and its tag will be provided
+                                          to the guest via config drive
+                                        type: string
                                     required:
                                     - deviceName
                                     - name