Support extra CPU flags when starting a VM

api/openapi-spec/swagger.json

@@ -4048,6 +4048,13 @@
       "description": "DedicatedCPUPlacement requests the scheduler to place the VirtualMachineInstance on a node\nwith enough dedicated pCPUs and pin the vCPUs to it.\n+optional",
       "type": "boolean"
      },
+     "features": {
+      "description": "Features specifies the CPU features list inside the VMI.\nIn sepcial cases where Model is set to \"host-model\" or \"host-passthrough\",\nit is possible to set additional details on the CPU using these features.\nFor more information see https://libvirt.org/formatdomain.html#elementsCPU.\n+optional",
+      "type": "array",
+      "items": {
+       "$ref": "#/definitions/v1.Feature"
+      }
+     },
      "model": {
       "description": "Model specifies the CPU model inside the VMI.\nList of available models https://github.com/libvirt/libvirt/blob/master/src/cpu/cpu_map.xml.\nIt is possible to specify special cases like \"host-passthrough\" to get the same CPU as the node\nand \"host-model\" to get CPU closest to the node one.\nFor more information see https://libvirt.org/formatdomain.html#elementsCPU.\nDefaults to host-model.\n+optional",
       "type": "string"
@@ -4436,6 +4443,19 @@
      }
     }
    },
+   "v1.Feature": {
+    "description": "Feature allows specifying a CPU feature.",
+    "properties": {
+     "name": {
+      "description": "Name of the CPU feature",
+      "type": "string"
+     },
+     "policy": {
+      "description": "Policy is the CPU feature attribute which can have the following attributes:\nforce    - The virtual CPU will claim the feature is supported regardless of it being supported by host CPU.\nrequire  - Guest creation will fail unless the feature is supported by the host CPU or the hypervisor is able to emulate it.\noptional - The feature will be supported by virtual CPU if and only if it is supported by host CPU.\ndisable  - The feature will not be supported by virtual CPU.\nforbid   - Guest creation will fail if the feature is supported by host CPU.\nDefaults to require\n+optional",
+      "type": "string"
+     }
+    }
+   },
    "v1.FeatureAPIC": {
     "properties": {
      "enabled": {

pkg/api/v1/schema.go

@@ -210,12 +210,35 @@ type CPU struct {
 	// Defaults to host-model.
 	// +optional
 	Model string `json:"model,omitempty"`
+	// Features specifies the CPU features list inside the VMI.
+	// In sepcial cases where Model is set to "host-model" or "host-passthrough",
+	// it is possible to set additional details on the CPU using these features.
+	// For more information see https://libvirt.org/formatdomain.html#elementsCPU.
+	// +optional
+	Features []Feature `json:"features,omitempty"`
 	// DedicatedCPUPlacement requests the scheduler to place the VirtualMachineInstance on a node
 	// with enough dedicated pCPUs and pin the vCPUs to it.
 	// +optional
 	DedicatedCPUPlacement bool `json:"dedicatedCpuPlacement,omitempty"`
 }
 
+// Feature allows specifying a CPU feature.
+// ---
+// +k8s:openapi-gen=true
+type Feature struct {
+	// Name of the CPU feature
+	Name string `json:"name,omitempty"`
+	// Policy is the CPU feature attribute which can have the following attributes:
+	// force    - The virtual CPU will claim the feature is supported regardless of it being supported by host CPU.
+	// require  - Guest creation will fail unless the feature is supported by the host CPU or the hypervisor is able to emulate it.
+	// optional - The feature will be supported by virtual CPU if and only if it is supported by host CPU.
+	// disable  - The feature will not be supported by virtual CPU.
+	// forbid   - Guest creation will fail if the feature is supported by host CPU.
+	// Defaults to require
+	// +optional
+	Policy string `json:"policy,omitempty"`
+}
+
 // Memory allows specifying the VirtualMachineInstance memory features.
 // ---
 // +k8s:openapi-gen=true

pkg/api/v1/schema_test.go

@@ -54,6 +54,16 @@ var exampleJSON = `{
         "sockets": 1,
         "threads": 1,
         "model": "Conroe",
+        "features": [
+          {
+            "name": "pcid",
+            "policy": "require"
+          },
+          {
+            "name": "monitor",
+            "policy": "disable"
+          }
+        ],
         "dedicatedCpuPlacement": true
       },
       "machine": {
@@ -349,10 +359,20 @@ var _ = Describe("Schema", func() {
 			UUID: "28a42a60-44ef-4428-9c10-1a6aee94627f",
 		}
 		exampleVMI.Spec.Domain.CPU = &CPU{
-			Cores:                 3,
-			Sockets:               1,
-			Threads:               1,
-			Model:                 "Conroe",
+			Cores:   3,
+			Sockets: 1,
+			Threads: 1,
+			Model:   "Conroe",
+			Features: []Feature{
+				{
+					Name:   "pcid",
+					Policy: "require",
+				},
+				{
+					Name:   "monitor",
+					Policy: "disable",
+				},
+			},
 			DedicatedCPUPlacement: true,
 		}
 		exampleVMI.Spec.Networks = []Network{

pkg/virt-controller/services/template.go

@@ -43,7 +43,7 @@ import (
 	"kubevirt.io/kubevirt/pkg/util/hardware"
 	"kubevirt.io/kubevirt/pkg/util/net/dns"
 	"kubevirt.io/kubevirt/pkg/util/types"
-	"kubevirt.io/kubevirt/pkg/virt-config"
+	virtconfig "kubevirt.io/kubevirt/pkg/virt-config"
 )
 
 const configMapName = "kubevirt-config"
@@ -61,9 +61,10 @@ const CAP_SYS_NICE = "SYS_NICE"
 // Libvirt needs roughly 10 seconds to start.
 const LibvirtStartupDelay = 10
 
-//This is a perfix for node feature discovery, used in a NodeSelector on the pod
-//to match a VirtualMachineInstance CPU model(Family) to nodes that support this model.
+//These perfixes for node feature discovery, are used in a NodeSelector on the pod
+//to match a VirtualMachineInstance CPU model(Family) and/or features to nodes that support them.
 const NFD_CPU_MODEL_PREFIX = "feature.node.kubernetes.io/cpu-model-"
+const NFD_CPU_FEATURE_PREFIX = "feature.node.kubernetes.io/cpu-feature-"
 
 const MULTUS_RESOURCE_NAME_ANNOTATION = "k8s.v1.cni.cncf.io/resourceName"
 
@@ -168,6 +169,18 @@ func CPUModelLabelFromCPUModel(vmi *v1.VirtualMachineInstance) (label string, er
 	return
 }
 
+func CPUFeatureLabelsFromCPUFeatures(vmi *v1.VirtualMachineInstance) []string {
+	var labels []string
+	if vmi.Spec.Domain.CPU != nil && vmi.Spec.Domain.CPU.Features != nil {
+		if len(vmi.Spec.Domain.CPU.Features) > 0 {
+			for _, feature := range vmi.Spec.Domain.CPU.Features {
+				labels = append(labels, NFD_CPU_FEATURE_PREFIX+feature.Name)
+			}
+		}
+	}
+	return labels
+}
+
 // Request a resource by name. This function bumps the number of resources,
 // both its limits and requests attributes.
 //
@@ -699,6 +712,9 @@ func (t *templateService) RenderLaunchManifest(vmi *v1.VirtualMachineInstance) (
 				nodeSelector[cpuModelLabel] = "true"
 			}
 		}
+		for _, cpuFeatureLable := range CPUFeatureLabelsFromCPUFeatures(vmi) {
+			nodeSelector[cpuFeatureLable] = "true"
+		}
 	}
 
 	nodeSelector[v1.NodeSchedulable] = "true"

pkg/virt-controller/services/template_test.go

@@ -41,7 +41,7 @@ import (
 	"kubevirt.io/kubevirt/pkg/hooks"
 	"kubevirt.io/kubevirt/pkg/kubecli"
 	"kubevirt.io/kubevirt/pkg/log"
-	"kubevirt.io/kubevirt/pkg/virt-config"
+	virtconfig "kubevirt.io/kubevirt/pkg/virt-config"
 )
 
 const namespaceKubevirt = "kubevirt"
@@ -271,17 +271,32 @@ var _ = Describe("Template", func() {
 						Domain: v1.DomainSpec{
 							CPU: &v1.CPU{
 								Model: "Conroe",
+								Features: []v1.Feature{
+									{
+										Name:   "lahf_lm",
+										Policy: "require",
+									},
+									{
+										Name:   "mmx",
+										Policy: "disable",
+									},
+								},
 							},
 						},
 					},
 				}
 
-				cpuModelLabel, err := CPUModelLabelFromCPUModel(&vmi)
-				Expect(err).ToNot(HaveOccurred())
 				pod, err := svc.RenderLaunchManifest(&vmi)
 				Expect(err).ToNot(HaveOccurred())
 
+				cpuModelLabel, err := CPUModelLabelFromCPUModel(&vmi)
+				Expect(err).ToNot(HaveOccurred())
 				Expect(pod.Spec.NodeSelector).Should(HaveKeyWithValue(cpuModelLabel, "true"))
+
+				cpuFeatureLabels := CPUFeatureLabelsFromCPUFeatures(&vmi)
+				for _, featureLabel := range cpuFeatureLabels {
+					Expect(pod.Spec.NodeSelector).Should(HaveKeyWithValue(featureLabel, "true"))
+				}
 			})
 
 			It("should add default cpu/memory resources to the sidecar container if cpu pinning was requested", func() {

pkg/virt-launcher/virtwrap/api/converter.go

@@ -871,6 +871,18 @@ func Convert_v1_VirtualMachine_To_api_Domain(vmi *v1.VirtualMachineInstance, dom
 			}
 		}
 
+		// Set VM CPU features
+		if vmi.Spec.Domain.CPU.Features != nil {
+			if len(vmi.Spec.Domain.CPU.Features) > 0 {
+				for _, feature := range vmi.Spec.Domain.CPU.Features {
+					domainFeature := Feature{}
+					domainFeature.Name = feature.Name
+					domainFeature.Policy = feature.Policy
+					domain.Spec.CPU.Features = append(domain.Spec.CPU.Features, domainFeature)
+				}
+			}
+		}
+
 		// Adjust guest vcpu config. Currenty will handle vCPUs to pCPUs pinning
 		if vmi.IsCPUDedicated() {
 			if err := formatDomainCPUTune(vmi, domain, c); err != nil {

pkg/virt-launcher/virtwrap/api/converter_test.go

@@ -559,13 +559,23 @@ var _ = Describe("Converter", func() {
 		})
 
 		Context("when CPU spec defined", func() {
-			It("should convert CPU cores and model", func() {
+			It("should convert CPU cores, model and features", func() {
 				v1.SetObjectDefaults_VirtualMachineInstance(vmi)
 				vmi.Spec.Domain.CPU = &v1.CPU{
 					Cores:   3,
 					Sockets: 2,
 					Threads: 2,
 					Model:   "Conroe",
+					Features: []v1.Feature{
+						{
+							Name:   "lahf_lm",
+							Policy: "require",
+						},
+						{
+							Name:   "mmx",
+							Policy: "disable",
+						},
+					},
 				}
 				domainSpec := vmiToDomainXMLToDomainSpec(vmi, c)
 
@@ -574,6 +584,10 @@ var _ = Describe("Converter", func() {
 				Expect(domainSpec.CPU.Topology.Threads).To(Equal(uint32(2)), "Expect threads")
 				Expect(domainSpec.CPU.Mode).To(Equal("custom"), "Expect cpu mode")
 				Expect(domainSpec.CPU.Model).To(Equal("Conroe"), "Expect cpu model")
+				Expect(domainSpec.CPU.Features[0].Name).To(Equal("lahf_lm"), "Expect cpu feature name")
+				Expect(domainSpec.CPU.Features[0].Policy).To(Equal("require"), "Expect cpu feature policy")
+				Expect(domainSpec.CPU.Features[1].Name).To(Equal("mmx"), "Expect cpu feature name")
+				Expect(domainSpec.CPU.Features[1].Policy).To(Equal("disable"), "Expect cpu feature policy")
 				Expect(domainSpec.VCPU.Placement).To(Equal("static"), "Expect vcpu placement")
 				Expect(domainSpec.VCPU.CPUs).To(Equal(uint32(12)), "Expect vcpus")
 			})