Merge pull request kubevirt#3178 from vladikr/increase_sriov_overhead

Increase memory overhead for VFIO devices

pkg/virt-controller/services/template.go

@@ -561,7 +561,7 @@ func (t *templateService) RenderLaunchManifest(vmi *v1.VirtualMachineInstance) (
 	gracePeriodKillAfter := gracePeriodSeconds + int64(15)
 
 	// Get memory overhead
-	memoryOverhead := getMemoryOverhead(vmi.Spec.Domain)
+	memoryOverhead := getMemoryOverhead(vmi)
 
 	// Consider CPU and memory requests and limits for pod scheduling
 	resources := k8sv1.ResourceRequirements{}
@@ -1095,7 +1095,8 @@ func appendUniqueImagePullSecret(secrets []k8sv1.LocalObjectReference, newsecret
 //
 // Note: This is the best estimation we were able to come up with
 //       and is still not 100% accurate
-func getMemoryOverhead(domain v1.DomainSpec) *resource.Quantity {
+func getMemoryOverhead(vmi *v1.VirtualMachineInstance) *resource.Quantity {
+	domain := vmi.Spec.Domain
 	vmiMemoryReq := domain.Resources.Requests.Memory()
 
 	overhead := resource.NewScaledQuantity(0, resource.Kilo)
@@ -1127,6 +1128,13 @@ func getMemoryOverhead(domain v1.DomainSpec) *resource.Quantity {
 		overhead.Add(resource.MustParse("16Mi"))
 	}
 
+	// Additional overhead of 1G for VFIO devices. VFIO requires all guest RAM to be locked
+	// in addition to MMIO memory space to allow DMA. 1G is often the size of reserved MMIO space on x86 systems.
+	// Additial information can be found here: https://www.redhat.com/archives/libvir-list/2015-November/msg00329.html
+	if util.IsSRIOVVmi(vmi) || util.IsGPUVMI(vmi) {
+		overhead.Add(resource.MustParse("1G"))
+	}
+
 	return overhead
 }
 

pkg/virt-controller/services/template_test.go

@@ -1342,6 +1342,30 @@ var _ = Describe("Template", func() {
 				Expect(pod.Spec.Containers[0].VolumeMounts[4].MountPath).To(Equal("/sys/devices/"))
 				Expect(pod.Spec.Volumes[0].HostPath.Path).To(Equal("/sys/devices/"))
 			})
+			It("should add 1G of memory overhead", func() {
+				sriovInterface := v1.InterfaceSRIOV{}
+				domain := v1.DomainSpec{
+					Resources: v1.ResourceRequirements{
+						Requests: kubev1.ResourceList{
+							kubev1.ResourceMemory: resource.MustParse("1G"),
+						},
+					},
+				}
+				domain.Devices.Interfaces = []v1.Interface{{Name: "testnet", InterfaceBindingMethod: v1.InterfaceBindingMethod{SRIOV: &sriovInterface}}}
+				vmi := v1.VirtualMachineInstance{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "testvmi", Namespace: "default", UID: "1234",
+					},
+					Spec: v1.VirtualMachineInstanceSpec{Domain: domain},
+				}
+
+				pod, err := svc.RenderLaunchManifest(&vmi)
+				Expect(err).ToNot(HaveOccurred())
+				expectedMemory := resource.NewScaledQuantity(0, resource.Kilo)
+				expectedMemory.Add(*getMemoryOverhead(&vmi))
+				expectedMemory.Add(*domain.Resources.Requests.Memory())
+				Expect(pod.Spec.Containers[0].Resources.Requests.Memory().Value()).To(Equal(expectedMemory.Value()))
+			})
 		})
 		Context("with slirp interface", func() {
 			It("Should have empty port list in the pod manifest", func() {

tests/vmi_multus_test.go

@@ -747,6 +747,37 @@ var _ = Describe("SRIOV", func() {
 			Expect(rootPortController).To(HaveLen(0), "libvirt should not add additional buses to the root one")
 		})
 
+		It("should create a virtual machine with sriov interface and dedicatedCPUs", func() {
+			// In addition to verifying that we can start a VMI with CPU pinning
+			// this also tests if we've correctly calculated the overhead for VFIO devices.
+			vmi := getSriovVmi([]string{"sriov"})
+			vmi.Spec.Domain.CPU = &v1.CPU{
+				Cores:                 2,
+				DedicatedCPUPlacement: true,
+			}
+			startVmi(vmi)
+			waitVmi(vmi)
+
+			By("checking KUBEVIRT_RESOURCE_NAME_<networkName> variable is defined in pod")
+			vmiPod := tests.GetRunningPodByVirtualMachineInstance(vmi, tests.NamespaceTestDefault)
+			out, err := tests.ExecuteCommandOnPod(
+				virtClient,
+				vmiPod,
+				"compute",
+				[]string{"sh", "-c", "echo $KUBEVIRT_RESOURCE_NAME_sriov"},
+			)
+			Expect(err).ToNot(HaveOccurred())
+
+			expectedSriovResourceName := fmt.Sprintf("%s\n", sriovResourceName)
+			Expect(out).To(Equal(expectedSriovResourceName))
+
+			checkDefaultInterfaceInPod(vmi)
+
+			By("checking virtual machine instance has two interfaces")
+			checkInterfacesInGuest(vmi, []string{"eth0", "eth1"})
+
+		})
+
 		It("should create a virtual machine with sriov interface with custom MAC address", func() {
 			vmi := getSriovVmi([]string{"sriov"})
 			vmi.Spec.Domain.Devices.Interfaces[1].MacAddress = "de:ad:00:00:be:ef"