add 1G of memory overhead for vfio devices

Signed-off-by: Vladik Romanovsky <[email protected]>
xiaoruiguo · Mar 20, 2020 · cf885d6 · cf885d6
1 parent fb3564a
commit cf885d6
Show file tree

Hide file tree

Showing 2 changed files with 31 additions and 2 deletions.
diff --git a/pkg/virt-controller/services/template.go b/pkg/virt-controller/services/template.go
@@ -561,7 +561,7 @@ func (t *templateService) RenderLaunchManifest(vmi *v1.VirtualMachineInstance) (
 	gracePeriodKillAfter := gracePeriodSeconds + int64(15)
 
 	// Get memory overhead
-	memoryOverhead := getMemoryOverhead(vmi.Spec.Domain)
+	memoryOverhead := getMemoryOverhead(vmi)
 
 	// Consider CPU and memory requests and limits for pod scheduling
 	resources := k8sv1.ResourceRequirements{}
@@ -1066,7 +1066,8 @@ func appendUniqueImagePullSecret(secrets []k8sv1.LocalObjectReference, newsecret
 //
 // Note: This is the best estimation we were able to come up with
 //       and is still not 100% accurate
-func getMemoryOverhead(domain v1.DomainSpec) *resource.Quantity {
+func getMemoryOverhead(vmi *v1.VirtualMachineInstance) *resource.Quantity {
+	domain := vmi.Spec.Domain
 	vmiMemoryReq := domain.Resources.Requests.Memory()
 
 	overhead := resource.NewScaledQuantity(0, resource.Kilo)
@@ -1098,6 +1099,13 @@ func getMemoryOverhead(domain v1.DomainSpec) *resource.Quantity {
 		overhead.Add(resource.MustParse("16Mi"))
 	}
 
+	// Additional overhead of 1G for VFIO devices. VFIO requires all guest RAM to be locked
+	// in addition to MMIO memory space to allow DMA. 1G is often the size of reserved MMIO space on x86 systems.
+	// Additial information can be found here: https://www.redhat.com/archives/libvir-list/2015-November/msg00329.html
+	if util.IsSRIOVVmi(vmi) || util.IsGPUVMI(vmi) {
+		overhead.Add(resource.MustParse("1G"))
+	}
+
 	return overhead
 }
 

diff --git a/pkg/virt-controller/services/template_test.go b/pkg/virt-controller/services/template_test.go
@@ -1293,6 +1293,27 @@ var _ = Describe("Template", func() {
 				Expect(pod.Spec.Containers[0].VolumeMounts[4].MountPath).To(Equal("/sys/devices/"))
 				Expect(pod.Spec.Volumes[0].HostPath.Path).To(Equal("/sys/devices/"))
 			})
+			It("should add 1G of memory overhead", func() {
+				sriovInterface := v1.InterfaceSRIOV{}
+				domain := v1.DomainSpec{
+					Resources: v1.ResourceRequirements{
+						Requests: kubev1.ResourceList{
+							kubev1.ResourceMemory: resource.MustParse("1G"),
+						},
+					},
+				}
+				domain.Devices.Interfaces = []v1.Interface{{Name: "testnet", InterfaceBindingMethod: v1.InterfaceBindingMethod{SRIOV: &sriovInterface}}}
+				vmi := v1.VirtualMachineInstance{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "testvmi", Namespace: "default", UID: "1234",
+					},
+					Spec: v1.VirtualMachineInstanceSpec{Domain: domain},
+				}
+
+				pod, err := svc.RenderLaunchManifest(&vmi)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(pod.Spec.Containers[0].Resources.Requests.Memory().String()).To(Equal("2163507557"))
+			})
 		})
 		Context("with slirp interface", func() {
 			It("Should have empty port list in the pod manifest", func() {