Skip to content

Commit

Permalink
virt-launcher: setup memory hotplug
Browse files Browse the repository at this point in the history 
Signed-off-by: Antonio Cardace <[email protected]>
  • Loading branch information
@acardace
acardace committed Sep 27, 2023
1 parent f9c3e75 commit ae0085c
Show file tree
Hide file tree
Showing 2 changed files with 240 additions and 74 deletions.
91 changes: 82 additions & 9 deletions pkg/virt-launcher/virtwrap/converter/converter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,13 @@ const (
vhostNetPath = "/dev/vhost-net"
)

const (
// must be a power of 2 and at least equal
// to the size of a transparent hugepage (2MiB on x84_64).
// Recommended value by QEMU is 2MiB
MemoryHotplugBlockAlignmentBytes = 2097152
)

var (
BootMenuTimeoutMS = uint(10000)
)
Expand Down Expand Up @@ -1221,6 +1228,69 @@ func isUSBNeeded(c *ConverterContext, vmi *v1.VirtualMachineInstance) bool {
return false
}

func setupDomainMemory(vmi *v1.VirtualMachineInstance, domain *api.Domain) error {
if vmi.Spec.Domain.Memory == nil ||
vmi.Spec.Domain.Memory.MaxGuest == nil ||
vmi.Spec.Domain.Memory.Guest.Equal(*vmi.Spec.Domain.Memory.MaxGuest) {
var err error

domain.Spec.Memory, err = vcpu.QuantityToByte(*vcpu.GetVirtualMemory(vmi))
if err != nil {
return err
}
return nil
}

maxMemory, err := vcpu.QuantityToByte(*vmi.Spec.Domain.Memory.MaxGuest)
if err != nil {
return err
}

domain.Spec.MaxMemory = &api.MaxMemory{
Unit: maxMemory.Unit,
Value: maxMemory.Value,
Slots: 1,
}

domain.Spec.Memory = maxMemory

initialMemoryApi, err := vcpu.QuantityToByte(*vmi.Status.Memory.GuestAtBoot)
if err != nil {
return err
}

// The usage of memory devices requires a NUMA configuration for
// the domain, even if it's just a single NUMA node
domain.Spec.CPU.NUMA = &api.NUMA{
Cells: []api.NUMACell{
{
ID: "0",
CPUs: fmt.Sprintf("0-%d", domain.Spec.VCPU.CPUs-1),
Memory: initialMemoryApi.Value,
Unit: initialMemoryApi.Unit,
},
},
}

pluggableMemory := vmi.Spec.Domain.Memory.MaxGuest.DeepCopy()
pluggableMemory.Sub(*vmi.Status.Memory.GuestAtBoot)
pluggableMemorySize, err := vcpu.QuantityToByte(pluggableMemory)
if err != nil {
return err
}

domain.Spec.Devices.Memory = &api.MemoryDevice{
Model: "virtio-mem",
Target: &api.MemoryTarget{
Size: pluggableMemorySize,
Node: "0",
Block: api.Memory{Unit: "b", Value: MemoryHotplugBlockAlignmentBytes},
},
}

return nil
}

func Convert_v1_VirtualMachineInstance_To_api_Domain(vmi *v1.VirtualMachineInstance, domain *api.Domain, c *ConverterContext) (err error) {
var controllerDriver *api.ControllerDriver

Expand Down Expand Up @@ -1424,7 +1494,7 @@ func Convert_v1_VirtualMachineInstance_To_api_Domain(vmi *v1.VirtualMachineInsta
}
}

if domain.Spec.Memory, err = vcpu.QuantityToByte(*vcpu.GetVirtualMemory(vmi)); err != nil {
if err = setupDomainMemory(vmi, domain); err != nil {
return err
}

Expand Down Expand Up @@ -1452,16 +1522,19 @@ func Convert_v1_VirtualMachineInstance_To_api_Domain(vmi *v1.VirtualMachineInsta
// Set memfd as memory backend to solve SELinux restrictions
// See the issue: https://github.com/kubevirt/kubevirt/issues/3781
domain.Spec.MemoryBacking.Source = &api.MemoryBackingSource{Type: "memfd"}

// NUMA is required in order to use memfd
domain.Spec.CPU.NUMA = &api.NUMA{
Cells: []api.NUMACell{
{
ID: "0",
CPUs: fmt.Sprintf("0-%d", domain.Spec.VCPU.CPUs-1),
Memory: uint64(vcpu.GetVirtualMemory(vmi).Value() / int64(1024)),
Unit: "KiB",
if domain.Spec.CPU.NUMA == nil {
domain.Spec.CPU.NUMA = &api.NUMA{
Cells: []api.NUMACell{
{
ID: "0",
CPUs: fmt.Sprintf("0-%d", domain.Spec.VCPU.CPUs-1),
Memory: uint64(vcpu.GetVirtualMemory(vmi).Value() / int64(1024)),
Unit: "KiB",
},
},
},
}
}
}

Expand Down
223 changes: 158 additions & 65 deletions pkg/virt-launcher/virtwrap/converter/converter_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2375,83 +2375,176 @@ var _ = Describe("Converter", func() {
var vmi *v1.VirtualMachineInstance
var c *ConverterContext

type ConverterFunc = func(name string, disk *api.Disk, c *ConverterContext) error
Context("disk", func() {

BeforeEach(func() {
vmi = &v1.VirtualMachineInstance{
ObjectMeta: k8smeta.ObjectMeta{
Name: "testvmi",
Namespace: "mynamespace",
},
}
type ConverterFunc = func(name string, disk *api.Disk, c *ConverterContext) error

v1.SetObjectDefaults_VirtualMachineInstance(vmi)
BeforeEach(func() {
vmi = &v1.VirtualMachineInstance{
ObjectMeta: k8smeta.ObjectMeta{
Name: "testvmi",
Namespace: "mynamespace",
},
}

c = &ConverterContext{
VirtualMachine: vmi,
AllowEmulation: true,
IsBlockPVC: map[string]bool{
"test-block-pvc": true,
},
IsBlockDV: map[string]bool{
"test-block-dv": true,
},
VolumesDiscardIgnore: []string{
"test-discard-ignore",
},
}
})
v1.SetObjectDefaults_VirtualMachineInstance(vmi)

It("should automatically add virtio-scsi controller", func() {
domain := vmiToDomain(vmi, c)
Expect(domain.Spec.Devices.Controllers).To(HaveLen(3))
foundScsiController := false
for _, controller := range domain.Spec.Devices.Controllers {
if controller.Type == "scsi" {
foundScsiController = true
Expect(controller.Model).To(Equal("virtio-non-transitional"))
c = &ConverterContext{
VirtualMachine: vmi,
AllowEmulation: true,
IsBlockPVC: map[string]bool{
"test-block-pvc": true,
},
IsBlockDV: map[string]bool{
"test-block-dv": true,
},
VolumesDiscardIgnore: []string{
"test-discard-ignore",
},
}
})

It("should automatically add virtio-scsi controller", func() {
domain := vmiToDomain(vmi, c)
Expect(domain.Spec.Devices.Controllers).To(HaveLen(3))
foundScsiController := false
for _, controller := range domain.Spec.Devices.Controllers {
if controller.Type == "scsi" {
foundScsiController = true
Expect(controller.Model).To(Equal("virtio-non-transitional"))

}
}
}
Expect(foundScsiController).To(BeTrue(), "did not find SCSI controller when expected")
Expect(foundScsiController).To(BeTrue(), "did not find SCSI controller when expected")
})

It("should not automatically add virtio-scsi controller, if hotplug disabled", func() {
vmi.Spec.Domain.Devices.DisableHotplug = true
domain := vmiToDomain(vmi, c)
Expect(domain.Spec.Devices.Controllers).To(HaveLen(2))
})

DescribeTable("should convert",
func(converterFunc ConverterFunc, volumeName string, isBlockMode bool, ignoreDiscard bool) {
expectedDisk := &api.Disk{}
expectedDisk.Driver = &api.DiskDriver{}
expectedDisk.Driver.Type = "raw"
expectedDisk.Driver.ErrorPolicy = "stop"
if isBlockMode {
expectedDisk.Type = "block"
expectedDisk.Source.Dev = filepath.Join(v1.HotplugDiskDir, volumeName)
} else {
expectedDisk.Type = "file"
expectedDisk.Source.File = fmt.Sprintf("%s.img", filepath.Join(v1.HotplugDiskDir, volumeName))
}
if !ignoreDiscard {
expectedDisk.Driver.Discard = "unmap"
}

disk := &api.Disk{
Driver: &api.DiskDriver{},
}
Expect(converterFunc(volumeName, disk, c)).To(Succeed())
Expect(disk).To(Equal(expectedDisk))
},
Entry("filesystem PVC", Convert_v1_Hotplug_PersistentVolumeClaim_To_api_Disk, "test-fs-pvc", false, false),
Entry("block mode PVC", Convert_v1_Hotplug_PersistentVolumeClaim_To_api_Disk, "test-block-pvc", true, false),
Entry("'discard ignore' PVC", Convert_v1_Hotplug_PersistentVolumeClaim_To_api_Disk, "test-discard-ignore", false, true),
Entry("filesystem DV", Convert_v1_Hotplug_DataVolume_To_api_Disk, "test-fs-dv", false, false),
Entry("block mode DV", Convert_v1_Hotplug_DataVolume_To_api_Disk, "test-block-dv", true, false),
Entry("'discard ignore' DV", Convert_v1_Hotplug_DataVolume_To_api_Disk, "test-discard-ignore", false, true),
)
})

It("should not automatically add virtio-scsi controller, if hotplug disabled", func() {
vmi.Spec.Domain.Devices.DisableHotplug = true
domain := vmiToDomain(vmi, c)
Expect(domain.Spec.Devices.Controllers).To(HaveLen(2))
})

DescribeTable("should convert",
func(converterFunc ConverterFunc, volumeName string, isBlockMode bool, ignoreDiscard bool) {
expectedDisk := &api.Disk{}
expectedDisk.Driver = &api.DiskDriver{}
expectedDisk.Driver.Type = "raw"
expectedDisk.Driver.ErrorPolicy = "stop"
if isBlockMode {
expectedDisk.Type = "block"
expectedDisk.Source.Dev = filepath.Join(v1.HotplugDiskDir, volumeName)
} else {
expectedDisk.Type = "file"
expectedDisk.Source.File = fmt.Sprintf("%s.img", filepath.Join(v1.HotplugDiskDir, volumeName))
Context("memory", func() {
var domain *api.Domain
var guestMemory resource.Quantity
var maxGuestMemory resource.Quantity

BeforeEach(func() {
guestMemory = resource.MustParse("32Mi")
maxGuestMemory = resource.MustParse("128Mi")

vmi = &v1.VirtualMachineInstance{
ObjectMeta: k8smeta.ObjectMeta{
Name: "testvmi",
Namespace: "mynamespace",
},
Spec: v1.VirtualMachineInstanceSpec{
Domain: v1.DomainSpec{
Memory: &v1.Memory{
Guest: &guestMemory,
MaxGuest: &maxGuestMemory,
},
},
},
Status: v1.VirtualMachineInstanceStatus{
Memory: &v1.MemoryStatus{
GuestAtBoot: &guestMemory,
GuestCurrent: &guestMemory,
},
},
}
if !ignoreDiscard {
expectedDisk.Driver.Discard = "unmap"

domain = &api.Domain{
Spec: api.DomainSpec{
VCPU: &api.VCPU{
CPUs: 2,
},
},
}

disk := &api.Disk{
Driver: &api.DiskDriver{},
v1.SetObjectDefaults_VirtualMachineInstance(vmi)

c = &ConverterContext{
VirtualMachine: vmi,
AllowEmulation: true,
}
Expect(converterFunc(volumeName, disk, c)).To(Succeed())
Expect(disk).To(Equal(expectedDisk))
},
Entry("filesystem PVC", Convert_v1_Hotplug_PersistentVolumeClaim_To_api_Disk, "test-fs-pvc", false, false),
Entry("block mode PVC", Convert_v1_Hotplug_PersistentVolumeClaim_To_api_Disk, "test-block-pvc", true, false),
Entry("'discard ignore' PVC", Convert_v1_Hotplug_PersistentVolumeClaim_To_api_Disk, "test-discard-ignore", false, true),
Entry("filesystem DV", Convert_v1_Hotplug_DataVolume_To_api_Disk, "test-fs-dv", false, false),
Entry("block mode DV", Convert_v1_Hotplug_DataVolume_To_api_Disk, "test-block-dv", true, false),
Entry("'discard ignore' DV", Convert_v1_Hotplug_DataVolume_To_api_Disk, "test-discard-ignore", false, true),
)
})

It("should not setup hotplug when maxGuest is missing", func() {
vmi.Spec.Domain.Memory.MaxGuest = nil
err := setupDomainMemory(vmi, domain)
Expect(err).ToNot(HaveOccurred())
Expect(domain.Spec.MaxMemory).To(BeNil())
})

It("should not setup hotplug when maxGuest equals guest memory", func() {
vmi.Spec.Domain.Memory.MaxGuest = &guestMemory
err := setupDomainMemory(vmi, domain)
Expect(err).ToNot(HaveOccurred())
Expect(domain.Spec.MaxMemory).To(BeNil())
})

It("should setup hotplug when maxGuest is set", func() {
err := setupDomainMemory(vmi, domain)
Expect(err).ToNot(HaveOccurred())

Expect(domain.Spec.MaxMemory).ToNot(BeNil())
Expect(domain.Spec.MaxMemory.Unit).To(Equal("b"))
Expect(domain.Spec.MaxMemory.Value).To(Equal(uint64(maxGuestMemory.Value())))

Expect(domain.Spec.Memory).ToNot(BeNil())
Expect(domain.Spec.Memory.Unit).To(Equal("b"))
Expect(domain.Spec.Memory.Value).To(Equal(uint64(maxGuestMemory.Value())))

Expect(domain.Spec.CPU.NUMA).ToNot(BeNil())
Expect(domain.Spec.CPU.NUMA.Cells).To(HaveLen(1))
Expect(domain.Spec.CPU.NUMA.Cells[0].Unit).To(Equal("b"))
Expect(domain.Spec.CPU.NUMA.Cells[0].Memory).To(Equal(uint64(guestMemory.Value())))

pluggableMemory := uint64(maxGuestMemory.Value() - guestMemory.Value())

Expect(domain.Spec.Devices.Memory).ToNot(BeNil())
Expect(domain.Spec.Devices.Memory.Model).To(Equal("virtio-mem"))
Expect(domain.Spec.Devices.Memory.Target).ToNot(BeNil())
Expect(domain.Spec.Devices.Memory.Target.Node).To(Equal("0"))
Expect(domain.Spec.Devices.Memory.Target.Size.Value).To(Equal(pluggableMemory))
Expect(domain.Spec.Devices.Memory.Target.Size.Unit).To(Equal("b"))
Expect(domain.Spec.Devices.Memory.Target.Block.Value).To(Equal(uint64(MemoryHotplugBlockAlignmentBytes)))
Expect(domain.Spec.Devices.Memory.Target.Block.Unit).To(Equal("b"))
})
})
})

Context("with AMD SEV LaunchSecurity", func() {
Expand Down

0 comments on commit ae0085c

Please sign in to comment.