Add TPM 2.0 emulated device option to VMIs

TPM emulation through libvirt (prior to version 8.1.0) requires the SYS_PTRACE capability. If we only ran as root, it would just be a matter of starting virt-launcher with that capability when TPM is enabled. However, to be compatible with non-root, the capability has to be added to the binary, and therefore to every virt-launcher instance. To do that, bazeldnf had to be upgraded. Non-root compatibility also requires /var/lib/swtpm-localca to be read-writable as the qemu user. For the functional test, the fedora containerdisk had to be updated to a version that contains the TPM2 tools. With all that out of the way, enabling TPM is just a matter of adding the right XML block to libvirt domains. Signed-off-by: Jed Lejosne <[email protected]>
attlee-wang · Apr 5, 2022 · 665ec67 · 665ec67
1 parent d578c2e
commit 665ec67
Show file tree

Hide file tree

Showing 23 changed files with 304 additions and 33 deletions.
diff --git a/WORKSPACE b/WORKSPACE
@@ -128,11 +128,11 @@ http_file(
 
 http_archive(
     name = "bazeldnf",
-    sha256 = "6a2af09c6a598a3c4e4fec9af78334fbec2b3c16473f4e2c692fe2e567dc6f56",
-    strip_prefix = "bazeldnf-0.5.1",
+    sha256 = "c37709d05ad7eae4d32d7a525f098fd026483ada5e11cdf84d47028222796605",
+    strip_prefix = "bazeldnf-0.5.2",
     urls = [
-        "https://github.com/rmohr/bazeldnf/archive/v0.5.1.tar.gz",
-        "https://storage.googleapis.com/builddeps/6a2af09c6a598a3c4e4fec9af78334fbec2b3c16473f4e2c692fe2e567dc6f56",
+        "https://github.com/rmohr/bazeldnf/archive/v0.5.2.tar.gz",
+        "https://storage.googleapis.com/builddeps/c37709d05ad7eae4d32d7a525f098fd026483ada5e11cdf84d47028222796605",
     ],
 )
 
@@ -333,9 +333,9 @@ container_pull(
 
 container_pull(
     name = "fedora_with_test_tooling_aarch64",
-    digest = "sha256:9ec3e137bff093597d192f5a4e346f25b614c3a94216b857de0e3d75b68bfb17",
+    digest = "sha256:9b1371260c05086a24ac9effdbedca9759c885ea8db93de7f0339df3bcd5a5c3",
     registry = "quay.io",
-    repository = "kubevirt/fedora-with-test-tooling",
+    repository = "kubevirtci/fedora-with-test-tooling",
 )
 
 container_pull(

diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json
@@ -12933,6 +12933,10 @@
       "description": "Whether to emulate a sound device.",
       "$ref": "#/definitions/v1.SoundDevice"
      },
+     "tpm": {
+      "description": "Whether to emulate a TPM device.",
+      "$ref": "#/definitions/v1.TPMDevice"
+     },
      "useVirtioTransitional": {
       "description": "Fall back to legacy virtio 0.9 support if virtio bus is selected on devices. This is helpful for old machines like CentOS6 or RHEL6 which do not understand virtio_non_transitional (virtio 1.0).",
       "type": "boolean"
@@ -14934,6 +14938,9 @@
      }
     }
    },
+   "v1.TPMDevice": {
+    "type": "object"
+   },
    "v1.Timer": {
     "description": "Represents all available timers in a vmi.",
     "type": "object",

diff --git a/cmd/virt-launcher/BUILD.bazel b/cmd/virt-launcher/BUILD.bazel
@@ -69,6 +69,7 @@ xattrs(
     capabilities = {
         "/usr/bin/virt-launcher": [
             "cap_net_bind_service",
+            "cap_sys_ptrace",
         ],
     },
     selinux_labels = {
@@ -110,7 +111,7 @@ group_file(
 passwd_entry(
     name = "qemu-user",
     gid = 107,
-    home = "",
+    home = "/home/qemu",
     shell = "/bin/bash",
     uid = 107,
     username = "qemu",
@@ -153,6 +154,15 @@ pkg_tar(
     package_dir = "/etc",
 )
 
+pkg_tar(
+    name = "swtpm-localca-tar",
+    empty_dirs = [
+        "var/lib/swtpm-localca",
+    ],
+    mode = "0750",
+    owner = "107.107",
+)
+
 container_image(
     name = "version-container",
     directory = "/",
@@ -162,12 +172,14 @@ container_image(
             ":libvirt-config",
             ":passwd-tar",
             ":nsswitch-tar",
+            ":swtpm-localca-tar",
             "//rpm:launcherbase_aarch64",
         ],
         "//conditions:default": [
             ":libvirt-config",
             ":passwd-tar",
             ":nsswitch-tar",
+            "swtpm-localca-tar",
             "//rpm:launcherbase_x86_64",
         ],
     }),

diff --git a/cmd/virt-launcher/virt-launcher.go b/cmd/virt-launcher/virt-launcher.go
@@ -473,7 +473,12 @@ func main() {
 
 	domain := waitForDomainUUID(*qemuTimeout, events, signalStopChan, domainManager)
 	if domain != nil {
-		mon := virtlauncher.NewProcessMonitor(domain.Spec.UUID,
+		// The first argument to NewProcessMonitor will end up being grepped in /proc/*/cmdline
+		// to find what is hopefully the qemu process.
+		// If any other process includes that string, it might wrongly be considered.
+		// FIXME: just read the pidfile libvirt creates for qemu instead of grepping /proc
+		// See: https://github.com/kubevirt/kubevirt/issues/7067
+		mon := virtlauncher.NewProcessMonitor("uuid="+domain.Spec.UUID,
 			*gracePeriodSeconds,
 			finalShutdownCallback,
 			gracefulShutdownCallback)

diff --git a/examples/vm-template-windows2012r2.yaml b/examples/vm-template-windows2012r2.yaml
@@ -53,6 +53,7 @@ objects:
             - masquerade: {}
               model: e1000
               name: default
+            tpm: {}
           features:
             acpi: {}
             apic: {}
@@ -61,7 +62,11 @@ objects:
               spinlocks:
                 spinlocks: 8191
               vapic: {}
+            smm: {}
           firmware:
+            bootloader:
+              efi:
+                secureBoot: true
             uuid: 5d307ca9-b3ef-428c-8861-06e72d69f223
           resources:
             requests:

diff --git a/examples/vmi-windows.yaml b/examples/vmi-windows.yaml
@@ -28,6 +28,7 @@ spec:
       - masquerade: {}
         model: e1000
         name: default
+      tpm: {}
     features:
       acpi: {}
       apic: {}
@@ -36,7 +37,11 @@ spec:
         spinlocks:
           spinlocks: 8191
         vapic: {}
+      smm: {}
     firmware:
+      bootloader:
+        efi:
+          secureBoot: true
       uuid: 5d307ca9-b3ef-428c-8861-06e72d69f223
     resources:
       requests:

diff --git a/pkg/virt-api/webhooks/validating-webhook/admitters/vmi-create-admitter.go b/pkg/virt-api/webhooks/validating-webhook/admitters/vmi-create-admitter.go
@@ -1705,7 +1705,7 @@ func validateDomainSpec(field *k8sfield.Path, spec *v1.DomainSpec) []metav1.Stat
 
 	if spec.Firmware != nil && spec.Firmware.Bootloader != nil && spec.Firmware.Bootloader.EFI != nil &&
 		(spec.Firmware.Bootloader.EFI.SecureBoot == nil || *spec.Firmware.Bootloader.EFI.SecureBoot) &&
-		(spec.Features == nil || spec.Features.SMM == nil || !*spec.Features.SMM.Enabled) {
+		(spec.Features == nil || spec.Features.SMM == nil || (spec.Features.SMM.Enabled != nil && !*spec.Features.SMM.Enabled)) {
 		causes = append(causes, metav1.StatusCause{
 			Type:    metav1.CauseTypeFieldValueInvalid,
 			Message: fmt.Sprintf("%s has EFI SecureBoot enabled. SecureBoot requires SMM, which is currently disabled.", field.String()),

diff --git a/pkg/virt-controller/services/template.go b/pkg/virt-controller/services/template.go
@@ -86,6 +86,7 @@ const (
 	CAP_NET_RAW          = "NET_RAW"
 	CAP_SYS_ADMIN        = "SYS_ADMIN"
 	CAP_SYS_NICE         = "SYS_NICE"
+	CAP_SYS_PTRACE       = "SYS_PTRACE"
 )
 
 // LibvirtStartupDelay is added to custom liveness and readiness probes initial delay value.
@@ -1805,20 +1806,25 @@ func haveSlirp(vmi *v1.VirtualMachineInstance) bool {
 }
 
 func getRequiredCapabilities(vmi *v1.VirtualMachineInstance) []k8sv1.Capability {
-	if util.IsNonRootVMI(vmi) {
-		return []k8sv1.Capability{CAP_NET_BIND_SERVICE}
-	}
-	capabilities := []k8sv1.Capability{}
-	if requireDHCP(vmi) || haveSlirp(vmi) {
+	var capabilities []k8sv1.Capability
+
+	if requireDHCP(vmi) || haveSlirp(vmi) || util.IsNonRootVMI(vmi) {
 		capabilities = append(capabilities, CAP_NET_BIND_SERVICE)
 	}
-	// add a CAP_SYS_NICE capability to allow setting cpu affinity
-	capabilities = append(capabilities, CAP_SYS_NICE)
-	// add CAP_SYS_ADMIN capability to allow virtiofs
-	if util.IsVMIVirtiofsEnabled(vmi) {
-		capabilities = append(capabilities, CAP_SYS_ADMIN)
-		capabilities = append(capabilities, getVirtiofsCapabilities()...)
+	if !util.IsNonRootVMI(vmi) {
+		// add a CAP_SYS_NICE capability to allow setting cpu affinity
+		capabilities = append(capabilities, CAP_SYS_NICE)
+		// add CAP_SYS_ADMIN capability to allow virtiofs
+		if util.IsVMIVirtiofsEnabled(vmi) {
+			capabilities = append(capabilities, CAP_SYS_ADMIN)
+			capabilities = append(capabilities, getVirtiofsCapabilities()...)
+		}
 	}
+	// add CAP_SYS_PTRACE capability needed by libvirt + swtpm
+	// TODO: drop SYS_PTRACE after updating libvirt to a release containing:
+	// https://github.com/libvirt/libvirt/commit/a9c500d2b50c5c041a1bb6ae9724402cf1cec8fe
+	capabilities = append(capabilities, CAP_SYS_PTRACE)
+
 	return capabilities
 }
 
@@ -1933,6 +1939,12 @@ func GetMemoryOverhead(vmi *v1.VirtualMachineInstance, cpuArch string) *resource
 		overhead.Add(resource.MustParse("256Mi"))
 	}
 
+	// Having a TPM device will spawn a swtpm process
+	// In `ps`, swtpm has VSZ of 53808 and RSS of 3496, so 53Mi should do
+	if vmi.Spec.Domain.Devices.TPM != nil {
+		overhead.Add(resource.MustParse("53Mi"))
+	}
+
 	return overhead
 }
 

diff --git a/pkg/virt-launcher/virtwrap/api/deepcopy_generated.go b/pkg/virt-launcher/virtwrap/api/deepcopy_generated.go
diff --git a/pkg/virt-launcher/virtwrap/api/schema.go b/pkg/virt-launcher/virtwrap/api/schema.go
@@ -459,6 +459,17 @@ type Devices struct {
 	Filesystems []FilesystemDevice `xml:"filesystem,omitempty"`
 	Redirs      []RedirectedDevice `xml:"redirdev,omitempty"`
 	SoundCards  []SoundCard        `xml:"sound,omitempty"`
+	TPMs        []TPM              `xml:"tpm,omitempty"`
+}
+
+type TPM struct {
+	Model   string     `xml:"model,attr"`
+	Backend TPMBackend `xml:"backend"`
+}
+
+type TPMBackend struct {
+	Type    string `xml:"type,attr"`
+	Version string `xml:"version,attr"`
 }
 
 // RedirectedDevice describes a device to be redirected

diff --git a/pkg/virt-launcher/virtwrap/converter/converter.go b/pkg/virt-launcher/virtwrap/converter/converter.go
@@ -1843,6 +1843,18 @@ func Convert_v1_VirtualMachineInstance_To_api_Domain(vmi *v1.VirtualMachineInsta
 			api.Arg{Value: "isa-debugcon,iobase=0x402,chardev=firmwarelog"})
 	}
 
+	if vmi.Spec.Domain.Devices.TPM != nil {
+		domain.Spec.Devices.TPMs = []api.TPM{
+			{
+				Model: "tpm-tis",
+				Backend: api.TPMBackend{
+					Type:    "emulator",
+					Version: "2.0",
+				},
+			},
+		}
+	}
+
 	return nil
 }
 

diff --git a/pkg/virt-launcher/virtwrap/util/libvirt_helper.go b/pkg/virt-launcher/virtwrap/util/libvirt_helper.go
@@ -238,7 +238,7 @@ func (l LibvirtWrapper) StartLibvirt(stopChan chan struct{}) {
 			cmd := exec.Command("/usr/sbin/libvirtd", args...)
 			if l.user != 0 {
 				cmd.SysProcAttr = &syscall.SysProcAttr{
-					AmbientCaps: []uintptr{unix.CAP_NET_BIND_SERVICE},
+					AmbientCaps: []uintptr{unix.CAP_NET_BIND_SERVICE, unix.CAP_SYS_PTRACE},
 				}
 			}
 

diff --git a/pkg/virt-operator/resource/generate/components/scc.go b/pkg/virt-operator/resource/generate/components/scc.go
@@ -75,10 +75,12 @@ func NewKubeVirtControllerSCC(namespace string) *secv1.SecurityContextConstraint
 		Type: secv1.SELinuxStrategyRunAsAny,
 	}
 	scc.AllowedCapabilities = []corev1.Capability{
-		// add a CAP_SYS_NICE capability to allow setting cpu affinity
+		// add CAP_SYS_NICE capability to allow setting cpu affinity
 		"SYS_NICE",
 		// add CAP_NET_BIND_SERVICE capability to allow dhcp and slirp operations
 		"NET_BIND_SERVICE",
+		// add CAP_SYS_PTRACE capability needed for libvirt <8.1.0 to find the pid of swtpm
+		"SYS_PTRACE",
 	}
 	scc.AllowHostDirVolumePlugin = true
 	scc.Users = []string{fmt.Sprintf("system:serviceaccount:%s:kubevirt-controller", namespace)}

diff --git a/pkg/virt-operator/resource/generate/components/validations_generated.go b/pkg/virt-operator/resource/generate/components/validations_generated.go
@@ -5286,6 +5286,9 @@ var CRDsValidation map[string]string = map[string]string{
                           required:
                           - name
                           type: object
+                        tpm:
+                          description: Whether to emulate a TPM device.
+                          type: object
                         useVirtioTransitional:
                           description: Fall back to legacy virtio 0.9 support if virtio
                             bus is selected on devices. This is helpful for old machines
@@ -8523,6 +8526,9 @@ var CRDsValidation map[string]string = map[string]string{
                   required:
                   - name
                   type: object
+                tpm:
+                  description: Whether to emulate a TPM device.
+                  type: object
                 useVirtioTransitional:
                   description: Fall back to legacy virtio 0.9 support if virtio bus
                     is selected on devices. This is helpful for old machines like
@@ -10644,6 +10650,9 @@ var CRDsValidation map[string]string = map[string]string{
                   required:
                   - name
                   type: object
+                tpm:
+                  description: Whether to emulate a TPM device.
+                  type: object
                 useVirtioTransitional:
                   description: Fall back to legacy virtio 0.9 support if virtio bus
                     is selected on devices. This is helpful for old machines like
@@ -12727,6 +12736,9 @@ var CRDsValidation map[string]string = map[string]string{
                           required:
                           - name
                           type: object
+                        tpm:
+                          description: Whether to emulate a TPM device.
+                          type: object
                         useVirtioTransitional:
                           description: Fall back to legacy virtio 0.9 support if virtio
                             bus is selected on devices. This is helpful for old machines
@@ -16356,6 +16368,9 @@ var CRDsValidation map[string]string = map[string]string{
                                   required:
                                   - name
                                   type: object
+                                tpm:
+                                  description: Whether to emulate a TPM device.
+                                  type: object
                                 useVirtioTransitional:
                                   description: Fall back to legacy virtio 0.9 support
                                     if virtio bus is selected on devices. This is
@@ -20355,6 +20370,9 @@ var CRDsValidation map[string]string = map[string]string{
                                       required:
                                       - name
                                       type: object
+                                    tpm:
+                                      description: Whether to emulate a TPM device.
+                                      type: object
                                     useVirtioTransitional:
                                       description: Fall back to legacy virtio 0.9
                                         support if virtio bus is selected on devices.