Skip to content

Commit

Permalink
Integrate with Pod security
Browse files Browse the repository at this point in the history
VMs are unfortunatly still privileged workload(in Kubevirt).
We have to integrate with new Pod Security Standards in order to allow
seamless integration, upgrades.

This means we now make sure that target namespace allows
privileged workloads if PSA feature gate is enabled.
This unfortunatly means users escalate their privileges,
in terms of Pod security, by having ability to create VMs.

Signed-off-by: L. Pivarc <[email protected]>
  • Loading branch information
xpivarc committed Sep 14, 2022
1 parent 4690f39 commit 8512fe3
Show file tree
Hide file tree
Showing 13 changed files with 244 additions and 1 deletion.
9 changes: 9 additions & 0 deletions manifests/generated/operator-csv.yaml.in
Original file line number Diff line number Diff line change
Expand Up @@ -478,6 +478,15 @@ spec:
- get
- list
- watch
- apiGroups:
- ""
resources:
- namespaces
verbs:
- get
- list
- watch
- patch
- apiGroups:
- policy
resources:
Expand Down
9 changes: 9 additions & 0 deletions manifests/generated/rbac-operator.authorization.k8s.yaml.in
Original file line number Diff line number Diff line change
Expand Up @@ -397,6 +397,15 @@ rules:
- get
- list
- watch
- apiGroups:
- ""
resources:
- namespaces
verbs:
- get
- list
- watch
- patch
- apiGroups:
- policy
resources:
Expand Down
5 changes: 5 additions & 0 deletions pkg/virt-config/feature-gates.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ const (
WorkloadEncryptionSEV = "WorkloadEncryptionSEV"
// DockerSELinuxMCSWorkaround sets the SELinux level of all the non-compute virt-launcher containers to "s0".
DockerSELinuxMCSWorkaround = "DockerSELinuxMCSWorkaround"
PSA = "PSA"
)

var deprecatedFeatureGates = [...]string{
Expand Down Expand Up @@ -173,3 +174,7 @@ func (config *ClusterConfig) WorkloadEncryptionSEVEnabled() bool {
func (config *ClusterConfig) DockerSELinuxMCSWorkaroundEnabled() bool {
return config.isFeatureGateEnabled(DockerSELinuxMCSWorkaround)
}

func (config *ClusterConfig) PSAEnabled() bool {
return config.isFeatureGateEnabled(PSA)
}
2 changes: 2 additions & 0 deletions pkg/virt-controller/watch/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ go_library(
"migrationpolicy.go",
"node.go",
"pool.go",
"psa.go",
"replicaset.go",
"util.go",
"vm.go",
Expand Down Expand Up @@ -105,6 +106,7 @@ go_test(
"migration_test.go",
"node_test.go",
"pool_test.go",
"psa_test.go",
"replicaset_test.go",
"vm_test.go",
"vmi_test.go",
Expand Down
6 changes: 5 additions & 1 deletion pkg/virt-controller/watch/application.go
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,8 @@ type VirtControllerApp struct {
vmiInformer cache.SharedIndexInformer
vmiRecorder record.EventRecorder

namespaceStore cache.Store

kubeVirtInformer cache.SharedIndexInformer

clusterConfig *virtconfig.ClusterConfig
Expand Down Expand Up @@ -344,7 +346,7 @@ func Execute() {
app.vmiInformer = app.informerFactory.VMI()
app.kvPodInformer = app.informerFactory.KubeVirtPod()
app.nodeInformer = app.informerFactory.KubeVirtNode()

app.namespaceStore = app.informerFactory.Namespace().GetStore()
app.vmiCache = app.vmiInformer.GetStore()
app.vmiRecorder = app.newRecorder(k8sv1.NamespaceAll, "virtualmachine-controller")

Expand Down Expand Up @@ -587,6 +589,7 @@ func (vca *VirtControllerApp) initCommon() {
vca.cdiConfigInformer,
vca.clusterConfig,
topologyHinter,
vca.namespaceStore,
)

recorder := vca.newRecorder(k8sv1.NamespaceAll, "node-controller")
Expand All @@ -603,6 +606,7 @@ func (vca *VirtControllerApp) initCommon() {
vca.vmiRecorder,
vca.clientSet,
vca.clusterConfig,
vca.namespaceStore,
)

vca.nodeTopologyUpdater = topology.NewNodeTopologyUpdater(vca.clientSet, topologyHinter, vca.nodeInformer)
Expand Down
2 changes: 2 additions & 0 deletions pkg/virt-controller/watch/application_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,7 @@ var _ = Describe("Application", func() {
cdiConfigInformer,
config,
topology.NewTopologyHinter(&cache.FakeCustomStore{}, &cache.FakeCustomStore{}, "amd64", nil),
nil,
)
app.rsController = NewVMIReplicaSet(vmiInformer, rsInformer, recorder, virtClient, uint(10))
app.vmController = NewVMController(vmiInformer,
Expand All @@ -151,6 +152,7 @@ var _ = Describe("Application", func() {
recorder,
virtClient,
config,
nil,
)
app.snapshotController = &snapshot.VMSnapshotController{
Client: virtClient,
Expand Down
18 changes: 18 additions & 0 deletions pkg/virt-controller/watch/migration.go
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@ type MigrationController struct {
pvcInformer cache.SharedIndexInformer
pdbInformer cache.SharedIndexInformer
migrationPolicyInformer cache.SharedIndexInformer
namespaceStore cache.Store
recorder record.EventRecorder
podExpectations *controller.UIDTrackingControllerExpectations
migrationStartLock *sync.Mutex
Expand All @@ -124,6 +125,7 @@ func NewMigrationController(templateService services.TemplateService,
recorder record.EventRecorder,
clientset kubecli.KubevirtClient,
clusterConfig *virtconfig.ClusterConfig,
namespaceStore cache.Store,
) *MigrationController {

c := &MigrationController{
Expand All @@ -146,6 +148,8 @@ func NewMigrationController(templateService services.TemplateService,

unschedulablePendingTimeoutSeconds: defaultUnschedulablePendingTimeoutSeconds,
catchAllPendingTimeoutSeconds: defaultCatchAllPendingTimeoutSeconds,

namespaceStore: namespaceStore,
}

c.vmiInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
Expand Down Expand Up @@ -639,6 +643,13 @@ func (c *MigrationController) createTargetPod(migration *virtv1.VirtualMachineIn
}
}

if c.clusterConfig.PSAEnabled() {
// Check my impact
if err := escalateNamespace(c.namespaceStore, c.clientset, vmi.GetNamespace()); err != nil {
return err
}
}

key := controller.MigrationKey(migration)
c.podExpectations.ExpectCreations(key, 1)
pod, err := c.clientset.CoreV1().Pods(vmi.GetNamespace()).Create(context.Background(), templatePod, v1.CreateOptions{})
Expand Down Expand Up @@ -925,8 +936,15 @@ func (c *MigrationController) createAttachmentPod(migration *virtv1.VirtualMachi
attachmentPodTemplate.ObjectMeta.Labels[virtv1.MigrationJobLabel] = string(migration.UID)
attachmentPodTemplate.ObjectMeta.Annotations[virtv1.MigrationJobNameAnnotation] = string(migration.Name)

if c.clusterConfig.PSAEnabled() {
// Check my impact
if err := escalateNamespace(c.namespaceStore, c.clientset, vmi.GetNamespace()); err != nil {
return err
}
}
key := controller.MigrationKey(migration)
c.podExpectations.ExpectCreations(key, 1)

attachmentPod, err := c.clientset.CoreV1().Pods(vmi.GetNamespace()).Create(context.Background(), attachmentPodTemplate, v1.CreateOptions{})
if err != nil {
c.podExpectations.CreationObserved(key)
Expand Down
3 changes: 3 additions & 0 deletions pkg/virt-controller/watch/migration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ var _ = Describe("Migration watcher", func() {
var nodeInformer cache.SharedIndexInformer
var pdbInformer cache.SharedIndexInformer
var migrationPolicyInformer cache.SharedIndexInformer
var namespaceStore cache.Store
var stop chan struct{}
var controller *MigrationController
var recorder *record.FakeRecorder
Expand Down Expand Up @@ -285,6 +286,7 @@ var _ = Describe("Migration watcher", func() {
recorder,
virtClient,
config,
namespaceStore,
)
// Wrap our workqueue to have a way to detect when we are done processing updates
mockQueue = testutils.NewMockWorkQueue(controller.Queue)
Expand Down Expand Up @@ -322,6 +324,7 @@ var _ = Describe("Migration watcher", func() {
podInformer, podSource = testutils.NewFakeInformerFor(&k8sv1.Pod{})
pdbInformer, _ = testutils.NewFakeInformerFor(&policyv1.PodDisruptionBudget{})
migrationPolicyInformer, _ = testutils.NewFakeInformerFor(&migrationsv1.MigrationPolicy{})
namespaceStore = cache.NewStore(cache.DeletionHandlingMetaNamespaceKeyFunc)
recorder = record.NewFakeRecorder(100)
recorder.IncludeObject = true
nodeInformer, _ = testutils.NewFakeInformerFor(&k8sv1.Node{})
Expand Down
52 changes: 52 additions & 0 deletions pkg/virt-controller/watch/psa.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
/*
* This file is part of the KubeVirt project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and
* limitations under the License.
*
* Copyright 2022 Red Hat, Inc.
*
*/

package watch

import (
"context"
"fmt"

k8sv1 "k8s.io/api/core/v1"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/tools/cache"
"kubevirt.io/client-go/kubecli"
)

const PSALabel = "pod-security.kubernetes.io/enforce"

func escalateNamespace(namespaceStore cache.Store, client kubecli.KubevirtClient, namespace string) error {
obj, exists, err := namespaceStore.GetByKey(namespace)
if err != nil {
return fmt.Errorf("Failed to get namespace, %w", err)
}
if !exists {
return fmt.Errorf("Namespace %s not observed, %w", namespace, err)
}
namespaceObj := obj.(*k8sv1.Namespace)
enforceLevel, labelExist := namespaceObj.Labels[PSALabel]
if !labelExist || enforceLevel != "privileged" {
data := []byte(fmt.Sprintf(`{"metadata": { "labels": {"%s": "privileged"}}}`, PSALabel))
_, err := client.CoreV1().Namespaces().Patch(context.TODO(), namespace, types.StrategicMergePatchType, data, v1.PatchOptions{})
if err != nil {
return &syncErrorImpl{err, fmt.Sprintf("Failed to apply enforce label on namespace %s", namespace)}
}
}
return nil
}
98 changes: 98 additions & 0 deletions pkg/virt-controller/watch/psa_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
package watch

import (
"encoding/json"

"github.com/golang/mock/gomock"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
k8sv1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
k8sruntime "k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes/fake"
"k8s.io/client-go/testing"
"k8s.io/client-go/tools/cache"
"kubevirt.io/client-go/kubecli"
)

var _ = Describe("PSA", func() {
var (
namespaceStore cache.Store
client *kubecli.MockKubevirtClient
kubeClient *fake.Clientset
ctrl *gomock.Controller
)

BeforeEach(func() {
namespaceStore = cache.NewStore(cache.DeletionHandlingMetaNamespaceKeyFunc)
ctrl = gomock.NewController(GinkgoT())
client = kubecli.NewMockKubevirtClient(ctrl)
kubeClient = fake.NewSimpleClientset()
client.EXPECT().CoreV1().Return(kubeClient.CoreV1()).AnyTimes()
})

Context("should patch namespace with enforce level", func() {
BeforeEach(func() {
kubeClient.Fake.PrependReactor("patch", "namespaces",
func(action testing.Action) (handled bool, obj k8sruntime.Object, err error) {
patchAction, ok := action.(testing.PatchAction)
Expect(ok).To(BeTrue())
patchBytes := patchAction.GetPatch()
namespace := &k8sv1.Namespace{}
Expect(json.Unmarshal(patchBytes, namespace)).To(Succeed())

Expect(namespace.Labels).To(HaveKeyWithValue(PSALabel, "privileged"))
return true, nil, nil
})
})

It("when label is missing", func() {
namespace := &k8sv1.Namespace{
TypeMeta: metav1.TypeMeta{
Kind: "Namespace",
},
ObjectMeta: metav1.ObjectMeta{
Name: "test",
},
}
Expect(namespaceStore.Add(namespace)).NotTo(HaveOccurred())

Expect(escalateNamespace(namespaceStore, client, "test")).To(Succeed())
})

It("when enforce label is not privileged", func() {
namespace := &k8sv1.Namespace{
TypeMeta: metav1.TypeMeta{
Kind: "Namespace",
},
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Labels: map[string]string{
PSALabel: "restricted",
},
},
}
Expect(namespaceStore.Add(namespace)).NotTo(HaveOccurred())

Expect(escalateNamespace(namespaceStore, client, "test")).To(Succeed())
})
})
It("should not patch namespace when enforce label is set to privileged", func() {
namespace := &k8sv1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Labels: map[string]string{
PSALabel: "privileged",
},
},
}
Expect(namespaceStore.Add(namespace)).NotTo(HaveOccurred())
kubeClient.Fake.PrependReactor("patch", "namespaces",
func(action testing.Action) (handled bool, obj k8sruntime.Object, err error) {
Expect("Patch namespaces is not expected").To(BeEmpty())
return true, nil, nil
})
Expect(escalateNamespace(namespaceStore, client, "test")).To(Succeed())
})

})
Loading

0 comments on commit 8512fe3

Please sign in to comment.