Ensure all CRs for all CRDs are removed before removing the CRD entirely

Signed-off-by: David Vossel <[email protected]>

pkg/virt-operator/resource/apply/delete.go

@@ -21,6 +21,7 @@ package apply
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"strings"
 
@@ -33,6 +34,7 @@ import (
 	rbacv1 "k8s.io/api/rbac/v1"
 	extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 	apiregv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
 
 	v1 "kubevirt.io/api/core/v1"
@@ -104,32 +106,18 @@ func DeleteAll(kv *v1.KubeVirt,
 	}
 
 	// first delete CRDs only
-	ext := clientset.ExtensionsClient()
-	objects := stores.CrdCache.List()
-	for _, obj := range objects {
-		if crd, ok := obj.(*extv1.CustomResourceDefinition); ok && crd.DeletionTimestamp == nil {
-			if key, err := controller.KeyFunc(crd); err == nil {
-				expectations.Crd.AddExpectedDeletion(kvkey, key)
-				err := ext.ApiextensionsV1().CustomResourceDefinitions().Delete(context.Background(), crd.Name, deleteOptions)
-				if err != nil {
-					expectations.Crd.DeletionObserved(kvkey, key)
-					log.Log.Errorf("Failed to delete crd %+v: %v", crd, err)
-					return err
-				}
-			}
-		} else if !ok {
-			log.Log.Errorf(castFailedFmt, obj)
-			return nil
-		}
-
+	err = crdHandleDeletion(kvkey, stores, clientset, expectations)
+	if err != nil {
+		return err
 	}
+
 	if !util.IsStoreEmpty(stores.CrdCache) {
 		// wait until CRDs are gone
 		return nil
 	}
 
 	// delete daemonsets
-	objects = stores.DaemonSetCache.List()
+	objects := stores.DaemonSetCache.List()
 	for _, obj := range objects {
 		if ds, ok := obj.(*appsv1.DaemonSet); ok && ds.DeletionTimestamp == nil {
 			if key, err := controller.KeyFunc(ds); err == nil {
@@ -461,3 +449,126 @@ func DeleteAll(kv *v1.KubeVirt,
 	}
 	return nil
 }
+
+func crdInstanceDeletionCompleted(crd *extv1.CustomResourceDefinition) bool {
+	// Below is an example of what is being looked for here.
+	// The CRD will have this condition once a CRD which is being
+	// deleted has all instances removed related to this CRD.
+	//
+	//    message: removed all instances
+	//    reason: InstanceDeletionCompleted
+	//    status: "False"
+	//    type: Terminating
+
+	if crd.DeletionTimestamp == nil {
+		return false
+	}
+
+	for _, condition := range crd.Status.Conditions {
+		if condition.Type == extv1.Terminating &&
+			condition.Status == extv1.ConditionFalse &&
+			condition.Reason == "InstanceDeletionCompleted" {
+			return true
+		}
+	}
+	return false
+}
+
+func crdFilterNeedFinalizerAdded(crds []*extv1.CustomResourceDefinition) []*extv1.CustomResourceDefinition {
+	filtered := []*extv1.CustomResourceDefinition{}
+
+	for _, crd := range crds {
+		if crd.DeletionTimestamp == nil && !controller.HasFinalizer(crd, v1.VirtOperatorComponentFinalizer) {
+			filtered = append(filtered, crd)
+		}
+	}
+
+	return filtered
+}
+
+func crdFilterNeedDeletion(crds []*extv1.CustomResourceDefinition) []*extv1.CustomResourceDefinition {
+	filtered := []*extv1.CustomResourceDefinition{}
+
+	for _, crd := range crds {
+		if crd.DeletionTimestamp == nil {
+			filtered = append(filtered, crd)
+		}
+	}
+	return filtered
+}
+
+func crdFilterNeedFinalizerRemoved(crds []*extv1.CustomResourceDefinition) []*extv1.CustomResourceDefinition {
+	filtered := []*extv1.CustomResourceDefinition{}
+	for _, crd := range crds {
+		if !crdInstanceDeletionCompleted(crd) {
+			// All crds must have all crs removed before any CRD finalizer can be removed
+			return []*extv1.CustomResourceDefinition{}
+		} else if controller.HasFinalizer(crd, v1.VirtOperatorComponentFinalizer) {
+			filtered = append(filtered, crd)
+		}
+	}
+	return filtered
+}
+
+func crdHandleDeletion(kvkey string,
+	stores util.Stores,
+	clientset kubecli.KubevirtClient,
+	expectations *util.Expectations) error {
+
+	ext := clientset.ExtensionsClient()
+	objects := stores.CrdCache.List()
+
+	crds := []*extv1.CustomResourceDefinition{}
+	for _, obj := range objects {
+		crd, ok := obj.(*extv1.CustomResourceDefinition)
+		if !ok {
+			log.Log.Errorf(castFailedFmt, obj)
+			return nil
+		}
+		crds = append(crds, crd)
+	}
+
+	needFinalizerAdded := crdFilterNeedFinalizerAdded(crds)
+	needDeletion := crdFilterNeedDeletion(crds)
+	needFinalizerRemoved := crdFilterNeedFinalizerRemoved(crds)
+
+	for _, crd := range needFinalizerAdded {
+		crdCopy := crd.DeepCopy()
+		controller.AddFinalizer(crdCopy, v1.VirtOperatorComponentFinalizer)
+
+		patchBytes, err := json.Marshal(crdCopy.Finalizers)
+		if err != nil {
+			return err
+		}
+		ops := fmt.Sprintf(`[{ "op": "add", "path": "/metadata/finalizers", "value": %s }]`, string(patchBytes))
+		_, err = ext.ApiextensionsV1().CustomResourceDefinitions().Patch(context.Background(), crd.Name, types.JSONPatchType, []byte(ops), metav1.PatchOptions{})
+		if err != nil {
+			return err
+		}
+	}
+
+	for _, crd := range needDeletion {
+		key, err := controller.KeyFunc(crd)
+		if err != nil {
+			return err
+		}
+
+		expectations.Crd.AddExpectedDeletion(kvkey, key)
+		err = ext.ApiextensionsV1().CustomResourceDefinitions().Delete(context.Background(), crd.Name, metav1.DeleteOptions{})
+		if err != nil {
+			expectations.Crd.DeletionObserved(kvkey, key)
+			log.Log.Errorf("Failed to delete crd %+v: %v", crd, err)
+			return err
+		}
+	}
+
+	for _, crd := range needFinalizerRemoved {
+		ops := `[ { "op": "remove", "path": "/metadata/finalizers" } ]`
+		_, err := ext.ApiextensionsV1().CustomResourceDefinitions().Patch(context.Background(), crd.Name, types.JSONPatchType, []byte(ops), metav1.PatchOptions{})
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

staging/src/kubevirt.io/api/core/v1/types.go

@@ -760,6 +760,9 @@ const (
 	// This label indicates the object is a part of the install strategy retrieval process.
 	InstallStrategyLabel = "kubevirt.io/install-strategy"
 
+	// Set by virt-operator to coordinate component deletion
+	VirtOperatorComponentFinalizer string = "kubevirt.io/virtOperatorFinalizer"
+
 	// Set by VMI controller to ensure VMIs are processed during deletion
 	VirtualMachineInstanceFinalizer string = "foregroundDeleteVirtualMachine"
 	// Set By VM controller on VMIs to ensure VMIs are processed by VM controller during deletion