Create mock CSI driver resources in different namespace

test/e2e/framework/util.go

@@ -244,7 +244,7 @@ OUTER:
 
 // WaitForNamespacesDeleted waits for the namespaces to be deleted.
 func WaitForNamespacesDeleted(c clientset.Interface, namespaces []string, timeout time.Duration) error {
-	ginkgo.By("Waiting for namespaces to vanish")
+	ginkgo.By(fmt.Sprintf("Waiting for namespaces %+v to vanish", namespaces))
 	nsMap := map[string]bool{}
 	for _, ns := range namespaces {
 		nsMap[ns] = true

test/e2e/storage/csi_mock_volume.go

@@ -48,8 +48,6 @@ import (
 	"github.com/onsi/gomega"
 )
 
-type cleanupFuncs func()
-
 const (
 	csiNodeLimitUpdateTimeout  = 5 * time.Minute
 	csiPodUnschedulableTimeout = 5 * time.Minute
@@ -102,7 +100,7 @@ var _ = utils.SIGDescribe("CSI mock volume", func() {
 	type mockDriverSetup struct {
 		cs           clientset.Interface
 		config       *testsuites.PerTestConfig
-		testCleanups []cleanupFuncs
+		testCleanups []func()
 		pods         []*v1.Pod
 		pvcs         []*v1.PersistentVolumeClaim
 		sc           map[string]*storagev1.StorageClass
@@ -377,7 +375,7 @@ var _ = utils.SIGDescribe("CSI mock volume", func() {
 				framework.ExpectNoError(err, "while deleting")
 
 				ginkgo.By("Checking CSI driver logs")
-				err = checkPodLogs(m.cs, f.Namespace.Name, driverPodName, driverContainerName, pod, test.expectPodInfo, test.expectEphemeral, csiInlineVolumesEnabled)
+				err = checkPodLogs(m.cs, m.config.DriverNamespace.Name, driverPodName, driverContainerName, pod, test.expectPodInfo, test.expectEphemeral, csiInlineVolumesEnabled)
 				framework.ExpectNoError(err)
 			})
 		}
@@ -700,7 +698,7 @@ var _ = utils.SIGDescribe("CSI mock volume", func() {
 						framework.Failf("timed out waiting for the CSI call that indicates that the pod can be deleted: %v", test.expectedCalls)
 					}
 					time.Sleep(1 * time.Second)
-					_, index, err := compareCSICalls(trackedCalls, test.expectedCalls, m.cs, f.Namespace.Name, driverPodName, driverContainerName)
+					_, index, err := compareCSICalls(trackedCalls, test.expectedCalls, m.cs, m.config.DriverNamespace.Name, driverPodName, driverContainerName)
 					framework.ExpectNoError(err, "while waiting for initial CSI calls")
 					if index == 0 {
 						// No CSI call received yet
@@ -724,7 +722,7 @@ var _ = utils.SIGDescribe("CSI mock volume", func() {
 
 				ginkgo.By("Waiting for all remaining expected CSI calls")
 				err = wait.Poll(time.Second, csiUnstageWaitTimeout, func() (done bool, err error) {
-					_, index, err := compareCSICalls(trackedCalls, test.expectedCalls, m.cs, f.Namespace.Name, driverPodName, driverContainerName)
+					_, index, err := compareCSICalls(trackedCalls, test.expectedCalls, m.cs, m.config.DriverNamespace.Name, driverPodName, driverContainerName)
 					if err != nil {
 						return true, err
 					}
@@ -852,7 +850,7 @@ var _ = utils.SIGDescribe("CSI mock volume", func() {
 
 				var calls []mockCSICall
 				err = wait.PollImmediateUntil(time.Second, func() (done bool, err error) {
-					c, index, err := compareCSICalls(deterministicCalls, expected, m.cs, f.Namespace.Name, driverPodName, driverContainerName)
+					c, index, err := compareCSICalls(deterministicCalls, expected, m.cs, m.config.DriverNamespace.Name, driverPodName, driverContainerName)
 					if err != nil {
 						return true, fmt.Errorf("error waiting for expected CSI calls: %s", err)
 					}
@@ -1090,7 +1088,7 @@ type mockCSICall struct {
 func checkPodLogs(cs clientset.Interface, namespace, driverPodName, driverContainerName string, pod *v1.Pod, expectPodInfo, ephemeralVolume, csiInlineVolumesEnabled bool) error {
 	expectedAttributes := map[string]string{
 		"csi.storage.k8s.io/pod.name":            pod.Name,
-		"csi.storage.k8s.io/pod.namespace":       namespace,
+		"csi.storage.k8s.io/pod.namespace":       pod.Namespace,
 		"csi.storage.k8s.io/pod.uid":             string(pod.UID),
 		"csi.storage.k8s.io/serviceAccount.name": "default",
 	}

test/e2e/storage/drivers/csi.go

@@ -73,6 +73,7 @@ const (
 type hostpathCSIDriver struct {
 	driverInfo       testsuites.DriverInfo
 	manifests        []string
+	cleanupHandle    framework.CleanupActionHandle
 	volumeAttributes []map[string]string
 }
 
@@ -169,8 +170,13 @@ func (h *hostpathCSIDriver) GetSnapshotClass(config *testsuites.PerTestConfig) *
 }
 
 func (h *hostpathCSIDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTestConfig, func()) {
+	// Create secondary namespace which will be used for creating driver
+	driverNamespace := utils.CreateDriverNamespace(f)
+	ns2 := driverNamespace.Name
+	ns1 := f.Namespace.Name
+
 	ginkgo.By(fmt.Sprintf("deploying %s driver", h.driverInfo.Name))
-	cancelLogging := testsuites.StartPodLogs(f)
+	cancelLogging := testsuites.StartPodLogs(f, driverNamespace)
 	cs := f.ClientSet
 
 	// The hostpath CSI driver only works when everything runs on the same node.
@@ -181,6 +187,7 @@ func (h *hostpathCSIDriver) PrepareTest(f *framework.Framework) (*testsuites.Per
 		Prefix:              "hostpath",
 		Framework:           f,
 		ClientNodeSelection: e2epod.NodeSelection{Name: node.Name},
+		DriverNamespace:     driverNamespace,
 	}
 
 	o := utils.PatchCSIOptions{
@@ -192,19 +199,33 @@ func (h *hostpathCSIDriver) PrepareTest(f *framework.Framework) (*testsuites.Per
 		SnapshotterContainerName: "csi-snapshotter",
 		NodeName:                 node.Name,
 	}
-	cleanup, err := utils.CreateFromManifests(config.Framework, func(item interface{}) error {
+	cleanup, err := utils.CreateFromManifests(config.Framework, driverNamespace, func(item interface{}) error {
 		return utils.PatchCSIDeployment(config.Framework, o, item)
-	},
-		h.manifests...)
+	}, h.manifests...)
+
 	if err != nil {
 		framework.Failf("deploying %s driver: %v", h.driverInfo.Name, err)
 	}
 
-	return config, func() {
-		ginkgo.By(fmt.Sprintf("uninstalling %s driver", h.driverInfo.Name))
-		cleanup()
-		cancelLogging()
+	// Cleanup CSI driver and namespaces. This function needs to be idempotent and can be
+	// concurrently called from defer (or AfterEach) and AfterSuite action hooks.
+	cleanupFunc := func() {
+		framework.RemoveCleanupAction(h.cleanupHandle)
+		ginkgo.By(fmt.Sprintf("deleting the test namespace: %s", ns1))
+		// Delete the primary namespace but its okay to fail here because this namespace will
+		// also be deleted by framework.Aftereach hook
+		tryFunc(deleteNamespaceFunc(f.ClientSet, ns1, framework.DefaultNamespaceDeletionTimeout))
+
+		ginkgo.By("uninstalling csi mock driver")
+		tryFunc(cleanup)
+		tryFunc(cancelLogging)
+
+		ginkgo.By(fmt.Sprintf("deleting the driver namespace: %s", ns2))
+		tryFunc(deleteNamespaceFunc(f.ClientSet, ns2, framework.DefaultNamespaceDeletionTimeout))
 	}
+	h.cleanupHandle = framework.AddCleanupAction(cleanupFunc)
+
+	return config, cleanupFunc
 }
 
 // mockCSI
@@ -216,6 +237,7 @@ type mockCSIDriver struct {
 	attachLimit         int
 	enableTopology      bool
 	enableNodeExpansion bool
+	cleanupHandle       framework.CleanupActionHandle
 	javascriptHooks     map[string]string
 }
 
@@ -299,8 +321,13 @@ func (m *mockCSIDriver) GetDynamicProvisionStorageClass(config *testsuites.PerTe
 }
 
 func (m *mockCSIDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTestConfig, func()) {
+	// Create secondary namespace which will be used for creating driver
+	driverNamespace := utils.CreateDriverNamespace(f)
+	ns2 := driverNamespace.Name
+	ns1 := f.Namespace.Name
+
 	ginkgo.By("deploying csi mock driver")
-	cancelLogging := testsuites.StartPodLogs(f)
+	cancelLogging := testsuites.StartPodLogs(f, driverNamespace)
 	cs := f.ClientSet
 
 	// pods should be scheduled on the node
@@ -311,6 +338,7 @@ func (m *mockCSIDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTest
 		Prefix:              "mock",
 		Framework:           f,
 		ClientNodeSelection: e2epod.NodeSelection{Name: node.Name},
+		DriverNamespace:     driverNamespace,
 	}
 
 	containerArgs := []string{"--name=csi-mock-" + f.UniqueName}
@@ -343,7 +371,8 @@ func (m *mockCSIDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTest
 			"hooks.yaml": string(hooksYaml),
 		},
 	}
-	_, err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(context.TODO(), hooks, metav1.CreateOptions{})
+
+	_, err = f.ClientSet.CoreV1().ConfigMaps(ns2).Create(context.TODO(), hooks, metav1.CreateOptions{})
 	framework.ExpectNoError(err)
 
 	if len(m.javascriptHooks) > 0 {
@@ -364,28 +393,46 @@ func (m *mockCSIDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTest
 			storagev1.VolumeLifecycleEphemeral,
 		},
 	}
-	cleanup, err := utils.CreateFromManifests(f, func(item interface{}) error {
+	cleanup, err := utils.CreateFromManifests(f, driverNamespace, func(item interface{}) error {
 		return utils.PatchCSIDeployment(f, o, item)
-	},
-		m.manifests...)
+	}, m.manifests...)
+
 	if err != nil {
 		framework.Failf("deploying csi mock driver: %v", err)
 	}
 
-	return config, func() {
+	// Cleanup CSI driver and namespaces. This function needs to be idempotent and can be
+	// concurrently called from defer (or AfterEach) and AfterSuite action hooks.
+	cleanupFunc := func() {
+		framework.RemoveCleanupAction(m.cleanupHandle)
+		ginkgo.By(fmt.Sprintf("deleting the test namespace: %s", ns1))
+		// Delete the primary namespace but its okay to fail here because this namespace will
+		// also be deleted by framework.Aftereach hook
+		tryFunc(deleteNamespaceFunc(f.ClientSet, ns1, framework.DefaultNamespaceDeletionTimeout))
+
 		ginkgo.By("uninstalling csi mock driver")
-		err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Delete(context.TODO(), hooksConfigMapName, metav1.DeleteOptions{})
-		if err != nil {
-			framework.Logf("deleting failed: %s", err)
-		}
-		cleanup()
-		cancelLogging()
+		tryFunc(func() {
+			err := f.ClientSet.CoreV1().ConfigMaps(ns2).Delete(context.TODO(), hooksConfigMapName, metav1.DeleteOptions{})
+			if err != nil {
+				framework.Logf("deleting failed: %s", err)
+			}
+		})
+
+		tryFunc(cleanup)
+		tryFunc(cancelLogging)
+		ginkgo.By(fmt.Sprintf("deleting the driver namespace: %s", ns2))
+		tryFunc(deleteNamespaceFunc(f.ClientSet, ns2, framework.DefaultNamespaceDeletionTimeout))
 	}
+
+	m.cleanupHandle = framework.AddCleanupAction(cleanupFunc)
+
+	return config, cleanupFunc
 }
 
 // gce-pd
 type gcePDCSIDriver struct {
-	driverInfo testsuites.DriverInfo
+	driverInfo    testsuites.DriverInfo
+	cleanupHandle framework.CleanupActionHandle
 }
 
 var _ testsuites.TestDriver = &gcePDCSIDriver{}
@@ -473,7 +520,12 @@ func (g *gcePDCSIDriver) GetSnapshotClass(config *testsuites.PerTestConfig) *uns
 
 func (g *gcePDCSIDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTestConfig, func()) {
 	ginkgo.By("deploying csi gce-pd driver")
-	cancelLogging := testsuites.StartPodLogs(f)
+	// Create secondary namespace which will be used for creating driver
+	driverNamespace := utils.CreateDriverNamespace(f)
+	ns2 := driverNamespace.Name
+	ns1 := f.Namespace.Name
+
+	cancelLogging := testsuites.StartPodLogs(f, driverNamespace)
 	// It would be safer to rename the gcePD driver, but that
 	// hasn't been done before either and attempts to do so now led to
 	// errors during driver registration, therefore it is disabled
@@ -486,7 +538,7 @@ func (g *gcePDCSIDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTes
 	// 	DriverContainerName:      "gce-driver",
 	// 	ProvisionerContainerName: "csi-external-provisioner",
 	// }
-	createGCESecrets(f.ClientSet, f.Namespace.Name)
+	createGCESecrets(f.ClientSet, ns2)
 
 	manifests := []string{
 		"test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml",
@@ -496,7 +548,7 @@ func (g *gcePDCSIDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTes
 		"test/e2e/testing-manifests/storage-csi/gce-pd/controller_ss.yaml",
 	}
 
-	cleanup, err := utils.CreateFromManifests(f, nil, manifests...)
+	cleanup, err := utils.CreateFromManifests(f, driverNamespace, nil, manifests...)
 	if err != nil {
 		framework.Failf("deploying csi gce-pd driver: %v", err)
 	}
@@ -505,15 +557,30 @@ func (g *gcePDCSIDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTes
 		framework.Failf("waiting for csi driver node registration on: %v", err)
 	}
 
+	// Cleanup CSI driver and namespaces. This function needs to be idempotent and can be
+	// concurrently called from defer (or AfterEach) and AfterSuite action hooks.
+	cleanupFunc := func() {
+		framework.RemoveCleanupAction(g.cleanupHandle)
+		ginkgo.By(fmt.Sprintf("deleting the test namespace: %s", ns1))
+		// Delete the primary namespace but its okay to fail here because this namespace will
+		// also be deleted by framework.Aftereach hook
+		tryFunc(deleteNamespaceFunc(f.ClientSet, ns1, framework.DefaultNamespaceDeletionTimeout))
+
+		ginkgo.By("uninstalling csi mock driver")
+		tryFunc(cleanup)
+		tryFunc(cancelLogging)
+
+		ginkgo.By(fmt.Sprintf("deleting the driver namespace: %s", ns2))
+		tryFunc(deleteNamespaceFunc(f.ClientSet, ns2, framework.DefaultNamespaceDeletionTimeout))
+	}
+	g.cleanupHandle = framework.AddCleanupAction(cleanupFunc)
+
 	return &testsuites.PerTestConfig{
-			Driver:    g,
-			Prefix:    "gcepd",
-			Framework: f,
-		}, func() {
-			ginkgo.By("uninstalling gce-pd driver")
-			cleanup()
-			cancelLogging()
-		}
+		Driver:          g,
+		Prefix:          "gcepd",
+		Framework:       f,
+		DriverNamespace: driverNamespace,
+	}, cleanupFunc
 }
 
 func waitForCSIDriverRegistrationOnAllNodes(driverName string, cs clientset.Interface) error {
@@ -549,3 +616,30 @@ func waitForCSIDriverRegistrationOnNode(nodeName string, driverName string, cs c
 	}
 	return nil
 }
+
+func deleteNamespaceFunc(cs clientset.Interface, ns string, timeout time.Duration) func() {
+	return func() {
+		err := cs.CoreV1().Namespaces().Delete(context.TODO(), ns, metav1.DeleteOptions{})
+		if err != nil && !apierrors.IsNotFound(err) {
+			framework.Logf("error deleting namespace %s: %v", ns, err)
+		}
+		err = framework.WaitForNamespacesDeleted(cs, []string{ns}, timeout)
+		if err != nil {
+			framework.Logf("error deleting namespace %s: %v", ns, err)
+		}
+	}
+}
+
+func tryFunc(f func()) error {
+	var err error
+	if f == nil {
+		return nil
+	}
+	defer func() {
+		if recoverError := recover(); recoverError != nil {
+			err = fmt.Errorf("%v", recoverError)
+		}
+	}()
+	f()
+	return err
+}

test/e2e/storage/external/external.go

@@ -282,7 +282,7 @@ func (d *driverDefinition) GetDynamicProvisionStorageClass(e2econfig *testsuites
 		framework.ExpectNoError(err, "load storage class from %s", d.StorageClass.FromFile)
 		framework.ExpectEqual(len(items), 1, "exactly one item from %s", d.StorageClass.FromFile)
 
-		err = utils.PatchItems(f, items...)
+		err = utils.PatchItems(f, f.Namespace, items...)
 		framework.ExpectNoError(err, "patch items")
 
 		sc, ok = items[0].(*storagev1.StorageClass)

test/e2e/storage/testsuites/base.go

@@ -554,10 +554,11 @@ func getSnapshot(claimName string, ns, snapshotClassName string) *unstructured.U
 //
 // The output goes to log files (when using --report-dir, as in the
 // CI) or the output stream (otherwise).
-func StartPodLogs(f *framework.Framework) func() {
+func StartPodLogs(f *framework.Framework, driverNamespace *v1.Namespace) func() {
 	ctx, cancel := context.WithCancel(context.Background())
 	cs := f.ClientSet
-	ns := f.Namespace
+
+	ns := driverNamespace.Name
 
 	to := podlogs.LogOutput{
 		StatusWriter: ginkgo.GinkgoWriter,
@@ -575,13 +576,13 @@ func StartPodLogs(f *framework.Framework) func() {
 		to.LogPathPrefix = framework.TestContext.ReportDir + "/" +
 			reg.ReplaceAllString(test.FullTestText, "_") + "/"
 	}
-	podlogs.CopyAllLogs(ctx, cs, ns.Name, to)
+	podlogs.CopyAllLogs(ctx, cs, ns, to)
 
 	// pod events are something that the framework already collects itself
 	// after a failed test. Logging them live is only useful for interactive
 	// debugging, not when we collect reports.
 	if framework.TestContext.ReportDir == "" {
-		podlogs.WatchPods(ctx, cs, ns.Name, ginkgo.GinkgoWriter)
+		podlogs.WatchPods(ctx, cs, ns, ginkgo.GinkgoWriter)
 	}
 
 	return cancel

test/e2e/storage/testsuites/testdriver.go

@@ -226,6 +226,9 @@ type PerTestConfig struct {
 	// the configuration that then has to be used to run tests.
 	// The values above are ignored for such tests.
 	ServerConfig *e2evolume.TestConfig
+
+	// Some drivers run in their own namespace
+	DriverNamespace *v1.Namespace
 }
 
 // GetUniqueDriverName returns unique driver name that can be used parallelly in tests