Add multi-vc configuration for e2e tests

hack/verify-flags/excluded-flags.txt

@@ -23,3 +23,4 @@ retry_time
 file_content_in_loop
 break_on_expected_content
 Premium_LRS
+VCP_STRESS_ITERATIONS

pkg/cloudprovider/providers/vsphere/BUILD

@@ -21,7 +21,6 @@ go_library(
         "//pkg/cloudprovider/providers/vsphere/vclib/diskmanagers:go_default_library",
         "//pkg/controller:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
-        "//vendor/github.com/vmware/govmomi:go_default_library",
         "//vendor/github.com/vmware/govmomi/vim25:go_default_library",
         "//vendor/github.com/vmware/govmomi/vim25/mo:go_default_library",
         "//vendor/golang.org/x/net/context:go_default_library",

pkg/cloudprovider/providers/vsphere/nodemanager.go

@@ -354,3 +354,10 @@ func (nm *NodeManager) renewNodeInfo(nodeInfo *NodeInfo, reconnect bool) (*NodeI
 	vm := nodeInfo.vm.RenewVM(vsphereInstance.conn.GoVmomiClient)
 	return &NodeInfo{vm: &vm, dataCenter: vm.Datacenter, vcServer: nodeInfo.vcServer}, nil
 }
+
+func (nodeInfo *NodeInfo) VM() *vclib.VirtualMachine {
+	if nodeInfo == nil {
+		return nil
+	}
+	return nodeInfo.vm
+}

pkg/cloudprovider/providers/vsphere/vsphere.go

@@ -1159,3 +1159,10 @@ func (vs *VSphere) NodeDeleted(obj interface{}) {
 	glog.V(4).Infof("Node deleted: %+v", node)
 	vs.nodeManager.UnRegisterNode(node)
 }
+
+func (vs *VSphere) NodeManager() (nodeManager *NodeManager) {
+	if vs == nil {
+		return nil
+	}
+	return vs.nodeManager
+}

pkg/cloudprovider/providers/vsphere/vsphere_util.go

@@ -21,20 +21,17 @@ import (
 	"errors"
 	"os"
 	"regexp"
-	"runtime"
 	"strings"
 	"time"
 
 	"github.com/golang/glog"
-	"github.com/vmware/govmomi"
 	"github.com/vmware/govmomi/vim25"
 
 	"fmt"
 
 	"path/filepath"
 
 	"github.com/vmware/govmomi/vim25/mo"
-	"k8s.io/api/core/v1"
 	k8stypes "k8s.io/apimachinery/pkg/types"
 	"k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere/vclib"
 	"k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere/vclib/diskmanagers"
@@ -46,63 +43,37 @@ const (
 	Folder                = "Folder"
 	VirtualMachine        = "VirtualMachine"
 	DummyDiskName         = "kube-dummyDisk.vmdk"
+	vSphereConfFileEnvVar = "VSPHERE_CONF_FILE"
 )
 
 // GetVSphere reads vSphere configuration from system environment and construct vSphere object
 func GetVSphere() (*VSphere, error) {
-	cfg := getVSphereConfig()
-	vSphereConn := getVSphereConn(cfg)
-	client, err := GetgovmomiClient(vSphereConn)
+	cfg, err := getVSphereConfig()
+	if err != nil {
+		return nil, err
+	}
+	vs, err := newControllerNode(*cfg)
 	if err != nil {
 		return nil, err
 	}
-	vSphereConn.GoVmomiClient = client
-	vsphereIns := &VSphereInstance{
-		conn: vSphereConn,
-		cfg: &VirtualCenterConfig{
-			User:              cfg.Global.User,
-			Password:          cfg.Global.Password,
-			VCenterPort:       cfg.Global.VCenterPort,
-			Datacenters:       cfg.Global.Datacenters,
-			RoundTripperCount: cfg.Global.RoundTripperCount,
-		},
-	}
-	vsphereInsMap := make(map[string]*VSphereInstance)
-	vsphereInsMap[cfg.Global.VCenterIP] = vsphereIns
-	// TODO: Initialize nodeManager and set it in VSphere.
-	vs := &VSphere{
-		vsphereInstanceMap: vsphereInsMap,
-		hostName:           "",
-		cfg:                cfg,
-		nodeManager: &NodeManager{
-			vsphereInstanceMap: vsphereInsMap,
-			nodeInfoMap:        make(map[string]*NodeInfo),
-			registeredNodes:    make(map[string]*v1.Node),
-		},
-	}
-	runtime.SetFinalizer(vs, logout)
 	return vs, nil
 }
 
-func getVSphereConfig() *VSphereConfig {
-	var cfg VSphereConfig
-	cfg.Global.VCenterIP = os.Getenv("VSPHERE_VCENTER")
-	cfg.Global.VCenterPort = os.Getenv("VSPHERE_VCENTER_PORT")
-	cfg.Global.User = os.Getenv("VSPHERE_USER")
-	cfg.Global.Password = os.Getenv("VSPHERE_PASSWORD")
-	cfg.Global.Datacenters = os.Getenv("VSPHERE_DATACENTER")
-	cfg.Global.DefaultDatastore = os.Getenv("VSPHERE_DATASTORE")
-	cfg.Global.WorkingDir = os.Getenv("VSPHERE_WORKING_DIR")
-	cfg.Global.VMName = os.Getenv("VSPHERE_VM_NAME")
-	cfg.Global.InsecureFlag = false
-	if strings.ToLower(os.Getenv("VSPHERE_INSECURE")) == "true" {
-		cfg.Global.InsecureFlag = true
-	}
-	cfg.Workspace.VCenterIP = cfg.Global.VCenterIP
-	cfg.Workspace.Datacenter = cfg.Global.Datacenters
-	cfg.Workspace.DefaultDatastore = cfg.Global.DefaultDatastore
-	cfg.Workspace.Folder = cfg.Global.WorkingDir
-	return &cfg
+func getVSphereConfig() (*VSphereConfig, error) {
+	confFileLocation := os.Getenv(vSphereConfFileEnvVar)
+	if confFileLocation == "" {
+		return nil, fmt.Errorf("Env variable 'VSPHERE_CONF_FILE' is not set.")
+	}
+	confFile, err := os.Open(confFileLocation)
+	if err != nil {
+		return nil, err
+	}
+	defer confFile.Close()
+	cfg, err := readConfig(confFile)
+	if err != nil {
+		return nil, err
+	}
+	return &cfg, nil
 }
 
 func getVSphereConn(cfg *VSphereConfig) *vclib.VSphereConnection {
@@ -117,16 +88,6 @@ func getVSphereConn(cfg *VSphereConfig) *vclib.VSphereConnection {
 	return vSphereConn
 }
 
-// GetgovmomiClient gets the goVMOMI client for the vsphere connection object
-func GetgovmomiClient(conn *vclib.VSphereConnection) (*govmomi.Client, error) {
-	if conn == nil {
-		cfg := getVSphereConfig()
-		conn = getVSphereConn(cfg)
-	}
-	client, err := conn.NewClient(context.TODO())
-	return client, err
-}
-
 // Returns the accessible datastores for the given node VM.
 func getAccessibleDatastores(ctx context.Context, nodeVmDetail *NodeDetails, nodeManager *NodeManager) ([]*vclib.DatastoreInfo, error) {
 	accessibleDatastores, err := nodeVmDetail.vm.GetAllAccessibleDatastores(ctx)
@@ -512,3 +473,40 @@ func (vs *VSphere) checkDiskAttached(ctx context.Context, nodes []k8stypes.NodeN
 	}
 	return nodesToRetry, nil
 }
+
+func (vs *VSphere) IsDummyVMPresent(vmName string) (bool, error) {
+	isDummyVMPresent := false
+
+	// Create context
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	vsi, err := vs.getVSphereInstanceForServer(vs.cfg.Workspace.VCenterIP, ctx)
+	if err != nil {
+		return isDummyVMPresent, err
+	}
+
+	dc, err := vclib.GetDatacenter(ctx, vsi.conn, vs.cfg.Workspace.Datacenter)
+	if err != nil {
+		return isDummyVMPresent, err
+	}
+
+	vmFolder, err := dc.GetFolderByPath(ctx, vs.cfg.Workspace.Folder)
+	if err != nil {
+		return isDummyVMPresent, err
+	}
+
+	vms, err := vmFolder.GetVirtualMachines(ctx)
+	if err != nil {
+		return isDummyVMPresent, err
+	}
+
+	for _, vm := range vms {
+		if vm.Name() == vmName {
+			isDummyVMPresent = true
+			break
+		}
+	}
+
+	return isDummyVMPresent, nil
+}

test/e2e/storage/vsphere/BUILD

@@ -11,6 +11,7 @@ go_library(
         "persistent_volumes-vsphere.go",
         "pv_reclaimpolicy.go",
         "pvc_label_selector.go",
+        "vsphere_common.go",
         "vsphere_scale.go",
         "vsphere_statefulsets.go",
         "vsphere_stress.go",
@@ -36,7 +37,6 @@ go_library(
         "//test/e2e/storage/utils:go_default_library",
         "//vendor/github.com/onsi/ginkgo:go_default_library",
         "//vendor/github.com/onsi/gomega:go_default_library",
-        "//vendor/github.com/vmware/govmomi/find:go_default_library",
         "//vendor/github.com/vmware/govmomi/vim25/types:go_default_library",
         "//vendor/golang.org/x/net/context:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",

test/e2e/storage/vsphere/vsphere_common.go

@@ -0,0 +1,66 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package vsphere
+
+import (
+	. "github.com/onsi/gomega"
+	"os"
+	"strconv"
+)
+
+const (
+	SPBMPolicyName            = "VSPHERE_SPBM_POLICY_NAME"
+	StorageClassDatastoreName = "VSPHERE_DATASTORE"
+	SecondSharedDatastore     = "VSPHERE_SECOND_SHARED_DATASTORE"
+	KubernetesClusterName     = "VSPHERE_KUBERNETES_CLUSTER"
+	SPBMTagPolicy             = "VSPHERE_SPBM_TAG_POLICY"
+)
+
+const (
+	VCPClusterDatastore        = "CLUSTER_DATASTORE"
+	SPBMPolicyDataStoreCluster = "VSPHERE_SPBM_POLICY_DS_CLUSTER"
+)
+
+const (
+	VCPScaleVolumeCount   = "VCP_SCALE_VOLUME_COUNT"
+	VCPScaleVolumesPerPod = "VCP_SCALE_VOLUME_PER_POD"
+	VCPScaleInstances     = "VCP_SCALE_INSTANCES"
+)
+
+const (
+	VCPStressInstances  = "VCP_STRESS_INSTANCES"
+	VCPStressIterations = "VCP_STRESS_ITERATIONS"
+)
+
+const (
+	VCPPerfVolumeCount   = "VCP_PERF_VOLUME_COUNT"
+	VCPPerfVolumesPerPod = "VCP_PERF_VOLUME_PER_POD"
+	VCPPerfIterations    = "VCP_PERF_ITERATIONS"
+)
+
+func GetAndExpectStringEnvVar(varName string) string {
+	varValue := os.Getenv(varName)
+	Expect(varValue).NotTo(BeEmpty(), "ENV "+varName+" is not set")
+	return varValue
+}
+
+func GetAndExpectIntEnvVar(varName string) int {
+	varValue := GetAndExpectStringEnvVar(varName)
+	varIntValue, err := strconv.Atoi(varValue)
+	Expect(err).NotTo(HaveOccurred(), "Error Parsing "+varName)
+	return varIntValue
+}

test/e2e/storage/vsphere/vsphere_scale.go

@@ -18,7 +18,6 @@ package vsphere
 
 import (
 	"fmt"
-	"os"
 	"strconv"
 
 	. "github.com/onsi/ginkgo"
@@ -63,12 +62,11 @@ var _ = utils.SIGDescribe("vcp at scale [Feature:vsphere] ", func() {
 		volumeCount       int
 		numberOfInstances int
 		volumesPerPod     int
-		nodeVolumeMapChan chan map[string][]string
-		nodes             *v1.NodeList
 		policyName        string
 		datastoreName     string
+		nodeVolumeMapChan chan map[string][]string
+		nodes             *v1.NodeList
 		scNames           = []string{storageclass1, storageclass2, storageclass3, storageclass4}
-		err               error
 	)
 
 	BeforeEach(func() {
@@ -78,27 +76,15 @@ var _ = utils.SIGDescribe("vcp at scale [Feature:vsphere] ", func() {
 		nodeVolumeMapChan = make(chan map[string][]string)
 
 		// Read the environment variables
-		volumeCountStr := os.Getenv("VCP_SCALE_VOLUME_COUNT")
-		Expect(volumeCountStr).NotTo(BeEmpty(), "ENV VCP_SCALE_VOLUME_COUNT is not set")
-		volumeCount, err = strconv.Atoi(volumeCountStr)
-		Expect(err).NotTo(HaveOccurred(), "Error Parsing VCP_SCALE_VOLUME_COUNT")
-
-		volumesPerPodStr := os.Getenv("VCP_SCALE_VOLUME_PER_POD")
-		Expect(volumesPerPodStr).NotTo(BeEmpty(), "ENV VCP_SCALE_VOLUME_PER_POD is not set")
-		volumesPerPod, err = strconv.Atoi(volumesPerPodStr)
-		Expect(err).NotTo(HaveOccurred(), "Error Parsing VCP_SCALE_VOLUME_PER_POD")
-
-		numberOfInstancesStr := os.Getenv("VCP_SCALE_INSTANCES")
-		Expect(numberOfInstancesStr).NotTo(BeEmpty(), "ENV VCP_SCALE_INSTANCES is not set")
-		numberOfInstances, err = strconv.Atoi(numberOfInstancesStr)
-		Expect(err).NotTo(HaveOccurred(), "Error Parsing VCP_SCALE_INSTANCES")
+		volumeCount = GetAndExpectIntEnvVar(VCPScaleVolumeCount)
+		volumesPerPod = GetAndExpectIntEnvVar(VCPScaleVolumesPerPod)
+
+		numberOfInstances = GetAndExpectIntEnvVar(VCPScaleInstances)
 		Expect(numberOfInstances > 5).NotTo(BeTrue(), "Maximum allowed instances are 5")
 		Expect(numberOfInstances > volumeCount).NotTo(BeTrue(), "Number of instances should be less than the total volume count")
 
-		policyName = os.Getenv("VSPHERE_SPBM_POLICY_NAME")
-		datastoreName = os.Getenv("VSPHERE_DATASTORE")
-		Expect(policyName).NotTo(BeEmpty(), "ENV VSPHERE_SPBM_POLICY_NAME is not set")
-		Expect(datastoreName).NotTo(BeEmpty(), "ENV VSPHERE_DATASTORE is not set")
+		policyName = GetAndExpectStringEnvVar(SPBMPolicyName)
+		datastoreName = GetAndExpectStringEnvVar(StorageClassDatastoreName)
 
 		nodes = framework.GetReadySchedulableNodesOrDie(client)
 		if len(nodes.Items) < 2 {

test/e2e/storage/vsphere/vsphere_stress.go

@@ -18,8 +18,6 @@ package vsphere
 
 import (
 	"fmt"
-	"os"
-	"strconv"
 	"sync"
 
 	. "github.com/onsi/ginkgo"
@@ -50,10 +48,10 @@ var _ = utils.SIGDescribe("vsphere cloud provider stress [Feature:vsphere]", fun
 		namespace     string
 		instances     int
 		iterations    int
-		err           error
-		scNames       = []string{storageclass1, storageclass2, storageclass3, storageclass4}
 		policyName    string
 		datastoreName string
+		err           error
+		scNames       = []string{storageclass1, storageclass2, storageclass3, storageclass4}
 	)
 
 	BeforeEach(func() {
@@ -67,23 +65,16 @@ var _ = utils.SIGDescribe("vsphere cloud provider stress [Feature:vsphere]", fun
 		// if VCP_STRESS_INSTANCES = 12 and VCP_STRESS_ITERATIONS is 10. 12 threads will run in parallel for 10 times.
 		// Resulting 120 Volumes and POD Creation. Volumes will be provisioned with each different types of Storage Class,
 		// Each iteration creates PVC, verify PV is provisioned, then creates a pod, verify volume is attached to the node, and then delete the pod and delete pvc.
-		instancesStr := os.Getenv("VCP_STRESS_INSTANCES")
-		Expect(instancesStr).NotTo(BeEmpty(), "ENV VCP_STRESS_INSTANCES is not set")
-		instances, err = strconv.Atoi(instancesStr)
-		Expect(err).NotTo(HaveOccurred(), "Error Parsing VCP-STRESS-INSTANCES")
+		instances = GetAndExpectIntEnvVar(VCPStressInstances)
 		Expect(instances <= volumesPerNode*len(nodeList.Items)).To(BeTrue(), fmt.Sprintf("Number of Instances should be less or equal: %v", volumesPerNode*len(nodeList.Items)))
 		Expect(instances > len(scNames)).To(BeTrue(), "VCP_STRESS_INSTANCES should be greater than 3 to utilize all 4 types of storage classes")
 
-		iterationStr := os.Getenv("VCP_STRESS_ITERATIONS")
-		Expect(instancesStr).NotTo(BeEmpty(), "ENV VCP_STRESS_ITERATIONS is not set")
-		iterations, err = strconv.Atoi(iterationStr)
+		iterations = GetAndExpectIntEnvVar(VCPStressIterations)
 		Expect(err).NotTo(HaveOccurred(), "Error Parsing VCP_STRESS_ITERATIONS")
 		Expect(iterations > 0).To(BeTrue(), "VCP_STRESS_ITERATIONS should be greater than 0")
 
-		policyName = os.Getenv("VSPHERE_SPBM_POLICY_NAME")
-		datastoreName = os.Getenv("VSPHERE_DATASTORE")
-		Expect(policyName).NotTo(BeEmpty(), "ENV VSPHERE_SPBM_POLICY_NAME is not set")
-		Expect(datastoreName).NotTo(BeEmpty(), "ENV VSPHERE_DATASTORE is not set")
+		policyName = GetAndExpectStringEnvVar(SPBMPolicyName)
+		datastoreName = GetAndExpectStringEnvVar(StorageClassDatastoreName)
 	})
 
 	It("vsphere stress tests", func() {