refactor: Move GetCloudProvider to cluster

This lets us use labels (or annotations), meaning we can experiment with different clouds without changing the API. We also add initial (experimental/undocumented) support for exposing a "Metal" provider.
JBOClara · Aug 26, 2024 · 3646a61 · 3646a61
1 parent bdb283e
commit 3646a61
Show file tree

Hide file tree

Showing 40 changed files with 207 additions and 155 deletions.
diff --git a/clusterapi/bootstrap/controllers/kopsconfig_controller.go b/clusterapi/bootstrap/controllers/kopsconfig_controller.go
@@ -305,7 +305,7 @@ func (r *KopsConfigReconciler) buildBootstrapData(ctx context.Context) ([]byte,
 	// See https://github.com/kubernetes/kops/issues/10206 for details.
 	// TODO: nodeupScript.SetSysctls = setSysctls()
 
-	nodeupScript.CloudProvider = string(cluster.Spec.GetCloudProvider())
+	nodeupScript.CloudProvider = string(cluster.GetCloudProvider())
 
 	nodeupScriptResource, err := nodeupScript.Build()
 	if err != nil {

diff --git a/cmd/kops/create_instancegroup.go b/cmd/kops/create_instancegroup.go
@@ -199,7 +199,7 @@ func RunCreateInstanceGroup(ctx context.Context, f *util.Factory, out io.Writer,
 	}
 
 	ig.AddInstanceGroupNodeLabel()
-	if cluster.Spec.GetCloudProvider() == kopsapi.CloudProviderGCE {
+	if cluster.GetCloudProvider() == kopsapi.CloudProviderGCE {
 		fmt.Println("detected a GCE cluster; labeling nodes to receive metadata-proxy.")
 		ig.Spec.NodeLabels["cloud.google.com/metadata-proxy-ready"] = "true"
 	}

diff --git a/cmd/kops/get_cluster.go b/cmd/kops/get_cluster.go
@@ -219,7 +219,7 @@ func clusterOutputTable(clusters []*kopsapi.Cluster, out io.Writer) error {
 		return c.ObjectMeta.Name
 	})
 	t.AddColumn("CLOUD", func(c *kopsapi.Cluster) string {
-		return string(c.Spec.GetCloudProvider())
+		return string(c.GetCloudProvider())
 	})
 	t.AddColumn("ZONES", func(c *kopsapi.Cluster) string {
 		zones := sets.NewString()

diff --git a/cmd/kops/toolbox_instance-selector.go b/cmd/kops/toolbox_instance-selector.go
@@ -238,7 +238,7 @@ func RunToolboxInstanceSelector(ctx context.Context, f commandutils.Factory, out
 		return err
 	}
 
-	if cluster.Spec.GetCloudProvider() != kops.CloudProviderAWS {
+	if cluster.GetCloudProvider() != kops.CloudProviderAWS {
 		return fmt.Errorf("cannot select instance types from non-aws cluster")
 	}
 

diff --git a/cmd/kops/update_cluster.go b/cmd/kops/update_cluster.go
@@ -493,7 +493,7 @@ func completeUpdateClusterTarget(f commandutils.Factory, options *UpdateClusterO
 			cloudup.TargetDryRun,
 		}
 		for _, cp := range cloudup.TerraformCloudProviders {
-			if cluster.Spec.GetCloudProvider() == cp {
+			if cluster.GetCloudProvider() == cp {
 				completions = append(completions, cloudup.TargetTerraform)
 			}
 		}

diff --git a/nodeup/pkg/model/kubelet_test.go b/nodeup/pkg/model/kubelet_test.go
@@ -240,7 +240,7 @@ func BuildNodeupModelContext(model *testutils.Model) (*NodeupModelContext, error
 	nodeupModelContext := &NodeupModelContext{
 		Architecture: "amd64",
 		BootConfig: &nodeup.BootConfig{
-			CloudProvider: model.Cluster.Spec.GetCloudProvider(),
+			CloudProvider: model.Cluster.GetCloudProvider(),
 		},
 	}
 

diff --git a/pkg/acls/gce/storage.go b/pkg/acls/gce/storage.go
@@ -36,7 +36,7 @@ var _ acls.ACLStrategy = &gcsAclStrategy{}
 
 // GetACL returns the ACL to use if this is a google cloud storage path
 func (s *gcsAclStrategy) GetACL(ctx context.Context, p vfs.Path, cluster *kops.Cluster) (vfs.ACL, error) {
-	if cluster.Spec.GetCloudProvider() != kops.CloudProviderGCE {
+	if cluster.GetCloudProvider() != kops.CloudProviderGCE {
 		return nil, nil
 	}
 	gcsPath, ok := p.(*vfs.GSPath)

diff --git a/pkg/apis/kops/channel.go b/pkg/apis/kops/channel.go
@@ -293,6 +293,14 @@ const (
 	CloudProviderOpenstack CloudProviderID = "openstack"
 	CloudProviderAzure     CloudProviderID = "azure"
 	CloudProviderScaleway  CloudProviderID = "scaleway"
+
+	// Experimental cloud providers
+	CloudProviderMetal CloudProviderID = "metal"
+
+	// If you add any cloud providers here, you must add them to:
+	// * Convert_kops_ClusterSpec_To_v1alpha2_ClusterSpec
+	// * cluster.GetCloudProvider()
+	// * type CloudProviderSpec
 )
 
 // FindImage returns the image for the cloudprovider, or nil if none found

diff --git a/pkg/apis/kops/cluster.go b/pkg/apis/kops/cluster.go
@@ -28,6 +28,10 @@ import (
 	"k8s.io/kops/upup/pkg/fi/utils"
 )
 
+// AlphaLabelCloudProvider lets us experiment with a cloud provider,
+// before adding it to the API.
+const AlphaLabelCloudProvider = "alpha.kops.k8s.io/cloud"
+
 // +genclient
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 
@@ -946,20 +950,25 @@ func (c *ClusterSpec) IsKopsControllerIPAM() bool {
 	return c.IsIPv6Only()
 }
 
-func (c *ClusterSpec) GetCloudProvider() CloudProviderID {
-	if c.CloudProvider.AWS != nil {
+func (c *Cluster) GetCloudProvider() CloudProviderID {
+	if c.Labels[AlphaLabelCloudProvider] == "metal" {
+		return CloudProviderMetal
+	}
+
+	spec := c.Spec
+	if spec.CloudProvider.AWS != nil {
 		return CloudProviderAWS
-	} else if c.CloudProvider.Azure != nil {
+	} else if spec.CloudProvider.Azure != nil {
 		return CloudProviderAzure
-	} else if c.CloudProvider.DO != nil {
+	} else if spec.CloudProvider.DO != nil {
 		return CloudProviderDO
-	} else if c.CloudProvider.GCE != nil {
+	} else if spec.CloudProvider.GCE != nil {
 		return CloudProviderGCE
-	} else if c.CloudProvider.Hetzner != nil {
+	} else if spec.CloudProvider.Hetzner != nil {
 		return CloudProviderHetzner
-	} else if c.CloudProvider.Openstack != nil {
+	} else if spec.CloudProvider.Openstack != nil {
 		return CloudProviderOpenstack
-	} else if c.CloudProvider.Scaleway != nil {
+	} else if spec.CloudProvider.Scaleway != nil {
 		return CloudProviderScaleway
 	}
 	return ""

diff --git a/pkg/apis/kops/model/features.go b/pkg/apis/kops/model/features.go
@@ -41,7 +41,7 @@ func UseChallengeCallback(cloudProvider kops.CloudProviderID) bool {
 // UseKopsControllerForNodeConfig checks if nodeup should use kops-controller to get nodeup.Config.
 func UseKopsControllerForNodeConfig(cluster *kops.Cluster) bool {
 	if cluster.UsesLegacyGossip() {
-		switch cluster.Spec.GetCloudProvider() {
+		switch cluster.GetCloudProvider() {
 		case kops.CloudProviderGCE:
 			// We can use cloud-discovery here.
 		case kops.CloudProviderHetzner, kops.CloudProviderScaleway, kops.CloudProviderDO:

diff --git a/pkg/apis/kops/v1alpha2/conversion.go b/pkg/apis/kops/v1alpha2/conversion.go
@@ -419,7 +419,27 @@ func Convert_kops_ClusterSpec_To_v1alpha2_ClusterSpec(in *kops.ClusterSpec, out
 	if out.API.IsEmpty() {
 		out.LegacyAPI = nil
 	}
-	out.LegacyCloudProvider = string(in.GetCloudProvider())
+	if in.CloudProvider.AWS != nil {
+		out.LegacyCloudProvider = string(kops.CloudProviderAWS)
+	}
+	if in.CloudProvider.Azure != nil {
+		out.LegacyCloudProvider = string(kops.CloudProviderAzure)
+	}
+	if in.CloudProvider.DO != nil {
+		out.LegacyCloudProvider = string(kops.CloudProviderDO)
+	}
+	if in.CloudProvider.GCE != nil {
+		out.LegacyCloudProvider = string(kops.CloudProviderGCE)
+	}
+	if in.CloudProvider.Hetzner != nil {
+		out.LegacyCloudProvider = string(kops.CloudProviderHetzner)
+	}
+	if in.CloudProvider.Openstack != nil {
+		out.LegacyCloudProvider = string(kops.CloudProviderOpenstack)
+	}
+	if in.CloudProvider.Scaleway != nil {
+		out.LegacyCloudProvider = string(kops.CloudProviderScaleway)
+	}
 	switch kops.CloudProviderID(out.LegacyCloudProvider) {
 	case kops.CloudProviderAWS:
 		aws := in.CloudProvider.AWS

diff --git a/pkg/apis/kops/validation/instancegroup.go b/pkg/apis/kops/validation/instancegroup.go
@@ -239,7 +239,7 @@ func CrossValidateInstanceGroup(g *kops.InstanceGroup, cluster *kops.Cluster, cl
 	}
 
 	if g.Spec.Role == kops.InstanceGroupRoleAPIServer {
-		if cluster.Spec.GetCloudProvider() != kops.CloudProviderAWS {
+		if cluster.GetCloudProvider() != kops.CloudProviderAWS {
 			allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "role"), "APIServer role only supported on AWS"))
 		}
 		if cluster.UsesNoneDNS() {
@@ -262,7 +262,7 @@ func CrossValidateInstanceGroup(g *kops.InstanceGroup, cluster *kops.Cluster, cl
 		}
 	}
 
-	if cluster.Spec.GetCloudProvider() == kops.CloudProviderAWS {
+	if cluster.GetCloudProvider() == kops.CloudProviderAWS {
 		if g.Spec.RootVolume != nil && g.Spec.RootVolume.Type != nil {
 			allErrs = append(allErrs, IsValidValue(field.NewPath("spec", "rootVolume", "type"), g.Spec.RootVolume.Type, []string{"standard", "gp3", "gp2", "io1", "io2"})...)
 		}
@@ -292,7 +292,7 @@ func CrossValidateInstanceGroup(g *kops.InstanceGroup, cluster *kops.Cluster, cl
 	}
 
 	if g.Spec.Containerd != nil {
-		allErrs = append(allErrs, validateContainerdConfig(&cluster.Spec, g.Spec.Containerd, field.NewPath("spec", "containerd"), false)...)
+		allErrs = append(allErrs, validateContainerdConfig(cluster, g.Spec.Containerd, field.NewPath("spec", "containerd"), false)...)
 	}
 
 	return allErrs

diff --git a/pkg/apis/kops/validation/legacy.go b/pkg/apis/kops/validation/legacy.go
@@ -150,7 +150,7 @@ func ValidateCluster(c *kops.Cluster, strict bool, vfsContext *vfs.VFSContext) f
 	{
 
 		var k8sCloudProvider string
-		switch c.Spec.GetCloudProvider() {
+		switch c.GetCloudProvider() {
 		case kops.CloudProviderAWS:
 			k8sCloudProvider = "aws"
 		case kops.CloudProviderGCE:
@@ -275,7 +275,7 @@ func DeepValidate(c *kops.Cluster, groups []*kops.InstanceGroup, strict bool, vf
 		errs := CrossValidateInstanceGroup(g, c, cloud, strict)
 
 		// Additional cloud-specific validation rules
-		if c.Spec.GetCloudProvider() != kops.CloudProviderAWS && len(g.Spec.Volumes) > 0 {
+		if c.GetCloudProvider() != kops.CloudProviderAWS && len(g.Spec.Volumes) > 0 {
 			errs = append(errs, field.Forbidden(field.NewPath("spec", "volumes"), "instancegroup volumes are only available with aws at present"))
 		}