From ddaa0dd5029d3b5d58be08510ee87f22662f1051 Mon Sep 17 00:00:00 2001 From: Jesse Haka Date: Sat, 22 Jun 2019 16:32:33 +0300 Subject: [PATCH] Possibility to use openstack without lbaas --- cmd/kops/create_cluster.go | 38 +++--- docs/tutorial/openstack.md | 12 ++ pkg/model/openstackmodel/servergroup.go | 40 +++--- pkg/resources/openstack/securitygroups.go | 3 +- upup/pkg/fi/cloudup/openstack/BUILD.bazel | 1 + upup/pkg/fi/cloudup/openstack/cloud.go | 115 ++++++++++++------ upup/pkg/fi/cloudup/openstack/floatingip.go | 2 +- upup/pkg/fi/cloudup/openstack/loadbalancer.go | 54 +++++++- .../pkg/fi/cloudup/openstacktasks/BUILD.bazel | 1 + .../fi/cloudup/openstacktasks/floatingip.go | 44 ++++++- upup/pkg/fi/cloudup/openstacktasks/subnet.go | 2 +- 11 files changed, 238 insertions(+), 74 deletions(-) diff --git a/cmd/kops/create_cluster.go b/cmd/kops/create_cluster.go index a8725206b2639..26ecc214e3abd 100644 --- a/cmd/kops/create_cluster.go +++ b/cmd/kops/create_cluster.go @@ -912,20 +912,10 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e if cluster.Spec.CloudConfig == nil { cluster.Spec.CloudConfig = &api.CloudConfiguration{} } - provider := "haproxy" - if c.OpenstackLBOctavia { - provider = "octavia" - } cluster.Spec.CloudConfig.Openstack = &api.OpenstackConfiguration{ Router: &api.OpenstackRouter{ ExternalNetwork: fi.String(c.OpenstackExternalNet), }, - Loadbalancer: &api.OpenstackLoadbalancerConfig{ - FloatingNetwork: fi.String(c.OpenstackExternalNet), - Method: fi.String("ROUND_ROBIN"), - Provider: fi.String(provider), - UseOctavia: fi.Bool(c.OpenstackLBOctavia), - }, BlockStorage: &api.OpenstackBlockStorageConfig{ Version: fi.String("v2"), IgnoreAZ: fi.Bool(c.OpenstackStorageIgnoreAZ), @@ -942,9 +932,6 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e if c.OpenstackExternalSubnet != "" { cluster.Spec.CloudConfig.Openstack.Router.ExternalSubnet = fi.String(c.OpenstackExternalSubnet) } - if c.OpenstackLbSubnet != "" { - cluster.Spec.CloudConfig.Openstack.Loadbalancer.FloatingSubnet = fi.String(c.OpenstackLbSubnet) - } } } @@ -1163,7 +1150,9 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e cluster.Spec.API = &api.AccessSpec{} } if cluster.Spec.API.IsEmpty() { - if c.APILoadBalancerType != "" { + if c.Cloud == "openstack" { + initializeOpenstackAPI(c, cluster) + } else if c.APILoadBalancerType != "" { cluster.Spec.API.LoadBalancer = &api.LoadBalancerAccessSpec{} } else { switch cluster.Spec.Topology.Masters { @@ -1443,6 +1432,27 @@ func parseCloudLabels(s string) (map[string]string, error) { return m, nil } +func initializeOpenstackAPI(c *CreateClusterOptions, cluster *api.Cluster) { + if c.APILoadBalancerType != "" { + cluster.Spec.API.LoadBalancer = &api.LoadBalancerAccessSpec{} + provider := "haproxy" + if c.OpenstackLBOctavia { + provider = "octavia" + } + + cluster.Spec.CloudConfig.Openstack.Loadbalancer = &api.OpenstackLoadbalancerConfig{ + FloatingNetwork: fi.String(c.OpenstackExternalNet), + Method: fi.String("ROUND_ROBIN"), + Provider: fi.String(provider), + UseOctavia: fi.Bool(c.OpenstackLBOctavia), + } + + if c.OpenstackLbSubnet != "" { + cluster.Spec.CloudConfig.Openstack.Loadbalancer.FloatingSubnet = fi.String(c.OpenstackLbSubnet) + } + } +} + func getZoneToSubnetProviderID(VPCID string, region string, subnetIDs []string) (map[string]string, error) { res := make(map[string]string) cloudTags := map[string]string{} diff --git a/docs/tutorial/openstack.md b/docs/tutorial/openstack.md index 5ab13846c6ce5..de5e8c46a1e21 100644 --- a/docs/tutorial/openstack.md +++ b/docs/tutorial/openstack.md @@ -139,3 +139,15 @@ Finally ``` kops update cluster --name --yes ``` + +# Using OpenStack without lbaas +Some OpenStack installations does not include installation of lbaas component. That is why we have added very-experimental support of installing OpenStack kops without lbaas. You can install it using: + +``` +kops create cluster \ + --cloud openstack \ + ... (like usually) + --api-loadbalancer-type="" +``` + +The biggest problem currently when installing without loadbalancer is that kubectl requests outside cluster is always going to first master. External loadbalancer is one option which can solve this issue. diff --git a/pkg/model/openstackmodel/servergroup.go b/pkg/model/openstackmodel/servergroup.go index b6c1796c55b36..b329905de6740 100644 --- a/pkg/model/openstackmodel/servergroup.go +++ b/pkg/model/openstackmodel/servergroup.go @@ -76,6 +76,14 @@ func (b *ServerGroupModelBuilder) buildInstances(c *fi.ModelBuilderContext, sg * igUserData = fi.String(startupStr) } + var securityGroups []*openstacktasks.SecurityGroup + securityGroupName := b.SecurityGroupName(ig.Spec.Role) + securityGroups = append(securityGroups, b.LinkToSecurityGroup(securityGroupName)) + + if b.Cluster.Spec.CloudConfig.Openstack.Loadbalancer == nil { + securityGroups = append(securityGroups, b.LinkToSecurityGroup(b.Cluster.Spec.MasterPublicName)) + } + // In the future, OpenStack will use Machine API to manage groups, // for now create d.InstanceGroups.Spec.MinSize amount of servers for i := int32(0); i < *ig.Spec.MinSize; i++ { @@ -87,8 +95,6 @@ func (b *ServerGroupModelBuilder) buildInstances(c *fi.ModelBuilderContext, sg * iName := strings.ToLower(fmt.Sprintf("%s-%d.%s", ig.Name, i+1, b.ClusterName())) instanceName := fi.String(strings.Replace(iName, ".", "-", -1)) - securityGroupName := b.SecurityGroupName(ig.Spec.Role) - securityGroup := b.LinkToSecurityGroup(securityGroupName) var az *string if len(ig.Spec.Subnets) > 0 { // bastion subnet name is not actual zone name, it contains "utility-" prefix @@ -102,7 +108,7 @@ func (b *ServerGroupModelBuilder) buildInstances(c *fi.ModelBuilderContext, sg * portTask := &openstacktasks.Port{ Name: fi.String(fmt.Sprintf("%s-%s", "port", *instanceName)), Network: b.LinkToNetwork(), - SecurityGroups: append([]*openstacktasks.SecurityGroup{}, securityGroup), + SecurityGroups: securityGroups, Lifecycle: b.Lifecycle, } c.AddTask(portTask) @@ -135,13 +141,14 @@ func (b *ServerGroupModelBuilder) buildInstances(c *fi.ModelBuilderContext, sg * } c.AddTask(t) case kops.InstanceGroupRoleMaster: - if !b.UseLoadBalancerForAPI() { + if b.Cluster.Spec.CloudConfig.Openstack.Loadbalancer == nil { t := &openstacktasks.FloatingIP{ Name: fi.String(fmt.Sprintf("%s-%s", "fip", *instanceTask.Name)), Server: instanceTask, Lifecycle: b.Lifecycle, } c.AddTask(t) + b.associateFIPToKeypair(c, t) } default: if !b.UsesSSHBastion() { @@ -158,6 +165,19 @@ func (b *ServerGroupModelBuilder) buildInstances(c *fi.ModelBuilderContext, sg * return nil } +func (b *ServerGroupModelBuilder) associateFIPToKeypair(c *fi.ModelBuilderContext, fipTask *openstacktasks.FloatingIP) error { + // Ensure the floating IP is included in the TLS certificate, + // if we're not going to use an alias for it + // TODO: I don't love this technique for finding the task by name & modifying it + masterKeypairTask, found := c.Tasks["Keypair/master"] + if !found { + return fmt.Errorf("keypair/master task not found") + } + masterKeypair := masterKeypairTask.(*fitasks.Keypair) + masterKeypair.AlternateNameTasks = append(masterKeypair.AlternateNameTasks, fipTask) + return nil +} + func (b *ServerGroupModelBuilder) Build(c *fi.ModelBuilderContext) error { clusterName := b.ClusterName() @@ -184,7 +204,7 @@ func (b *ServerGroupModelBuilder) Build(c *fi.ModelBuilderContext) error { } } - if b.UseLoadBalancerForAPI() { + if b.Cluster.Spec.CloudConfig.Openstack.Loadbalancer != nil { lbSubnetName := b.MasterInstanceGroups()[0].Spec.Subnets[0] lbTask := &openstacktasks.LB{ Name: fi.String(b.Cluster.Spec.MasterPublicName), @@ -202,15 +222,7 @@ func (b *ServerGroupModelBuilder) Build(c *fi.ModelBuilderContext) error { c.AddTask(lbfipTask) if dns.IsGossipHostname(b.Cluster.Name) || b.UsePrivateDNS() { - // Ensure the floating IP is included in the TLS certificate, - // if we're not going to use an alias for it - // TODO: I don't love this technique for finding the task by name & modifying it - masterKeypairTask, found := c.Tasks["Keypair/master"] - if !found { - return fmt.Errorf("keypair/master task not found") - } - masterKeypair := masterKeypairTask.(*fitasks.Keypair) - masterKeypair.AlternateNameTasks = append(masterKeypair.AlternateNameTasks, lbfipTask) + b.associateFIPToKeypair(c, lbfipTask) } poolTask := &openstacktasks.LBPool{ diff --git a/pkg/resources/openstack/securitygroups.go b/pkg/resources/openstack/securitygroups.go index 992aab01be4af..98ccfcd54ee57 100644 --- a/pkg/resources/openstack/securitygroups.go +++ b/pkg/resources/openstack/securitygroups.go @@ -17,6 +17,7 @@ limitations under the License. package openstack import ( + "fmt" "strings" sg "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/security/groups" @@ -39,7 +40,7 @@ func (os *clusterDiscoveryOS) ListSecurityGroups() ([]*resources.Resource, error } for _, sg := range sgs { - if strings.HasSuffix(sg.Name, os.clusterName) { + if strings.HasSuffix(sg.Name, fmt.Sprintf(".%s", os.clusterName)) { resourceTracker := &resources.Resource{ Name: sg.Name, ID: sg.ID, diff --git a/upup/pkg/fi/cloudup/openstack/BUILD.bazel b/upup/pkg/fi/cloudup/openstack/BUILD.bazel index 7508f1afa1295..4d25cd785d0cb 100644 --- a/upup/pkg/fi/cloudup/openstack/BUILD.bazel +++ b/upup/pkg/fi/cloudup/openstack/BUILD.bazel @@ -57,6 +57,7 @@ go_library( "//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/ports:go_default_library", "//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/subnets:go_default_library", "//vendor/github.com/gophercloud/gophercloud/pagination:go_default_library", + "//vendor/github.com/mitchellh/mapstructure:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//vendor/k8s.io/klog:go_default_library", diff --git a/upup/pkg/fi/cloudup/openstack/cloud.go b/upup/pkg/fi/cloudup/openstack/cloud.go index 3269a91ab549c..983002fa9c04b 100644 --- a/upup/pkg/fi/cloudup/openstack/cloud.go +++ b/upup/pkg/fi/cloudup/openstack/cloud.go @@ -47,6 +47,7 @@ import ( "github.com/gophercloud/gophercloud/openstack/networking/v2/networks" "github.com/gophercloud/gophercloud/openstack/networking/v2/ports" "github.com/gophercloud/gophercloud/openstack/networking/v2/subnets" + "github.com/mitchellh/mapstructure" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog" @@ -407,30 +408,36 @@ func NewOpenstackCloud(tags map[string]string, spec *kops.ClusterSpec) (Openstac } spec.CloudConfig.Openstack.Loadbalancer.FloatingNetworkID = fi.String(lbNet[0].ID) } - if spec.CloudConfig.Openstack.Loadbalancer.UseOctavia != nil { - octavia = fi.BoolValue(spec.CloudConfig.Openstack.Loadbalancer.UseOctavia) - } - if spec.CloudConfig.Openstack.Loadbalancer.FloatingSubnet != nil { - c.floatingSubnet = spec.CloudConfig.Openstack.Loadbalancer.FloatingSubnet + if spec.CloudConfig.Openstack.Loadbalancer != nil { + if spec.CloudConfig.Openstack.Loadbalancer.UseOctavia != nil { + octavia = fi.BoolValue(spec.CloudConfig.Openstack.Loadbalancer.UseOctavia) + } + if spec.CloudConfig.Openstack.Loadbalancer.FloatingSubnet != nil { + c.floatingSubnet = spec.CloudConfig.Openstack.Loadbalancer.FloatingSubnet + } } } c.useOctavia = octavia var lbClient *gophercloud.ServiceClient - if octavia { - klog.V(2).Infof("Openstack using Octavia lbaasv2 api") - lbClient, err = os.NewLoadBalancerV2(provider, gophercloud.EndpointOpts{ - Region: region, - }) - if err != nil { - return nil, fmt.Errorf("error building lb client: %v", err) - } - } else { - klog.V(2).Infof("Openstack using deprecated lbaasv2 api") - lbClient, err = os.NewNetworkV2(provider, gophercloud.EndpointOpts{ - Region: region, - }) - if err != nil { - return nil, fmt.Errorf("error building lb client: %v", err) + if spec != nil && spec.CloudConfig != nil && spec.CloudConfig.Openstack != nil { + if spec.CloudConfig.Openstack.Loadbalancer != nil && octavia { + klog.V(2).Infof("Openstack using Octavia lbaasv2 api") + lbClient, err = os.NewLoadBalancerV2(provider, gophercloud.EndpointOpts{ + Region: region, + }) + if err != nil { + return nil, fmt.Errorf("error building lb client: %v", err) + } + } else if spec.CloudConfig.Openstack.Loadbalancer != nil { + klog.V(2).Infof("Openstack using deprecated lbaasv2 api") + lbClient, err = os.NewNetworkV2(provider, gophercloud.EndpointOpts{ + Region: region, + }) + if err != nil { + return nil, fmt.Errorf("error building lb client: %v", err) + } + } else { + klog.V(2).Infof("Openstack disabled loadbalancer support") } } c.lbClient = lbClient @@ -547,30 +554,64 @@ func (c *openstackCloud) GetCloudTags() map[string]string { return c.tags } +type Address struct { + IPType string `mapstructure:"OS-EXT-IPS:type"` + Addr string +} + func (c *openstackCloud) GetApiIngressStatus(cluster *kops.Cluster) ([]kops.ApiIngressStatus, error) { var ingresses []kops.ApiIngressStatus - if cluster.Spec.MasterPublicName != "" { - // Note that this must match OpenstackModel lb name - klog.V(2).Infof("Querying Openstack to find Loadbalancers for API (%q)", cluster.Name) - lbList, err := c.ListLBs(loadbalancers.ListOpts{ - Name: cluster.Spec.MasterPublicName, - }) - if err != nil { - return ingresses, fmt.Errorf("GetApiIngressStatus: Failed to list openstack loadbalancers: %v", err) + if cluster.Spec.CloudConfig.Openstack.Loadbalancer != nil { + if cluster.Spec.MasterPublicName != "" { + // Note that this must match OpenstackModel lb name + klog.V(2).Infof("Querying Openstack to find Loadbalancers for API (%q)", cluster.Name) + lbList, err := c.ListLBs(loadbalancers.ListOpts{ + Name: cluster.Spec.MasterPublicName, + }) + if err != nil { + return ingresses, fmt.Errorf("GetApiIngressStatus: Failed to list openstack loadbalancers: %v", err) + } + // Must Find Floating IP related to this lb + fips, err := c.ListFloatingIPs() + if err != nil { + return ingresses, fmt.Errorf("GetApiIngressStatus: Failed to list floating IP's: %v", err) + } + + for _, lb := range lbList { + for _, fip := range fips { + if fip.FixedIP == lb.VipAddress { + + ingresses = append(ingresses, kops.ApiIngressStatus{ + IP: fip.IP, + }) + } + } + } } - // Must Find Floating IP related to this lb - fips, err := c.ListFloatingIPs() + } else { + instances, err := c.ListInstances(servers.ListOpts{}) if err != nil { - return ingresses, fmt.Errorf("GetApiIngressStatus: Failed to list floating IP's: %v", err) + return ingresses, fmt.Errorf("GetApiIngressStatus: Failed to list master nodes: %v", err) } - for _, lb := range lbList { - for _, fip := range fips { - if fip.FixedIP == lb.VipAddress { + for _, instance := range instances { + val, ok := instance.Metadata["k8s"] + val2, ok2 := instance.Metadata["KopsInstanceGroup"] + if ok && val == cluster.Name && ok2 && strings.HasPrefix(val2, "master") { + var addresses map[string][]Address + err := mapstructure.Decode(instance.Addresses, &addresses) + if err != nil { + return nil, err + } - ingresses = append(ingresses, kops.ApiIngressStatus{ - IP: fip.IP, - }) + for _, addrList := range addresses { + for _, props := range addrList { + if props.IPType == "floating" { + ingresses = append(ingresses, kops.ApiIngressStatus{ + IP: props.Addr, + }) + } + } } } } diff --git a/upup/pkg/fi/cloudup/openstack/floatingip.go b/upup/pkg/fi/cloudup/openstack/floatingip.go index 9c00e991552e2..13f683da75300 100644 --- a/upup/pkg/fi/cloudup/openstack/floatingip.go +++ b/upup/pkg/fi/cloudup/openstack/floatingip.go @@ -30,7 +30,7 @@ func (c *openstackCloud) GetFloatingIP(id string) (fip *floatingips.FloatingIP, fip, err = floatingips.Get(c.ComputeClient(), id).Extract() if err != nil { - return false, fmt.Errorf("GetFloatingIP: fetching floating IP failed: %v", err) + return false, fmt.Errorf("GetFloatingIP: fetching floating IP (%s) failed: %v", id, err) } return true, nil }) diff --git a/upup/pkg/fi/cloudup/openstack/loadbalancer.go b/upup/pkg/fi/cloudup/openstack/loadbalancer.go index 2b2078c13bffb..91dfce638cbf2 100644 --- a/upup/pkg/fi/cloudup/openstack/loadbalancer.go +++ b/upup/pkg/fi/cloudup/openstack/loadbalancer.go @@ -29,7 +29,9 @@ import ( ) func (c *openstackCloud) ListMonitors(opts monitors.ListOpts) (monitorList []monitors.Monitor, err error) { - + if c.LoadBalancerClient() == nil { + return monitorList, fmt.Errorf("Loadbalancer support not available in this deployment") + } done, err := vfs.RetryWithBackoff(readBackoff, func() (bool, error) { allPages, err := monitors.List(c.LoadBalancerClient(), opts).AllPages() if err != nil { @@ -51,6 +53,9 @@ func (c *openstackCloud) ListMonitors(opts monitors.ListOpts) (monitorList []mon } func (c *openstackCloud) DeleteMonitor(monitorID string) error { + if c.LoadBalancerClient() == nil { + return fmt.Errorf("Loadbalancer support not available in this deployment") + } done, err := vfs.RetryWithBackoff(writeBackoff, func() (bool, error) { err := monitors.Delete(c.LoadBalancerClient(), monitorID).ExtractErr() if err != nil && !isNotFound(err) { @@ -68,6 +73,10 @@ func (c *openstackCloud) DeleteMonitor(monitorID string) error { } func (c *openstackCloud) DeletePool(poolID string) error { + if c.LoadBalancerClient() == nil { + return fmt.Errorf("Loadbalancer support not available in this deployment") + } + done, err := vfs.RetryWithBackoff(writeBackoff, func() (bool, error) { err := v2pools.Delete(c.LoadBalancerClient(), poolID).ExtractErr() if err != nil && !isNotFound(err) { @@ -85,6 +94,10 @@ func (c *openstackCloud) DeletePool(poolID string) error { } func (c *openstackCloud) DeleteListener(listenerID string) error { + if c.LoadBalancerClient() == nil { + return fmt.Errorf("Loadbalancer support not available in this deployment") + } + done, err := vfs.RetryWithBackoff(writeBackoff, func() (bool, error) { err := listeners.Delete(c.LoadBalancerClient(), listenerID).ExtractErr() if err != nil && !isNotFound(err) { @@ -102,6 +115,10 @@ func (c *openstackCloud) DeleteListener(listenerID string) error { } func (c *openstackCloud) DeleteLB(lbID string, opts loadbalancers.DeleteOpts) error { + if c.LoadBalancerClient() == nil { + return fmt.Errorf("Loadbalancer support not available in this deployment") + } + done, err := vfs.RetryWithBackoff(writeBackoff, func() (bool, error) { err := loadbalancers.Delete(c.LoadBalancerClient(), lbID, opts).ExtractErr() if err != nil && !isNotFound(err) { @@ -119,8 +136,11 @@ func (c *openstackCloud) DeleteLB(lbID string, opts loadbalancers.DeleteOpts) er } func (c *openstackCloud) CreateLB(opt loadbalancers.CreateOptsBuilder) (*loadbalancers.LoadBalancer, error) { - var i *loadbalancers.LoadBalancer + if c.LoadBalancerClient() == nil { + return nil, fmt.Errorf("Loadbalancer support not available in this deployment") + } + var i *loadbalancers.LoadBalancer done, err := vfs.RetryWithBackoff(writeBackoff, func() (bool, error) { v, err := loadbalancers.Create(c.LoadBalancerClient(), opt).Extract() if err != nil { @@ -139,6 +159,9 @@ func (c *openstackCloud) CreateLB(opt loadbalancers.CreateOptsBuilder) (*loadbal } func (c *openstackCloud) GetLB(loadbalancerID string) (lb *loadbalancers.LoadBalancer, err error) { + if c.LoadBalancerClient() == nil { + return nil, fmt.Errorf("Loadbalancer support not available in this deployment") + } done, err := vfs.RetryWithBackoff(readBackoff, func() (bool, error) { lb, err = loadbalancers.Get(c.LoadBalancerClient(), loadbalancerID).Extract() @@ -158,6 +181,10 @@ func (c *openstackCloud) GetLB(loadbalancerID string) (lb *loadbalancers.LoadBal // ListLBs will list load balancers func (c *openstackCloud) ListLBs(opt loadbalancers.ListOptsBuilder) (lbs []loadbalancers.LoadBalancer, err error) { + if c.LoadBalancerClient() == nil { + // skip error because cluster delete will otherwise fail + return lbs, nil + } done, err := vfs.RetryWithBackoff(readBackoff, func() (bool, error) { allPages, err := loadbalancers.List(c.LoadBalancerClient(), opt).AllPages() @@ -180,6 +207,10 @@ func (c *openstackCloud) ListLBs(opt loadbalancers.ListOptsBuilder) (lbs []loadb } func (c *openstackCloud) GetPool(poolID string, memberID string) (member *v2pools.Member, err error) { + if c.LoadBalancerClient() == nil { + return nil, fmt.Errorf("Loadbalancer support not available in this deployment") + } + done, err := vfs.RetryWithBackoff(readBackoff, func() (bool, error) { member, err = v2pools.GetMember(c.LoadBalancerClient(), poolID, memberID).Extract() if err != nil { @@ -197,6 +228,9 @@ func (c *openstackCloud) GetPool(poolID string, memberID string) (member *v2pool } func (c *openstackCloud) AssociateToPool(server *servers.Server, poolID string, opts v2pools.CreateMemberOpts) (association *v2pools.Member, err error) { + if c.LoadBalancerClient() == nil { + return nil, fmt.Errorf("Loadbalancer support not available in this deployment") + } done, err := vfs.RetryWithBackoff(writeBackoff, func() (bool, error) { association, err = v2pools.GetMember(c.LoadBalancerClient(), poolID, server.ID).Extract() @@ -221,6 +255,10 @@ func (c *openstackCloud) AssociateToPool(server *servers.Server, poolID string, } func (c *openstackCloud) CreatePool(opts v2pools.CreateOpts) (pool *v2pools.Pool, err error) { + if c.LoadBalancerClient() == nil { + return nil, fmt.Errorf("Loadbalancer support not available in this deployment") + } + done, err := vfs.RetryWithBackoff(writeBackoff, func() (bool, error) { pool, err = v2pools.Create(c.LoadBalancerClient(), opts).Extract() if err != nil { @@ -238,6 +276,10 @@ func (c *openstackCloud) CreatePool(opts v2pools.CreateOpts) (pool *v2pools.Pool } func (c *openstackCloud) ListPools(opts v2pools.ListOpts) (poolList []v2pools.Pool, err error) { + if c.LoadBalancerClient() == nil { + return poolList, fmt.Errorf("Loadbalancer support not available in this deployment") + } + done, err := vfs.RetryWithBackoff(readBackoff, func() (bool, error) { poolPage, err := v2pools.List(c.LoadBalancerClient(), opts).AllPages() if err != nil { @@ -259,6 +301,10 @@ func (c *openstackCloud) ListPools(opts v2pools.ListOpts) (poolList []v2pools.Po } func (c *openstackCloud) ListListeners(opts listeners.ListOpts) (listenerList []listeners.Listener, err error) { + if c.LoadBalancerClient() == nil { + return listenerList, fmt.Errorf("Loadbalancer support not available in this deployment") + } + done, err := vfs.RetryWithBackoff(readBackoff, func() (bool, error) { listenerPage, err := listeners.List(c.LoadBalancerClient(), opts).AllPages() if err != nil { @@ -280,6 +326,10 @@ func (c *openstackCloud) ListListeners(opts listeners.ListOpts) (listenerList [] } func (c *openstackCloud) CreateListener(opts listeners.CreateOpts) (listener *listeners.Listener, err error) { + if c.LoadBalancerClient() == nil { + return nil, fmt.Errorf("Loadbalancer support not available in this deployment") + } + done, err := vfs.RetryWithBackoff(readBackoff, func() (bool, error) { listener, err = listeners.Create(c.LoadBalancerClient(), opts).Extract() if err != nil { diff --git a/upup/pkg/fi/cloudup/openstacktasks/BUILD.bazel b/upup/pkg/fi/cloudup/openstacktasks/BUILD.bazel index d4e22122e4791..a4bc28be4f02c 100644 --- a/upup/pkg/fi/cloudup/openstacktasks/BUILD.bazel +++ b/upup/pkg/fi/cloudup/openstacktasks/BUILD.bazel @@ -60,6 +60,7 @@ go_library( "//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/networks:go_default_library", "//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/ports:go_default_library", "//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/subnets:go_default_library", + "//vendor/github.com/mitchellh/mapstructure:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//vendor/k8s.io/klog:go_default_library", ], diff --git a/upup/pkg/fi/cloudup/openstacktasks/floatingip.go b/upup/pkg/fi/cloudup/openstacktasks/floatingip.go index eab3f214d68d3..29fa680a5a791 100644 --- a/upup/pkg/fi/cloudup/openstacktasks/floatingip.go +++ b/upup/pkg/fi/cloudup/openstacktasks/floatingip.go @@ -22,6 +22,7 @@ import ( "github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/floatingips" l3floatingip "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/floatingips" + "github.com/mitchellh/mapstructure" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog" "k8s.io/kops/upup/pkg/fi" @@ -70,6 +71,36 @@ func findL3Floating(cloud openstack.OpenstackCloud, opts l3floatingip.ListOpts) return result, nil } +func (e *FloatingIP) findServerFloatingIP(context *fi.Context, cloud openstack.OpenstackCloud) (*string, error) { + var result *string + _, err := vfs.RetryWithBackoff(readBackoff, func() (bool, error) { + server, err := cloud.GetInstance(fi.StringValue(e.Server.ID)) + if err != nil { + return true, fmt.Errorf("Failed to find server with id (\"%s\"): %v", fi.StringValue(e.Server.ID), err) + } + + var addresses map[string][]openstack.Address + err = mapstructure.Decode(server.Addresses, &addresses) + if err != nil { + return true, err + } + + for _, addrList := range addresses { + for _, props := range addrList { + if props.IPType == "floating" { + result = fi.String(props.Addr) + return true, nil + } + } + } + return false, nil + }) + if result == nil || err != nil { + return nil, fmt.Errorf("Could not find floating ip associated to server (\"%s\") %v", fi.StringValue(e.Server.Name), err) + } + return result, nil +} + func (e *FloatingIP) FindIPAddress(context *fi.Context) (*string, error) { if e.ID == nil { if e.Server != nil && e.Server.ID == nil { @@ -92,8 +123,13 @@ func (e *FloatingIP) FindIPAddress(context *fi.Context) (*string, error) { if len(fips) == 1 && fips[0].PortID == fi.StringValue(e.LB.PortID) { return &fips[0].FloatingIP, nil } - klog.V(2).Infof("Could not find port floatingips port=%s", fi.StringValue(e.LB.PortID)) - return nil, nil + return nil, fmt.Errorf("Could not find port floatingips port=%s", fi.StringValue(e.LB.PortID)) + } else if e.ID == nil && e.Server != nil { + floating, err := e.findServerFloatingIP(context, cloud) + if err != nil { + return nil, fmt.Errorf("Could not find server (\"%s\") floating ip: %v", fi.StringValue(e.Server.ID), err) + } + return floating, nil } fip, err := cloud.GetFloatingIP(fi.StringValue(e.ID)) @@ -164,14 +200,14 @@ func (e *FloatingIP) Find(c *fi.Context) (*FloatingIP, error) { return nil, nil } var actual *FloatingIP - for _, fip := range fips { + for i, fip := range fips { if fip.InstanceID == fi.StringValue(e.Server.ID) { if actual != nil { return nil, fmt.Errorf("Multiple floating ip's associated to server: %s", fi.StringValue(e.Server.ID)) } actual = &FloatingIP{ Name: e.Name, - ID: fi.String(fips[0].ID), + ID: fi.String(fips[i].ID), Server: e.Server, Lifecycle: e.Lifecycle, } diff --git a/upup/pkg/fi/cloudup/openstacktasks/subnet.go b/upup/pkg/fi/cloudup/openstacktasks/subnet.go index 10051fab44961..04f46db81d646 100644 --- a/upup/pkg/fi/cloudup/openstacktasks/subnet.go +++ b/upup/pkg/fi/cloudup/openstacktasks/subnet.go @@ -123,7 +123,7 @@ func (_ *Subnet) CheckChanges(a, e, changes *Subnet) error { if changes.DNSServers != nil { return fi.CannotChangeField("DNSServers") } - if changes.Network != nil { + if fi.StringValue(a.Network.ID) != fi.StringValue(e.Network.ID) { return fi.CannotChangeField("Network") } if changes.CIDR != nil {