Support multiple k8s versions (sapcc#393)

* Adjust api server arguments according to kube version

* Make etcd backup optional (enabled by default)

* Maintain hyperkube image version for multiple kubernetes versions

* Add migration to make sure klusters < 1.10 have a missing role binding

* Remove chart version from configmap to avoid unnecesarry redeployments

The configmap is checksummed in the control plane nodes and tthis would force a redeployment everytime the chart version changes

* Add image registry

This adds a yaml file listing all supported kubernetes versions and their images.

charts/images.yaml

@@ -0,0 +1,19 @@
+imagesForVersion:
+  '1.10.11':
+    default: true
+    hyperkube:
+      repository: 'sapcc/hyperkube'
+      tag: 'v1.10.11'
+  '1.9.11':
+    hyperkube:
+      repository: 'quay.io/coreos/hyperkube'
+      tag: 'v1.9.11_coreos.1'
+  '1.8.14':
+    hyperkube: 
+      repository: 'quay.io/coreos/hyperkube'
+      tag: 'v1.8.14_coreos.1'
+  '1.7.16':
+    hyperkube:
+      repository: 'quay.io/coreos/hyperkube'
+      tag: 'v1.7.16_coreos.0'
+

charts/kube-master/Chart.yaml

@@ -1,4 +1,4 @@
 apiVersion: v1
 description: A Helm chart for Kubernetes
 name: kube-master
-version: 1.9.0-kubernikus.3
+version: 2.0.0

charts/kube-master/charts/etcd/templates/deployment.yaml

@@ -27,17 +27,27 @@ data:
       "Failed")
             continue;;
       "Successful")
-            exec etcd \
-              --name=kubernikus \
-              --data-dir=/var/lib/etcd/new.etcd \
-              --advertise-client-urls=http://${ETCD_IP}:2379 \
-              --initial-advertise-peer-urls=http://${ETCD_IP}:2380 \
-              --initial-cluster=kubernikus=http://${ETCD_IP}:2380 \
-              --listen-client-urls=http://0.0.0.0:2379 \
-              --listen-peer-urls=http://${ETCD_IP}:2380
+            . /bootstrap/etcd.sh
             ;;
       esac;
     done
+  etcd.sh: |-
+    #!/bin/sh
+    if [ ! -d /var/lib/etcd/new.etcd ]; then
+        mkdir /var/lib/etcd/new.etcd
+    fi
+    if [ -d /var/lib/etcd/member ]; then
+        mv /var/lib/etcd/member /var/lib/etcd/new.etcd/member
+    fi
+
+    exec etcd \
+      --name=kubernikus \
+      --data-dir=/var/lib/etcd/new.etcd \
+      --advertise-client-urls=http://${ETCD_IP}:2379 \
+      --initial-advertise-peer-urls=http://${ETCD_IP}:2380 \
+      --initial-cluster=kubernikus=http://${ETCD_IP}:2380 \
+      --listen-client-urls=http://0.0.0.0:2379 \
+      --listen-peer-urls=http://${ETCD_IP}:2380
 ---
 apiVersion: extensions/v1beta1
 kind: Deployment
@@ -78,7 +88,7 @@ spec:
                 fieldRef:
                   fieldPath: status.podIP
           command:
-            - /bootstrap/bootstrap.sh
+            - /bootstrap/{{ if .Values.backup.enabled }}bootstrap.sh{{ else }}etcd.sh{{ end }}
           volumeMounts:
             - mountPath: /var/lib/etcd
               name: data
@@ -92,14 +102,17 @@ spec:
                 - ETCDCTL_API=3 etcdctl get foo
             initialDelaySeconds: 15
             periodSeconds: 5
+          {{- if .Values.backup.enabled }}
           readinessProbe:
             httpGet:
               path: /healthz
               port: 8080
             initialDelaySeconds: 15
             periodSeconds: 10
+          {{- end }}
           resources:
 {{ toYaml .Values.resources | indent 12 }}
+{{- if .Values.backup.enabled }}
         - name: backup
           command:
             - etcdbrctl
@@ -148,3 +161,4 @@ spec:
               name: data
           resources:
 {{ toYaml .Values.backup.resources | indent 12 }}
+{{- end }}

charts/kube-master/charts/etcd/templates/secrets.yaml

@@ -1,4 +1,5 @@
 {{/* vim: set filetype=gotexttmpl: */ -}}
+{{ if .Values.backup.enabled -}}
 apiVersion: v1
 kind: Secret
 metadata:
@@ -13,3 +14,4 @@ data:
   openstack-password: {{ required "missing openstack-password" .Values.openstack.password | b64enc }}
   openstack-domain-name: {{ required "missing openstack-domain-name" .Values.openstack.domainName | b64enc }}
   openstack-project-id: {{ required "missing openstack-project-id" .Values.openstack.projectID | b64enc }}
+{{ end -}}

charts/kube-master/charts/etcd/values.yaml

@@ -3,7 +3,7 @@
 # Declare variables to be passed into your templates.
 image:
   repository: sapcc/etcd
-  tag: v3.3.10
+  tag: v3.3.12
   pullPolicy: IfNotPresent
 ## Persist data to a persitent volume
 persistence:
@@ -20,9 +20,10 @@ resources:
     cpu: 750m
     memory: 2560Mi
 backup:
+  enabled: true
   image:
     repository: sapcc/etcdbrctl
-    tag: 0.5.0
+    tag: 0.5.2
     pullPolicy: IfNotPresent
   config:
     # do a full-backup every hour

charts/kube-master/templates/_helpers.tpl

@@ -18,3 +18,12 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
 {{- $name := default "etcd" .Values.etcd.nameOverride -}}
 {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
+
+{{- define "hyperkube.image" }}
+{{- $images := required "imagesForVersion undefined" .Values.imagesForVersion}}
+{{- $version := required "version.kubernetes undefined" .Values.version.kubernetes }}
+{{- $imagesForVersion := required (printf "unsupported kubernetes version %s" $version) (index $images $version) }}
+{{- $hyperkube := required (printf "No hyperkube image found for version %s" $version) (index $imagesForVersion "hyperkube") }}
+{{- required (printf "repository for hyperkube missing for version %s" $version) $hyperkube.repository }}:
+  {{- required (printf "tag for hyperkube missing for version %s" $version) $hyperkube.tag }}
+{{- end -}}

charts/kube-master/templates/api.yaml

@@ -100,15 +100,15 @@ spec:
                 path: kubeconfig
       initContainers:
         - name: etcd-wait
-          image: sapcc/etcd:3.1.12
+          image: "{{ required "etcd.image.repository undefined" .Values.etcd.image.repository }}:{{ required "etcd.image.tag undefined" .Values.etcd.image.tag }}"
           command:
             - sh
             - -c
           args:
             - until etcdctl --total-timeout=4s --endpoints http://{{ include "etcd.fullname" . }}:2379 cluster-health; do sleep 5; done;
       containers:
         - name: apiserver
-          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          image: {{ include "hyperkube.image" . | quote }}
           args:
             - /hyperkube
             - apiserver
@@ -119,12 +119,18 @@ spec:
             - --authorization-mode=Node,RBAC
             - --cloud-config=/etc/kubernetes/cloudprovider/openstack.config
             - --cloud-provider=openstack
-            - --enable-admission-plugins=ExtendedResourceToleration
+{{- if (semverCompare ">= 1.8" .Values.version.kubernetes) }}
             - --enable-bootstrap-token-auth=true
             - --external-hostname={{ required "missing .api.apiserverHost" .Values.api.apiserverHost }}
+{{- else }}{{/* 1.7 */}}
+            - --experimental-bootstrap-token-auth=true
+            - --runtime-config=rbac.authorization.k8s.io/v1alpha1,extensions/v1beta1=true,extensions/v1beta1/thirdpartyresources=true
+{{- end }}
             - --token-auth-file=/etc/kubernetes/bootstrap/token.csv
             - --service-cluster-ip-range={{ .Values.serviceCIDR }}
             - --kubelet-preferred-address-types=InternalIP
+{{- if (semverCompare ">= 1.10" .Values.version.kubernetes) }}
+            - --enable-admission-plugins=ExtendedResourceToleration
             # Aggregation Layer
             - --requestheader-client-ca-file=/etc/kubernetes/certs/aggregation-ca.pem
             - --requestheader-allowed-names=aggregator
@@ -134,6 +140,10 @@ spec:
             - --proxy-client-cert-file=/etc/kubernetes/certs/aggregation-aggregator.pem
             - --proxy-client-key-file=/etc/kubernetes/certs/aggregation-aggregator-key.pem
             - --enable-aggregator-routing=true
+{{- else }}{{/* < 1.10 */}}
+            - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota
+            - --tls-ca-file=/etc/kubernetes/certs/tls-ca.pem
+{{- end }}
             #Cert Spratz
             - --client-ca-file=/etc/kubernetes/certs/apiserver-clients-and-nodes-ca.pem
             - --etcd-cafile=/etc/kubernetes/certs/etcd-clients-ca.pem

charts/kube-master/templates/configmap.yaml

@@ -4,7 +4,6 @@ kind: ConfigMap
 metadata:
   name: {{ include "master.fullname" . }}
   labels:
-    chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}"
     release: {{ .Release.Name }}
 data:
   kubeconfig: |-

charts/kube-master/templates/controller-manager.yaml

@@ -61,7 +61,7 @@ spec:
             name: {{ include "master.fullname" . }}-os-hacks
       initContainers:
         - name: apiserver-wait
-          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          image: {{ include "hyperkube.image" . }}
           command:
             - sh
             - -c
@@ -76,7 +76,7 @@ spec:
               readOnly: true
       containers:
         - name: controller-manager
-          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          image: {{ include "hyperkube.image" . | quote }}
           args:
             - /hyperkube
             - controller-manager

charts/kube-master/templates/scheduler.yaml

@@ -41,7 +41,7 @@ spec:
             name: {{ include "master.fullname" . }}
       containers:
         - name: scheduler
-          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          image: {{ include "hyperkube.image" . | quote }}
           args:
             - /hyperkube
             - scheduler

charts/kube-master/test-values.yaml

@@ -27,4 +27,6 @@ etcd:
     projectID: xyz
 imagesForVersion:
   "1.0.0":
-    hyperkube: xyz
+    hyperkube:
+      repository: abc
+      tag: xyz

charts/kube-master/values.yaml

@@ -2,8 +2,7 @@
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
 image:
-  repository: sapcc/hyperkube
-  tag: v1.10.11
+  #repository and tag from from hyperkube.image helper
   pullPolicy: IfNotPresent
 
 # Settings for the openstack cloudprovider
@@ -36,7 +35,7 @@ advertiseAddress: 198.18.128.1
 
 version: {}
 # kubernikus:
-# kubernetes: 1.10.7
+# kubernetes: 1.10.11
 
 api:
   replicaCount: 1

pkg/api/handlers/get_cluster_values.go

@@ -44,7 +44,7 @@ func (d *getClusterValues) Handle(params operations.GetClusterValuesParams, prin
 		return NewErrorResponse(&operations.GetClusterCredentialsDefault{}, 500, "Couldn't determine access mode for pvc: %s", err)
 	}
 
-	yamlData, err := helm.KlusterToHelmValues(kluster, secret, accessMode)
+	yamlData, err := helm.KlusterToHelmValues(kluster, secret, nil, accessMode)
 	if err != nil {
 		return NewErrorResponse(&operations.GetClusterCredentialsDefault{}, 500, "Failed to generate helm values: %s", err)
 	}

pkg/apis/kubernikus/factory.go

@@ -6,10 +6,9 @@ import (
 
 	"github.com/sapcc/kubernikus/pkg/api/models"
 	"github.com/sapcc/kubernikus/pkg/api/spec"
-	"github.com/sapcc/kubernikus/pkg/apis/kubernikus/v1"
+	v1 "github.com/sapcc/kubernikus/pkg/apis/kubernikus/v1"
 	"github.com/sapcc/kubernikus/pkg/controller/ground/bootstrap/dns"
 	"github.com/sapcc/kubernikus/pkg/migration"
-	"github.com/sapcc/kubernikus/pkg/util"
 	"github.com/sapcc/kubernikus/pkg/util/ip"
 	"github.com/sapcc/kubernikus/pkg/version"
 )
@@ -76,18 +75,6 @@ func (klusterFactory) KlusterFor(spec models.KlusterSpec) (*v1.Kluster, error) {
 		k.Spec.DNSDomain = dns.DEFAULT_DOMAIN
 	}
 
-	if k.Spec.Version == "" {
-		k.Spec.Version = util.DEFAULT_KUBERNETES_VERSION
-	}
-
-	if k.Spec.Version == "" {
-		k.Spec.Version = util.DEFAULT_KUBERNETES_VERSION
-	}
-
-	if k.Spec.Version != util.DEFAULT_KUBERNETES_VERSION {
-		return nil, fmt.Errorf("Unable to create cluster. Unsupported Kubernetes version.")
-	}
-
 	if k.ObjectMeta.Name == "" {
 		k.ObjectMeta.Name = spec.Name
 	}

pkg/cmd/kubernikus/helm.go

@@ -15,6 +15,7 @@ import (
 	"github.com/sapcc/kubernikus/pkg/cmd"
 	"github.com/sapcc/kubernikus/pkg/util"
 	"github.com/sapcc/kubernikus/pkg/util/helm"
+	"github.com/sapcc/kubernikus/pkg/version"
 )
 
 func NewHelmCommand() *cobra.Command {
@@ -44,6 +45,7 @@ type HelmOptions struct {
 	AuthProject       string
 	AuthProjectDomain string
 	ProjectID         string
+	ImagesFile        string
 }
 
 func NewHelmOptions() *HelmOptions {
@@ -52,6 +54,7 @@ func NewHelmOptions() *HelmOptions {
 		AuthDomain:        "ccadmin",
 		AuthProject:       "cloud_admin",
 		AuthProjectDomain: "ccadmin",
+		ImagesFile:        "charts/images.yaml",
 	}
 }
 func (o *HelmOptions) BindFlags(flags *pflag.FlagSet) {
@@ -62,6 +65,7 @@ func (o *HelmOptions) BindFlags(flags *pflag.FlagSet) {
 	flags.StringVar(&o.AuthProject, "auth-project", o.AuthProject, "Scope service user to this project")
 	flags.StringVar(&o.AuthProjectDomain, "auth-project-domain", o.AuthProjectDomain, "Domain of the project")
 	flags.StringVar(&o.ProjectID, "project-id", o.ProjectID, "Project ID where the kublets will be running")
+	flags.StringVar(&o.ImagesFile, "images-file", o.ImagesFile, "Yaml file for populating the image registry")
 }
 
 func (o *HelmOptions) Validate(c *cobra.Command, args []string) error {
@@ -93,6 +97,11 @@ func (o *HelmOptions) Complete(args []string) error {
 
 func (o *HelmOptions) Run(c *cobra.Command) error {
 	nameA := strings.SplitN(o.Name, ".", 2)
+	registry, err := version.NewImageRegistry(o.ImagesFile)
+	if err != nil {
+		return fmt.Errorf("Failed to load images from file: %s", err)
+	}
+
 	kluster, err := kubernikus.NewKlusterFactory().KlusterFor(models.KlusterSpec{
 		Name: nameA[0],
 		Openstack: models.OpenstackSpec{
@@ -116,7 +125,7 @@ func (o *HelmOptions) Run(c *cobra.Command) error {
 	secret.Openstack.ProjectID = o.ProjectID
 	secret.BootstrapToken = util.GenerateBootstrapToken()
 
-	result, err := helm.KlusterToHelmValues(kluster, &secret, "")
+	result, err := helm.KlusterToHelmValues(kluster, &secret, registry, "")
 	if err != nil {
 		return err
 	}

pkg/controller/config/config.go

@@ -13,6 +13,7 @@ import (
 	"github.com/sapcc/kubernikus/pkg/controller/nodeobservatory"
 	kubernikus_clientset "github.com/sapcc/kubernikus/pkg/generated/clientset"
 	kubernikus_informers "github.com/sapcc/kubernikus/pkg/generated/informers/externalversions"
+	"github.com/sapcc/kubernikus/pkg/version"
 )
 
 type Controller interface {
@@ -44,6 +45,7 @@ type Config struct {
 	Openstack  OpenstackConfig
 	Kubernikus KubernikusConfig
 	Helm       HelmConfig
+	Images     version.ImageRegistry
 }
 
 type Clients struct {

pkg/controller/ground.go

@@ -471,7 +471,7 @@ func (op *GroundControl) createKluster(kluster *v1.Kluster) error {
 		return fmt.Errorf("Failed to create container for etcd backups. Check if the project has quota for object-store usage: %s", err)
 	}
 
-	rawValues, err := helm_util.KlusterToHelmValues(kluster, klusterSecret, accessMode)
+	rawValues, err := helm_util.KlusterToHelmValues(kluster, klusterSecret, &op.Config.Images, accessMode)
 	if err != nil {
 		return err
 	}
@@ -566,7 +566,7 @@ func (op *GroundControl) requiresOpenstackInfo(kluster *v1.Kluster) bool {
 }
 
 func (op *GroundControl) requiresKubernikusInfo(kluster *v1.Kluster) bool {
-	return kluster.Status.Apiserver == "" || kluster.Status.Wormhole == ""
+	return kluster.Status.Apiserver == "" || kluster.Status.Wormhole == "" || kluster.Spec.Version == ""
 }
 
 func (op *GroundControl) discoverKubernikusInfo(kluster *v1.Kluster) error {
@@ -576,6 +576,13 @@ func (op *GroundControl) discoverKubernikusInfo(kluster *v1.Kluster) error {
 		"project", kluster.Account(),
 		"v", 5)
 
+	if kluster.Spec.Version == "" {
+		kluster.Spec.Version = op.Config.Images.DefaultVersion
+	}
+	if _, found := op.Config.Images.Versions[kluster.Spec.Version]; !found {
+		return fmt.Errorf("Unsupported Kubernetes version specified: %s", kluster.Spec.Version)
+	}
+
 	if kluster.Status.Apiserver == "" {
 		kluster.Status.Apiserver = fmt.Sprintf("https://%s.%s", kluster.GetName(), op.Config.Kubernikus.Domain)
 		op.Logger.Log(