Support deploying multiple injector replicas with auto-TLS (hashicorp…

…#436)

templates/injector-certs-secret.yaml

@@ -0,0 +1,10 @@
+{{- if and (eq (.Values.injector.enabled | toString) "true" ) (eq (.Values.global.enabled | toString) "true") (eq (.Values.injector.leaderElector.enabled | toString) "true") (gt (.Values.injector.replicas | int) 1) }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: vault-injector-certs
+  labels:
+    app.kubernetes.io/name: {{ include "vault.name" . }}-agent-injector
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}

templates/injector-deployment.yaml

@@ -11,7 +11,7 @@ metadata:
     app.kubernetes.io/managed-by: {{ .Release.Service }}
     component: webhook
 spec:
-  replicas: 1
+  replicas: {{ .Values.injector.replicas }}
   selector:
     matchLabels:
       app.kubernetes.io/name: {{ template "vault.name" . }}-agent-injector
@@ -88,6 +88,14 @@ spec:
             - name: AGENT_INJECT_TELEMETRY_PATH
               value: "/metrics"
             {{- end }}
+            {{- if and (eq (.Values.injector.leaderElector.enabled | toString) "true") (gt (.Values.injector.replicas | int) 1) }}
+            - name: AGENT_INJECT_USE_LEADER_ELECTOR
+              value: "true"
+            - name: NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            {{- end }}
             {{- include "vault.extraEnvironmentVars" .Values.injector | nindent 12 }}
           args:
             - agent-inject
@@ -112,6 +120,35 @@ spec:
             periodSeconds: 2
             successThreshold: 1
             timeoutSeconds: 5
+        {{- if and (eq (.Values.injector.leaderElector.enabled | toString) "true") (gt (.Values.injector.replicas | int) 1) }}
+        - name: leader-elector
+          image: {{ .Values.injector.leaderElector.image.repository }}:{{ .Values.injector.leaderElector.image.tag }}
+          args:
+            - --election={{ template "vault.fullname" . }}-agent-injector-leader
+            - --election-namespace={{ .Release.Namespace }}
+            - --http=0.0.0.0:4040
+            - --ttl={{ .Values.injector.leaderElector.ttl }}
+          livenessProbe:
+            httpGet:
+              path: /
+              port: 4040
+              scheme: HTTP
+            failureThreshold: 2
+            initialDelaySeconds: 1
+            periodSeconds: 2
+            successThreshold: 1
+            timeoutSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /
+              port: 4040
+              scheme: HTTP
+            failureThreshold: 2
+            initialDelaySeconds: 2
+            periodSeconds: 2
+            successThreshold: 1
+            timeoutSeconds: 5
+        {{- end }}
 {{- if .Values.injector.certs.secretName }}
           volumeMounts:
             - name: webhook-certs

templates/injector-leader-endpoint.yaml

@@ -0,0 +1,12 @@
+{{- if and (eq (.Values.injector.enabled | toString) "true" ) (eq (.Values.global.enabled | toString) "true") (eq (.Values.injector.leaderElector.enabled | toString) "true") (gt (.Values.injector.replicas | int) 1) }}
+# This is created here so it can be cleaned up easily, since if
+# the endpoint is left around the leader won't expire for about a minute.
+apiVersion: v1
+kind: Endpoints
+metadata:
+  name: {{ template "vault.fullname" . }}-agent-injector-leader
+  labels:
+    app.kubernetes.io/name: {{ include "vault.name" . }}-agent-injector
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}

templates/injector-role.yaml

@@ -0,0 +1,19 @@
+{{- if and (eq (.Values.injector.enabled | toString) "true" ) (eq (.Values.global.enabled | toString) "true") (eq (.Values.injector.leaderElector.enabled | toString) "true") (gt (.Values.injector.replicas | int) 1) }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ template "vault.fullname" . }}-agent-injector-leader-elector-role
+  labels:
+    app.kubernetes.io/name: {{ include "vault.name" . }}-agent-injector
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+rules:
+  - apiGroups: [""]
+    resources: ["endpoints", "secrets"]
+    verbs:
+      - "create"
+      - "get"
+      - "watch"
+      - "list"
+      - "update"
+{{- end }}

templates/injector-rolebinding.yaml

@@ -0,0 +1,18 @@
+{{- if and (eq (.Values.injector.enabled | toString) "true" ) (eq (.Values.global.enabled | toString) "true") (eq (.Values.injector.leaderElector.enabled | toString) "true") (gt (.Values.injector.replicas | int) 1) }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ template "vault.fullname" . }}-agent-injector-leader-elector-binding
+  labels:
+    app.kubernetes.io/name: {{ include "vault.name" . }}-agent-injector
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ template "vault.fullname" . }}-agent-injector-leader-elector-role
+subjects:
+  - kind: ServiceAccount
+    name: {{ template "vault.fullname" . }}-agent-injector
+    namespace: {{ .Release.Namespace }}
+{{- end }}

test/acceptance/injector-leader-elector.bats

@@ -0,0 +1,46 @@
+#!/usr/bin/env bats
+
+load _helpers
+
+@test "injector: testing leader elector" {
+  cd `chart_dir`
+
+  kubectl delete namespace acceptance --ignore-not-found=true
+  kubectl create namespace acceptance
+  kubectl config set-context --current --namespace=acceptance
+
+  helm install "$(name_prefix)" \
+    --set="injector.replicas=3" .
+  kubectl wait --for condition=Ready pod -l app.kubernetes.io/name=vault-agent-injector
+
+  pods=($(kubectl get pods -l app.kubernetes.io/name=vault-agent-injector -o json | jq -r '.items[] | .metadata.name'))
+  [ "${#pods[@]}" == 3 ]
+
+  leader="$(echo "$(kubectl exec ${pods[0]} -c sidecar-injector -- wget --quiet --output-document - localhost:4040)" | jq -r .name)"
+  # Check the leader name is valid - i.e. one of the 3 pods
+  [[ " ${pods[@]} " =~ " ${leader} " ]]
+
+  # Check every pod agrees on who the leader is
+  for pod in "${pods[@]}"
+  do
+    pod_leader="$(echo "$(kubectl exec $pod -c sidecar-injector -- wget --quiet --output-document - localhost:4040)" | jq -r .name)"
+    [ "${pod_leader}" == "${leader}" ]
+  done
+}
+
+setup() {
+  kubectl delete namespace acceptance --ignore-not-found=true
+  kubectl create namespace acceptance
+  kubectl config set-context --current --namespace=acceptance
+}
+
+# Clean up
+teardown() {
+  if [[ ${CLEANUP:-true} == "true" ]]
+  then
+      echo "helm/pvc teardown"
+      helm delete vault
+      kubectl delete --all pvc
+      kubectl delete namespace acceptance
+  fi
+}

test/unit/injector-deployment.bats

@@ -425,13 +425,13 @@ load _helpers
 #--------------------------------------------------------------------
 # affinity
 
-@test "injector/deployment: affinity not set by default" {
+@test "injector/deployment: affinity set by default" {
   cd `chart_dir`
   local actual=$(helm template \
       --show-only templates/injector-deployment.yaml  \
       . | tee /dev/stderr |
       yq '.spec.template.spec | .affinity? == null' | tee /dev/stderr)
-  [ "${actual}" = "true" ]
+  [ "${actual}" = "false" ]
 }
 
 @test "injector/deployment: affinity can be set" {