Support crd auto upgrader with helm hook (fluid-cloudnative#2062)

* Support crd auto upgrader with helm hook Signed-off-by: dongyun.xzh <[email protected]> * Add empty line Signed-off-by: dongyun.xzh <[email protected]> * Add upgrade crd scirpts into docker image Signed-off-by: dongyun.xzh <[email protected]> * Add empty line to end of script Signed-off-by: dongyun.xzh <[email protected]> * Rename template folder to "upgrade" Signed-off-by: dongyun.xzh <[email protected]> * Enable helm chart hook when upgrading only Signed-off-by: dongyun.xzh <[email protected]> Signed-off-by: dongyun.xzh <[email protected]>
shiwenwenya · Aug 16, 2022 · d2bc81a · d2bc81a
1 parent ffdd935
commit d2bc81a
Show file tree

Hide file tree

Showing 5 changed files with 101 additions and 0 deletions.
diff --git a/Makefile b/Makefile
@@ -14,6 +14,7 @@ CSI_IMG ?= ${IMG_REPO}/fluid-csi
 LOADER_IMG ?= ${IMG_REPO}/fluid-dataloader
 INIT_USERS_IMG ?= ${IMG_REPO}/init-users
 WEBHOOK_IMG ?= ${IMG_REPO}/fluid-webhook
+CRD_UPGRADER_IMG?= ${IMG_REPO}/fluid-crd-upgrader
 GO_MODULE ?= off
 GC_FLAGS ?= -gcflags="all=-N -l"
 ARCH ?= amd64
@@ -62,6 +63,7 @@ DOCKER_BUILD += docker-build-csi
 DOCKER_BUILD += docker-build-webhook
 DOCKER_BUILD += docker-build-juicefsruntime-controller
 DOCKER_BUILD += docker-build-init-users
+DOCKER_BUILD += docker-build-crd-upgrader
 
 # Push docker images
 DOCKER_PUSH := docker-push-dataset-controller
@@ -73,6 +75,7 @@ DOCKER_PUSH += docker-push-webhook
 DOCKER_PUSH += docker-push-goosefsruntime-controller
 DOCKER_PUSH += docker-push-juicefsruntime-controller
 DOCKER_PUSH += docker-push-init-users
+DOCKER_PUSH += docker-push-crd-upgrader
 
 # Buildx and push docker images
 DOCKER_BUILDX_PUSH := docker-buildx-push-dataset-controller
@@ -84,6 +87,7 @@ DOCKER_BUILDX_PUSH += docker-buildx-push-csi
 DOCKER_BUILDX_PUSH += docker-buildx-push-webhook
 DOCKER_BUILDX_PUSH += docker-buildx-push-juicefsruntime-controller
 DOCKER_BUILDX_PUSH += docker-buildx-push-init-users
+DOCKER_BUILDX_PUSH += docker-buildx-push-crd-upgrader
 
 override LDFLAGS += \
   -X ${PACKAGE}.version=${VERSION} \
@@ -206,6 +210,9 @@ docker-build-init-users:
 docker-build-webhook:
 	docker build --no-cache --build-arg TARGETARCH=${ARCH} . -f docker/Dockerfile.webhook -t ${WEBHOOK_IMG}:${GIT_VERSION}
 
+docker-build-crd-upgrader:
+	docker build --no-cache --build-arg TARGETARCH=${ARCH} . -f docker/Dockerfile.crds -t ${CRD_UPGRADER_IMG}:${GIT_VERSION}
+
 # Push the docker image
 docker-push-dataset-controller: docker-build-dataset-controller
 	docker push ${DATASET_CONTROLLER_IMG}:${GIT_VERSION}
@@ -237,6 +244,9 @@ docker-push-init-users: docker-build-init-users
 docker-push-webhook: docker-build-webhook
 	docker push ${WEBHOOK_IMG}:${GIT_VERSION}
 
+docker-push-crd-upgrader: docker-build-crd-upgrader
+	docker push ${CRD_UPGRADER_IMG}:${GIT_VERSION}
+
 # Buildx and push the docker image
 docker-buildx-push-dataset-controller: generate gen-openapi fmt vet
 	docker buildx build --push --platform linux/amd64,linux/arm64 --no-cache . -f docker/Dockerfile.dataset -t ${DATASET_CONTROLLER_IMG}:${GIT_VERSION}
@@ -265,6 +275,9 @@ docker-buildx-push-init-users:
 docker-buildx-push-webhook:
 	docker buildx build --push --platform linux/amd64,linux/arm64 --no-cache . -f docker/Dockerfile.webhook -t ${WEBHOOK_IMG}:${GIT_VERSION}
 
+docker-buildx-push-crd-upgrader:
+	docker buildx build --push --platform linux/amd64,linux/arm64 --no-cache . -f docker/Dockerfile.crds -t ${CRD_UPGRADER_IMG}:${GIT_VERSION}
+
 docker-build-all: ${DOCKER_BUILD}
 docker-push-all: ${DOCKER_PUSH}
 docker-buildx-all-push: ${DOCKER_BUILDX_PUSH}

diff --git a/charts/fluid/fluid/templates/upgrade/crd-upgrade.yaml b/charts/fluid/fluid/templates/upgrade/crd-upgrade.yaml
@@ -0,0 +1,59 @@
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  namespace: {{ .Release.Namespace }}
+  name: fluid-crds-upgrade-{{ .Chart.AppVersion }}
+  annotations:
+    "helm.sh/hook": pre-upgrade
+    "helm.sh/hook-weight": "-4"
+    "helm.sh/hook-delete-policy": before-hook-creation
+spec:
+  template:
+    spec:
+      serviceAccountName: fluid-crds-upgrade
+      containers:
+        - name: fluid-crds-upgrade
+          image: {{ .Values.crdUpgrade.image }}
+          command: ["bash", "/fluid/upgrade-crds.sh"]
+      restartPolicy: OnFailure
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: fluid-crds-upgrade
+  annotations:
+    "helm.sh/hook": pre-upgrade
+    "helm.sh/hook-weight": "-5"
+    "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation
+rules:
+  - apiGroups: ["apiextensions.k8s.io"]
+    resources: ["customresourcedefinitions"]
+    verbs: ["create", "get", "update"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: fluid-crds-upgrade
+  annotations:
+    "helm.sh/hook": pre-upgrade
+    "helm.sh/hook-weight": "-5"
+    "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: fluid-crds-upgrade
+subjects:
+  - kind: ServiceAccount
+    name: fluid-crds-upgrade
+    namespace: {{ .Release.Namespace }}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: fluid-crds-upgrade
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    "helm.sh/hook": pre-upgrade
+    "helm.sh/hook-weight": "-5"
+    "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation
diff --git a/charts/fluid/fluid/values.yaml b/charts/fluid/fluid/values.yaml
@@ -3,6 +3,8 @@
 # Declare variables to be passed into your templates.
 
 workdir: /tmp
+crdUpgrade:
+  image: fluidcloudnative/fluid-crd-upgrader:v0.8.0-5bb4677
 
 image:
   imagePullSecrets: []

diff --git a/docker/Dockerfile.crds b/docker/Dockerfile.crds
@@ -0,0 +1,13 @@
+FROM alpine:3.10
+
+COPY ./charts/fluid/fluid/crds /fluid/crds
+COPY ./tools/crd-upgrade/upgrade-crds.sh /fluid/upgrade-crds.sh
+
+RUN apk add --update curl tzdata iproute2 bash libc6-compat vim &&  \
+ 	rm -rf /var/cache/apk/* && \
+ 	cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \
+ 	echo "Asia/Shanghai" >  /etc/timezone
+
+ENV K8S_VERSION v1.14.8
+ARG TARGETARCH
+RUN curl -o /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/${TARGETARCH}/kubectl && chmod +x /usr/local/bin/kubectl
diff --git a/tools/crd-upgrade/upgrade-crds.sh b/tools/crd-upgrade/upgrade-crds.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+set -e
+
+for crdfile in $(find /fluid/crds/*.yaml);
+do
+  crdshort=${crdfile#*_}
+  if [[ $(kubectl get --ignore-not-found -f $crdfile | wc -l) -gt 0 ]]; then
+    echo "$crdshort founded, replacing its crd..."
+    kubectl replace -f $crdfile
+  else
+    echo "$crdshort not founded, applying its crd..."
+    kubectl apply -f $crdfile
+  fi
+done