fix: add built-in node affinity and toleration for addons (apecloud#2154

)
zjx20 · Mar 22, 2023 · 7af0a51 · 7af0a51
1 parent 3b03918
commit 7af0a51
Show file tree

Hide file tree

Showing 14 changed files with 642 additions and 7 deletions.
diff --git a/deploy/csi-hostpath-driver/values.yaml b/deploy/csi-hostpath-driver/values.yaml
@@ -51,9 +51,22 @@ resources: {}
 
 nodeSelector: {}
 
-tolerations: []
+tolerations:
+  - key: kb-controller
+    operator: Equal
+    value: "true"
+    effect: NoSchedule
 
-affinity: {}
+affinity:
+  nodeAffinity:
+    preferredDuringSchedulingIgnoredDuringExecution:
+      - weight: 100
+        preference:
+          matchExpressions:
+            - key: kb-controller
+              operator: In
+              values:
+                - "true"
 
 sidecars:
   healthmonitor:
@@ -87,4 +100,4 @@ sidecars:
 
 storageClass:
   create: true
-  default: true
+  default: true
diff --git a/deploy/csi-s3/Chart.yaml b/deploy/csi-s3/Chart.yaml
@@ -0,0 +1,11 @@
+apiVersion: v1
+appVersion: 0.31.3
+description: Container Storage Interface (CSI) driver for S3 volumes
+home: https://github.com/yandex-cloud/k8s-csi-s3
+icon: https://raw.githubusercontent.com/yandex-cloud/geesefs/master/doc/geesefs.png
+keywords:
+- s3
+name: csi-s3
+sources:
+- https://github.com/yandex-cloud/k8s-csi-s3/deploy/helm
+version: 0.31.3
diff --git a/deploy/csi-s3/README.md b/deploy/csi-s3/README.md
@@ -0,0 +1,37 @@
+# Helm chart for csi-s3
+
+This chart adds S3 volume support to your cluster.
+
+## Install chart
+
+- Helm 2.x: `helm install [--set secret.accessKey=... --set secret.secretKey=... ...] --namespace kube-system --name csi-s3 .`
+- Helm 3.x: `helm install [--set secret.accessKey=... --set secret.secretKey=... ...] --namespace kube-system csi-s3 .`
+
+After installation succeeds, you can get a status of Chart: `helm status csi-s3`.
+
+## Delete Chart
+
+- Helm 2.x: `helm delete --purge csi-s3`
+- Helm 3.x: `helm uninstall csi-s3 --namespace kube-system`
+
+## Configuration
+
+By default, this chart creates a secret and a storage class. You should at least set `secret.accessKey` and `secret.secretKey`
+to your [Yandex Object Storage](https://cloud.yandex.com/en-ru/services/storage) keys for it to work.
+
+The following table lists all configuration parameters and their default values.
+
+| Parameter                    | Description                                                            | Default                                                |
+| ---------------------------- | ---------------------------------------------------------------------- | ------------------------------------------------------ |
+| `storageClass.create`        | Specifies whether the storage class should be created                  | true                                                   |
+| `storageClass.name`          | Storage class name                                                     | csi-s3                                                 |
+| `storageClass.singleBucket`  | Use a single bucket for all dynamically provisioned persistent volumes |                                                        |
+| `storageClass.mounter`       | Mounter to use. Either geesefs, s3fs or rclone. geesefs recommended    | geesefs                                                |
+| `storageClass.mountOptions`  | GeeseFS mount options                                                  | `--memory-limit 1000 --dir-mode 0777 --file-mode 0666` |
+| `storageClass.reclaimPolicy` | Volume reclaim policy                                                  | Delete                                                 |
+| `storageClass.annotations`   | Annotations for the storage class                                      |                                                        |
+| `secret.create`              | Specifies whether the secret should be created                         | true                                                   |
+| `secret.name`                | Name of the secret                                                     | csi-s3-secret                                          |
+| `secret.accessKey`           | S3 Access Key                                                          |                                                        |
+| `secret.secretKey`           | S3 Secret Key                                                          |                                                        |
+| `secret.endpoint`            | Endpoint                                                               | https://storage.yandexcloud.net                        |
diff --git a/deploy/csi-s3/manifest.yaml b/deploy/csi-s3/manifest.yaml
@@ -0,0 +1,104 @@
+helm_chart:
+  name: cr.yandex/crp9ftr22d26age3hulg/yandex-cloud/csi-s3/csi-s3
+  tag: 0.31.3
+requirements:
+  k8s_version: ">=1.13"
+images:
+  - full: images.attacher
+  - full: images.registrar
+  - full: images.provisioner
+  - full: images.csi
+user_values:
+  - name: storageClass.create
+    title:
+      en: Create storage class
+      ru: Создать класс хранения
+    description:
+      en: Specifies whether the storage class should be created
+      ru: 'Если "да", при установке будет создан класс хранения S3'
+    boolean_value:
+      default_value: true
+  - name: secret.create
+    title:
+      en: Create secret
+      ru: Создать секрет
+    description:
+      en: Specifies whether the secret should be created
+      ru: 'Если "да", при установке будет создан секрет, иначе для класса хранения будет использован существующий'
+    boolean_value:
+      default_value: true
+  - name: secret.accessKey
+    title:
+      en: S3 Access Key ID
+      ru: Идентификатор ключа S3
+    description:
+      en: S3 Access Key ID
+      ru: Идентификатор ключа S3
+    string_value:
+      default_value: ""
+  - name: secret.secretKey
+    title:
+      en: S3 Secret Key
+      ru: Секретный ключ S3
+    description:
+      en: S3 Secret Key
+      ru: Секретный ключ S3
+    string_value:
+      default_value: ""
+  - name: storageClass.singleBucket
+    title:
+      en: Single S3 bucket for volumes
+      ru: Общий S3 бакет для томов
+    description:
+      en: Single S3 bucket to use for all dynamically provisioned persistent volumes
+      ru: Общий бакет S3, в котором будут создаваться все динамически распределяемые тома. Если пусто, под каждый том будет создаваться новый бакет
+    string_value:
+      default_value: ""
+  - name: secret.endpoint
+    title:
+      en: S3 endpoint
+      ru: Адрес S3-сервиса
+    description:
+      en: S3 service endpoint to use
+      ru: Адрес S3-сервиса, который будет использоваться
+    string_value:
+      default_value: "https://storage.yandexcloud.net"
+  - name: storageClass.mountOptions
+    title:
+      en: GeeseFS mount options
+      ru: Опции монтирования GeeseFS
+    description:
+      en: GeeseFS mount options to use. Consult GeeseFS (https://github.com/yandex-cloud/geesefs) help for the full option list
+      ru: Опции монтирования GeeseFS. Смотрите справку GeeseFS (https://github.com/yandex-cloud/geesefs) для полного перечня опций
+    string_value:
+      default_value: "--memory-limit 1000 --dir-mode 0777 --file-mode 0666"
+  - name: storageClass.reclaimPolicy
+    title:
+      en: Volume reclaim policy
+      ru: Политика очистки томов
+    description:
+      en: Volume reclaim policy for the storage class (Retain or Delete)
+      ru: Политика очистки PV, связанных с PVC (Retain - сохранять при удалении PVC, Delete - удалять при удалении PVC)
+    string_selector_value:
+      default_value: Delete
+      values:
+        - Delete
+        - Retain
+  - name: storageClass.name
+    title:
+      en: Storage class name
+      ru: Название класса хранения
+    description:
+      en: Name of the storage class that will be created
+      ru: Название класса хранения, который будет создан при установке
+    string_value:
+      default_value: csi-s3
+  - name: secret.name
+    title:
+      en: Name of the secret
+      ru: Название секрета
+    description:
+      en: Name of the secret to create or use for the storage class
+      ru: Название секрета, который будет создан или использован для класса хранения
+    string_value:
+      default_value: csi-s3-secret
diff --git a/deploy/csi-s3/templates/attacher.yaml b/deploy/csi-s3/templates/attacher.yaml
@@ -0,0 +1,104 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: csi-attacher-sa
+  namespace: {{ .Release.Namespace }}
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: external-attacher-runner
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "list"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["csinodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments/status"]
+    verbs: ["patch"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-attacher-role
+subjects:
+  - kind: ServiceAccount
+    name: csi-attacher-sa
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: external-attacher-runner
+  apiGroup: rbac.authorization.k8s.io
+---
+# needed for StatefulSet
+kind: Service
+apiVersion: v1
+metadata:
+  name: csi-attacher-s3
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: csi-attacher-s3
+spec:
+  selector:
+    app: csi-attacher-s3
+  ports:
+    - name: csi-s3-dummy
+      port: 65535
+---
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: csi-attacher-s3
+  namespace: {{ .Release.Namespace }}
+spec:
+  serviceName: "csi-attacher-s3"
+  replicas: 1
+  selector:
+    matchLabels:
+      app: csi-attacher-s3
+  template:
+    metadata:
+      labels:
+        app: csi-attacher-s3
+    spec:
+      serviceAccount: csi-attacher-sa
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      containers:
+        - name: csi-attacher
+          image: {{ .Values.images.attacher }}
+          args:
+            - "--v=4"
+            - "--csi-address=$(ADDRESS)"
+          env:
+            - name: ADDRESS
+              value: /var/lib/kubelet/plugins/ru.yandex.s3.csi/csi.sock
+          imagePullPolicy: "IfNotPresent"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/kubelet/plugins/ru.yandex.s3.csi
+      volumes:
+        - name: socket-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins/ru.yandex.s3.csi
+            type: DirectoryOrCreate