Add suport for multiple disks per kafka broker

vivekjain123 · Dec 19, 2018 · 454a4fb · 454a4fb
1 parent c5b56a1
commit 454a4fb
Show file tree

Hide file tree

Showing 4 changed files with 35 additions and 9 deletions.
diff --git a/charts/cp-kafka/README.md b/charts/cp-kafka/README.md
@@ -140,6 +140,7 @@ The configuration parameters in this section control the resources requested and
 | `persistence.enabled` | Whether to create a PVC. If `false`, an `emptyDir` on the host will be used. | `true` |
 | `persistence.size` | Size for log dir, where Kafka will store log data. | `5Gi` |
 | `persistence.storageClass` | Valid options: `nil`, `"-"`, or storage class name. | `nil` |
+| `persistence.disksPerBroker` | The amount of disks that will be attached per instance of Kafka broker. | 1 |
 
 ### Kafka JVM Heap Options
 

diff --git a/charts/cp-kafka/templates/_helpers.tpl b/charts/cp-kafka/templates/_helpers.tpl
@@ -52,4 +52,14 @@ else use user-provided URL
 {{- $zookeeperConnectOverride := (index .Values "configurationOverrides" "zookeeper.connect") }}
 {{- default $zookeeperConnect $zookeeperConnectOverride }}
 {{- end -}}
-{{- end -}}
+{{- end -}}
+
+{{/*
+Create a variable containing a comma-sepparated list of all the log dirs.
+*/}}
+{{- define "cp-kafka.log.dirs" -}}
+{{- range $k, $e := until (.Values.persistence.disksPerBroker|int) -}}
+{{- if $k}}{{- printf ","}}{{end}}
+{{- printf "/opt/kafka/data/logs-%d" $k -}}
+{{- end -}}
+{{- end -}}
diff --git a/charts/cp-kafka/templates/statefulset.yaml b/charts/cp-kafka/templates/statefulset.yaml
@@ -101,6 +101,11 @@ spec:
           value: {{ .Values.heapOptions }}
         - name: KAFKA_ZOOKEEPER_CONNECT
           value: {{ include "cp-kafka.cp-zookeeper.service-name" . | quote }}
+        {{- end }}
+        {{- if not (hasKey .Values.configurationOverrides "log.dirs") }}
+        - name: KAFKA_LOG_DIRS
+          value: {{ include "cp-kafka.log.dirs" . | quote }}
+        {{- end }}
         {{- range $key, $value := .Values.configurationOverrides }}
         - name: {{ printf "KAFKA_%s" $key | replace "." "_" | upper | quote }}
           value: {{ $value | quote }}
@@ -125,8 +130,12 @@ spec:
           export KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://${POD_IP}:9092{{ if kindIs "string" $advertisedListenersOverride }}{{ printf ",%s" $advertisedListenersOverride }}{{ end }} && \
           exec /etc/confluent/docker/run
         volumeMounts:
-        - name: datadir
-          mountPath: /opt/kafka/data
+        {{- $disksPerBroker := .Values.persistence.disksPerBroker | int }}
+        {{- $root := . }}
+        {{- range $k, $e := until $disksPerBroker }}
+        - name: datadir-{{$k}}
+          mountPath: /opt/kafka/data-{{$k}}
+        {{- end }}
       {{- if .Values.imagePullSecrets }}
       imagePullSecrets:
 {{ toYaml .Values.imagePullSecrets | indent 8 }}
@@ -151,18 +160,22 @@ spec:
       {{- end }}
   {{- if .Values.persistence.enabled }}
   volumeClaimTemplates:
+  {{- $disksPerBroker := .Values.persistence.disksPerBroker | int }}
+  {{- $root := . }}
+  {{- range $k, $e := until $disksPerBroker }}
   - metadata:
-      name: datadir
+      name: datadir-{{$k}}
     spec:
       accessModes: [ "ReadWriteOnce" ]
       resources:
         requests:
-          storage: "{{ .Values.persistence.size }}"
-      {{- if .Values.persistence.storageClass }}
-      {{- if (eq "-" .Values.persistence.storageClass) }}
+          storage: "{{ $root.Values.persistence.size }}"
+      {{- if $root.Values.persistence.storageClass }}
+      {{- if (eq "-" $root.Values.persistence.storageClass) }}
       storageClassName: ""
       {{- else }}
-      storageClassName: "{{ .Values.persistence.storageClass }}"
+      storageClassName: "{{ $root.Values.persistence.storageClass }}"
       {{- end }}
       {{- end }}
-  {{- end }}
+{{- end }}
+{{- end }}
diff --git a/charts/cp-kafka/values.yaml b/charts/cp-kafka/values.yaml
@@ -71,6 +71,8 @@ persistence:
   ##
   # storageClass: ""
 
+  disksPerBroker: 1
+
 ## Kafka JVM Heap Option
 heapOptions: "-Xms1G -Xmx1G"