Add StatefulSet option for BookKeeper on Google Kubernetes Engine (ap…

…ache#1047)

* add statefulset bookie config

* re-add daemonset config

* fix indentation issue

* return to original pulsar env vars

* fix minor indentation issues

* fix metadata label issue

* revert to original Deployment definition

* add load balancer service

* use only one bookie.yaml config

* add proxy and cluster metadata configs

* specify cluster metadata initialization as a Job rather than Pod

* create separate storage classes for ledger and journal

* fix YAML indentation issue

* update docs for pulsar-admin pod

* more documentation updates

* update cluster metadata initialization instructions

* add reference to pulsar proxy in docs

* one more small doc update

* avoid name clash in StorageClass definitions

* add selector to statefulset definition in bookie.yaml

* add pod antiaffinity to bookie config

* remove headless Service definition for bookies

* update statefulset to eliminate advertisedAddress

* Added ASF header

* Added ASF header

* move stray YAML files into /deployment

* change containers/initContainers order in config for readability

* add section on client connections

deployment/kubernetes/google-kubernetes-engine/bookie.yaml

@@ -17,120 +17,143 @@
 # under the License.
 #
 
+## Credits to Franck Cuny : https://github.com/fcuny/distributedlog-on-k8s/blob/master/bookkeeper.statefulset.yaml
 
-apiVersion: v1
-kind: ConfigMap
+# SSDs for bookie journal storage
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
 metadata:
-    name: bookie-config
-data:
-    PULSAR_MEM: "\" -Xms4g -Xmx4g -XX:MaxDirectMemorySize=4g\""
-    PULSAR_GC: "\" -XX:+UseG1GC \""
-    dbStorage_writeCacheMaxSizeMb: "1024"
-    dbStorage_readAheadCacheMaxSizeMb: "1024"
-    zkServers: zk-0.zookeeper,zk-1.zookeeper,zk-2.zookeeper
-    statsProviderClass: org.apache.bookkeeper.stats.PrometheusMetricsProvider
+  name: bookie-ssd
+provisioner: kubernetes.io/gce-pd
+parameters:
+  type: pd-ssd
+  zones: us-central1-a
 ---
-
-## BookKeeper servers need to access the local disks and the pods
-## cannot be moved across different nodes.
-## For this reason, we run BK as a daemon set, one for each node in the
-## cluster, unless restricted by label selectors
-apiVersion: extensions/v1beta1
-kind: DaemonSet
+# HDDs for bookie ledger storage
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
 metadata:
-    name: bookie
-    labels:
-        app: pulsar
-        component: bookkeeper
-spec:
-    template:
-        metadata:
-            labels:
-                app: pulsar
-                component: bookkeeper
-                # Specify cluster to allow aggregation by cluster in
-                # the metrics
-                cluster: pulsar-gke
-            annotations:
-                prometheus.io/scrape: "true"
-                prometheus.io/port: "8000"
-
-        spec:
-            containers:
-              - name: bookie
-                image: apachepulsar/pulsar:latest
-                command: ["sh", "-c"]
-                args:
-                  - >
-                    bin/apply-config-from-env.py conf/bookkeeper.conf &&
-                    bin/apply-config-from-env.py conf/pulsar_env.sh &&
-                    bin/pulsar bookie
-                ports:
-                  - containerPort: 3181
-                    hostPort: 3181
-                    name: client
-                envFrom:
-                  - configMapRef:
-                        name: bookie-config
-                env:
-                  - name: advertisedAddress
-                    valueFrom:
-                        fieldRef:
-                            fieldPath: status.hostIP
-
-                volumeMounts:
-                  - name: journal-disk
-                    mountPath: /pulsar/data/bookkeeper/journal
-                  - name: ledgers-disk
-                    mountPath: /pulsar/data/bookkeeper/ledgers
-
-                # bin/bookkeeper shell bookiesanity
-
-            initContainers:
-                # The first time, initialize BK metadata in zookeeper
-                 # Otherwise ignore error if it's already there
-              - name: bookie-metaformat
-                image: apachepulsar/pulsar:latest
-                command: ["sh", "-c"]
-                args:
-                  - >
-                    bin/apply-config-from-env.py conf/bookkeeper.conf &&
-                    bin/bookkeeper shell metaformat --nonInteractive || true;
-                envFrom:
-                  - configMapRef:
-                        name: bookie-config
-
-            volumes:
-                # Mount local disks
-              - name: journal-disk
-                hostPath:
-                    path: /mnt/disks/ssd0
-              - name: ledgers-disk
-                hostPath:
-                    path: /mnt/disks/ssd1
-
+  name: bookie-hdd
+provisioner: kubernetes.io/gce-pd
+parameters:
+  type: pd-standard
+  zones: us-central1-a
 ---
-
-##
-## Define the Bookie headless service
-## In practice, in this case, it is only useful to have a view of
-## all the bookie pods that are present
-##
 apiVersion: v1
-kind: Service
+kind: ConfigMap
 metadata:
-    name: bookkeeper
-    labels:
-        app: pulsar
-        component: bookkeeper
+  name: bookie-config
+data:
+  PULSAR_MEM: "\" -Xms4g -Xmx4g -XX:MaxDirectMemorySize=4g\""
+  PULSAR_GC: "\" -XX:+UseG1GC \""
+  dbStorage_writeCacheMaxSizeMb: "1024"
+  dbStorage_readAheadCacheMaxSizeMb: "1024"
+  zkServers: zk-0.zookeeper,zk-1.zookeeper,zk-2.zookeeper
+  statsProviderClass: org.apache.bookkeeper.stats.PrometheusMetricsProvider
+  useHostNameAsBookieID: "true"
+---
+apiVersion: apps/v1beta1
+kind: StatefulSet
+metadata:
+  name: bookkeeper
+  labels:
+    app: pulsar
+    component: bookkeeper
 spec:
-    ports:
-      - port: 3181
-        name: server
-    clusterIP: None
-    selector:
+  serviceName: bookkeeper
+  replicas: 3
+  template:
+    metadata:
+      labels:
         app: pulsar
         component: bookkeeper
+        cluster: pulsar-gke
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "8000"
+    spec:
+      terminationGracePeriodSeconds: 0
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 1
+              podAffinityTerm:
+                labelSelector:
+                  matchExpressions:
+                    - key: "app"
+                      operator: In
+                      values:
+                        - bookkeeper
+      initContainers:
+        - name: bookie-metaformat
+          image: apachepulsar/pulsar:latest
+          command: ["sh", "-c"]
+          args:
+            - >
+              bin/apply-config-from-env.py conf/bookkeeper.conf &&
+              bin/bookkeeper shell metaformat --nonInteractive || true;
+          envFrom:
+            - configMapRef:
+                name: bookie-config
+      containers:
+        - name: bookie
+          image: apachepulsar/pulsar:latest
+          command: ["sh", "-c"]
+          args:
+            - >
+              bin/apply-config-from-env.py conf/bookkeeper.conf &&
+              bin/apply-config-from-env.py conf/pulsar_env.sh &&
+              bin/pulsar bookie
+          ports:
+            - containerPort: 3181
+              hostPort: 3181
+              name: client
+          envFrom:
+            - configMapRef: 
+                name: bookie-config
+          volumeMounts:
+            - name: journal-disk
+              mountPath: /pulsar/data/bookkeeper/journal
+            - name: ledgers-disk
+              mountPath: /pulsar/data/bookkeeper/ledgers
+      initContainers:
+        # The first time, initialize BK metadata in zookeeper
+        # Otherwise ignore error if it's already there
+        - name: bookie-metaformat
+          image: apachepulsar/pulsar:latest
+          command: ["sh", "-c"]
+          args:
+            - >
+              bin/apply-config-from-env.py conf/bookkeeper.conf &&
+              bin/bookkeeper shell metaformat --nonInteractive || true;
+          envFrom:
+            - configMapRef:
+                name: bookie-config
+  volumeClaimTemplates:
+    - metadata:
+        name: journal-disk
+        annotations:
+          volume.alpha.kubernetes.io/storage-class: default
+        labels:
+          component: bookkeeper
+      spec:
+        accessModes: [ "ReadWriteOnce" ]
+        resources:
+          requests:
+            storage: 10Gi
+        storageClassName: bookie-ssd
+    - metadata:
+        name: ledger-disk
+        annotations:
+          volume.alpha.kubernetes.io/storage-class: default
+        labels:
+          component: bookkeeper
+      spec:
+        accessModes: [ "ReadWriteOnce" ]
+        resources:
+          requests:
+            storage: 10Gi
+        storageClassName: bookie-hdd
 
 ---
 ##
@@ -141,29 +164,29 @@ spec:
 apiVersion: apps/v1beta1
 kind: Deployment
 metadata:
-    name: bookie-autorecovery
+  name: bookie-autorecovery
 spec:
-    replicas: 2
-    template:
-        metadata:
-            labels:
-                app: pulsar
-                component: bookkeeper-replication
-        spec:
-            containers:
-              - name: replication-worker
-                image: apachepulsar/pulsar:latest
-                command: ["sh", "-c"]
-                args:
-                  - >
-                    bin/apply-config-from-env.py conf/bookkeeper.conf &&
-                    bin/bookkeeper autorecovery
-                envFrom:
-                  - configMapRef:
-                        name: bookie-config
-                env:
-                    ## Configure for lower mem usage
-                  - name: PULSAR_MEM
-                    value: "\" -Xmx256m \""
-                  - name: PULSAR_GC
-                    value: "\"  \""
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        app: pulsar
+        component: bookkeeper-replication
+    spec:
+      containers:
+        - name: replication-worker
+          image: apachepulsar/pulsar:latest
+          command: ["sh", "-c"]
+          args:
+            - >
+              bin/apply-config-from-env.py conf/bookkeeper.conf &&
+              bin/bookkeeper autorecovery
+          envFrom:
+            - configMapRef:
+                name: bookie-config
+          env:
+              ## Configure for lower mem usage
+            - name: PULSAR_MEM
+              value: "\" -Xmx256m \""
+            - name: PULSAR_GC
+              value: "\"  \""

deployment/kubernetes/google-kubernetes-engine/broker.yaml

@@ -67,7 +67,7 @@ spec:
                 env:
                  - name: advertisedAddress
                    valueFrom:
-                       fieldRef:
+                        fieldRef:
                            fieldPath: status.podIP
 ---
 
@@ -96,8 +96,6 @@ spec:
 
 ---
 
-###
-
 apiVersion: v1
 kind: Pod
 metadata:

deployment/kubernetes/google-kubernetes-engine/cluster-metadata.yaml

@@ -0,0 +1,42 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: pulsar-cluster-metadata-init
+  labels:
+    app: pulsar
+    component: broker
+spec:
+  template:
+    spec:
+      containers:
+        - name: pulsar-cluster-metadata-init-container
+          image: apachepulsar/pulsar:latest
+          command: ["sh", "-c"]
+          args:
+            - >
+              bin/pulsar initialize-cluster-metadata \
+                --cluster us-central \
+                --zookeeper zookeeper \
+                --global-zookeeper zookeeper \
+                --web-service-url http://broker.default.svc.cluster.local:8080/ \
+                --broker-service-url pulsar://broker.default.svc.cluster.local:6650/ || true;
+      restartPolicy: Never