Pass env config variables to hadoop if needed (apache-spark-on-k8s#15)

* Pass env config variables to hadoop if needed

* s/hadoop_env/customHadoopConfig/

It is a more compliant name (camelCase) and also less confusing as
hadoop also accepts env variables. Also clarified the meaning of these
env variables so that they do not get confused with hadoop env variables

charts/hdfs-datanode-k8s/templates/datanode-daemonset.yaml

@@ -50,6 +50,13 @@ spec:
             # K8s version 1.6 will fix this, per https://github.com/kubernetes/kubernetes/pull/29378.
             - name: CORE_CONF_fs_defaultFS
               value: hdfs://hdfs-namenode-0.hdfs-namenode.default.svc.cluster.local:8020
+            # We now add custom hadoop configuration provided
+            {{- range $key, $value := .Values.customHadoopConfig }}
+            {{- if and (ne $key "HDFS_CONF_dfs_datanode_data_dir") (ne $key "CORE_CONF_fs_defaultFS") }}
+            - name: {{ $key | quote }}
+              value: {{ $value | quote }}
+            {{- end }}
+            {{- end }}
           livenessProbe:
             initialDelaySeconds: 30
             httpGet:

charts/hdfs-datanode-k8s/values.yaml

@@ -21,3 +21,13 @@
 # e.g. --set "dataNodeHostPath={/hdfs-data,/hdfs-data1}"
 dataNodeHostPath:
   - /hdfs-data
+
+# Custom hadoop config keys passed through env variables to hadoop uhopper images.
+# See https://hub.docker.com/r/uhopper/hadoop/ to get more details
+# Please note that these are not hadoop env variables, but docker env variables that
+# will be transformed into hadoop config keys
+# HDFS_CONF_dfs_datanode_data_dir and CORE_CONF_fs_defaultFS need special handling and
+# they're already set by the chart so any value coming from below config will be ignored
+customHadoopConfig: {}
+  # Set variables through a hash where env variable is the key, e.g.
+  # HDFS_CONF_dfs_datanode_use_datanode_hostname: "false"

charts/hdfs-namenode-k8s/templates/namenode-statefulset.yaml

@@ -53,6 +53,13 @@ spec:
           env:
             - name: CLUSTER_NAME
               value: hdfs-k8s
+            # We now add custom hadoop configuration provided
+            {{- range $key, $value := .Values.customHadoopConfig }}
+            {{- if ne $key "CLUSTER_NAME" }}
+            - name: {{ $key | quote }}
+              value: {{ $value | quote }}
+            {{- end }}
+            {{- end }}
           ports:
           - containerPort: 8020
             name: fs

charts/hdfs-namenode-k8s/values.yaml

@@ -19,3 +19,13 @@
 # fsimage and edit logs. This will be mounted to the namenode as a k8s HostPath
 # volume.
 nameNodeHostPath: /hdfs-name
+
+# Custom hadoop config keys passed through env variables to hadoop uhopper images.
+# See https://hub.docker.com/r/uhopper/hadoop/ to get more details
+# Please note that these are not hadoop env variables, but docker env variables that
+# will be transformed into hadoop config keys
+# CLUSTER_NAME is already set by the chart so any value coming from below config
+# will be ignored
+customHadoopConfig: {}
+  # Set variables through a hash where env variable is the key, e.g.
+  # HDFS_CONF_dfs_datanode_use_datanode_hostname: "false"