adjust

datanode/run.sh

@@ -1,9 +1,17 @@
 #!/bin/bash
 
-datadir=`echo $HDFS_CONF_dfs_datanode_data_dir | perl -pe 's#file://##'`
-if [ ! -d $datadir ]; then
-  echo "Datanode data directory not found: $datadir"
-  exit 2
-fi
+. "/spark/sbin/spark-config.sh"
 
-$HADOOP_HOME/bin/hdfs --config $HADOOP_CONF_DIR datanode
+. "/spark/bin/load-spark-env.sh"
+
+mkdir -p $SPARK_WORKER_LOG
+
+export SPARK_HOME=/spark
+
+ln -sf /dev/stdout $SPARK_WORKER_LOG/spark-worker.out
+
+/spark/sbin/../bin/spark-class org.apache.spark.deploy.worker.Worker \
+    --webui-port $SPARK_WORKER_WEBUI_PORT $SPARK_MASTER >> $SPARK_WORKER_LOG/spark-worker.out
+
+# Keep the container running
+tail -f /dev/null

docker-compose.yml

@@ -132,6 +132,17 @@ services:
     environment:
       - "SPARK_MASTER=spark://spark-master:7077"
       - "SPARK_HOME=/spark"
+
+  ubuntu-spark-worker:
+    image: ubuntu-spark-worker
+    depends_on:
+      - spark-master
+    ports:
+      - "8082:8081"
+    environment:
+      - "SPARK_MASTER=spark://spark-master:7077"
+      - CORE_CONF_fs_defaultFS=hdfs://namenode:9000
+
 
 volumes:
   hadoop_namenode: