Now includes Hive too

bbsoft0 · Oct 23, 2020 · 2cffc41 · 2cffc41
1 parent 0f1cbe1
commit 2cffc41
Showing 1 changed file with 41 additions and 18 deletions.
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -15,8 +15,6 @@ services:
       - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
     env_file:
       - ./hadoop.env
-    networks:
-      - spark-net
 
   datanode:
     image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
@@ -27,10 +25,10 @@ services:
     environment:
       SERVICE_PRECONDITION: "namenode:9870"
       CORE_CONF_fs_defaultFS: hdfs://namenode:8020
+    ports:
+      - "9864:9864"
     env_file:
       - ./hadoop.env
-    networks:
-      - spark-net
 
   resourcemanager:
     image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.2.1-java8
@@ -40,8 +38,6 @@ services:
       SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864"
     env_file:
       - ./hadoop.env
-    networks:
-      - spark-net
 
   nodemanager1:
     image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.2.1-java8
@@ -51,8 +47,6 @@ services:
       SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864 resourcemanager:8088"
     env_file:
       - ./hadoop.env
-    networks:
-      - spark-net
 
   historyserver:
     image: bde2020/hadoop-historyserver:2.0.0-hadoop3.2.1-java8
@@ -64,20 +58,19 @@ services:
       - hadoop_historyserver:/hadoop/yarn/timeline
     env_file:
       - ./hadoop.env
-    networks:
-      - spark-net
 
   spark-master:
     image: bde2020/spark-master:3.0.0-hadoop3.2
     container_name: spark-master
+    depends_on:
+      - namenode
+      - datanode
     ports:
       - "8080:8080"
       - "7077:7077"
     environment:
       - INIT_DAEMON_STEP=setup_spark
       - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
-    networks:
-      - spark-net
 
   spark-worker-1:
     image: bde2020/spark-worker:3.0.0-hadoop3.2
@@ -89,14 +82,44 @@ services:
     environment:
       - "SPARK_MASTER=spark://spark-master:7077"
       - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
-    networks:
-      - spark-net
-
+
+  hive-server:
+    image: bde2020/hive:2.3.2-postgresql-metastore
+    container_name: hive-server
+    depends_on:
+      - namenode
+      - datanode
+    env_file:
+      - ./hadoop-hive.env
+    environment:
+      HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://hive-metastore/metastore"
+      SERVICE_PRECONDITION: "hive-metastore:9083"
+    ports:
+      - "10000:10000"
+
+  hive-metastore:
+    image: bde2020/hive:2.3.2-postgresql-metastore
+    container_name: hive-metastore
+    env_file:
+      - ./hadoop-hive.env
+    command: /opt/hive/bin/hive --service metastore
+    environment:
+      SERVICE_PRECONDITION: "namenode:9870 datanode:9864 hive-metastore-postgresql:5432"
+    ports:
+      - "9083:9083"
+
+  hive-metastore-postgresql:
+    image: bde2020/hive-metastore-postgresql:2.3.0
+    container_name: hive-metastore-postgresql
+
+  presto-coordinator:
+    image: shawnzhu/prestodb:0.181
+    container_name: presto-coordinator
+    ports:
+      - "8089:8089"
+
 volumes:
   hadoop_namenode:
   hadoop_datanode:
   hadoop_historyserver:
 
-networks:
-  spark-net:
-    name: spark-net