Merge pull request big-data-europe#52 from nevmmv/2.0.0-hadoop3.2.1-j…

…ava8 moved to 3.2.1/stable hadoop version
irina-shchetinina · Feb 4, 2020 · e1e1e21 · e1e1e21
2 parents bc861a6 + 1835c7b
commit e1e1e21
Show file tree

Hide file tree

Showing 17 changed files with 39 additions and 33 deletions.
diff --git a/Makefile b/Makefile
@@ -13,7 +13,7 @@ build:
 wordcount:
 	docker build -t hadoop-wordcount ./submit
 	docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -mkdir -p /input/
-	docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -copyFromLocal /opt/hadoop-3.1.3/README.txt /input/
+	docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -copyFromLocal -f /opt/hadoop-3.2.1/README.txt /input/
 	docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} hadoop-wordcount
 	docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -cat /output/*
 	docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -rm -r /output

diff --git a/base/Dockerfile b/base/Dockerfile
@@ -17,7 +17,7 @@ RUN curl -O https://dist.apache.org/repos/dist/release/hadoop/common/KEYS
 
 RUN gpg --import KEYS
 
-ENV HADOOP_VERSION 3.1.3
+ENV HADOOP_VERSION 3.2.1
 ENV HADOOP_URL https://www.apache.org/dist/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz
 
 RUN set -x \
@@ -33,11 +33,11 @@ RUN mkdir /opt/hadoop-$HADOOP_VERSION/logs
 
 RUN mkdir /hadoop-data
 
-ENV HADOOP_PREFIX=/opt/hadoop-$HADOOP_VERSION
+ENV HADOOP_HOME=/opt/hadoop-$HADOOP_VERSION
 ENV HADOOP_CONF_DIR=/etc/hadoop
 ENV MULTIHOMED_NETWORK=1
 ENV USER=root
-ENV PATH $HADOOP_PREFIX/bin/:$PATH
+ENV PATH $HADOOP_HOME/bin/:$PATH
 
 ADD entrypoint.sh /entrypoint.sh
 

diff --git a/datanode/Dockerfile b/datanode/Dockerfile
@@ -1,4 +1,4 @@
-FROM bde2020/hadoop-base
+FROM bde2020/hadoop-base:2.0.0-hadoop3.2.1-java8
 
 MAINTAINER Ivan Ermilov <[email protected]>
 

diff --git a/datanode/run.sh b/datanode/run.sh
@@ -6,4 +6,4 @@ if [ ! -d $datadir ]; then
   exit 2
 fi
 
-$HADOOP_PREFIX/bin/hdfs --config $HADOOP_CONF_DIR datanode
+$HADOOP_HOME/bin/hdfs --config $HADOOP_CONF_DIR datanode
diff --git a/docker-compose-v3.yml b/docker-compose-v3.yml
@@ -2,7 +2,7 @@ version: '3'
 
 services:
   namenode:
-    image: bde2020/hadoop-namenode:2.0.0-hadoop3.1.3-java8
+    image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8
     networks:
       - hbase
     volumes:
@@ -24,7 +24,7 @@ services:
         traefik.port: 50070
 
   datanode:
-    image: bde2020/hadoop-datanode:2.0.0-hadoop3.1.3-java8
+    image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
     networks:
       - hbase
     volumes:
@@ -42,7 +42,7 @@ services:
         traefik.port: 50075
 
   resourcemanager:
-    image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.1.3-java8
+    image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.2.1-java8
     networks:
       - hbase
     environment:
@@ -64,7 +64,7 @@ services:
       disable: true
 
   nodemanager:
-    image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.1.3-java8
+    image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.2.1-java8
     networks:
       - hbase
     environment:
@@ -80,7 +80,7 @@ services:
         traefik.port: 8042
 
   historyserver:
-    image: bde2020/hadoop-historyserver:2.0.0-hadoop3.1.3-java8
+    image: bde2020/hadoop-historyserver:2.0.0-hadoop3.2.1-java8
     networks:
       - hbase
     volumes:

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -2,10 +2,12 @@ version: "3"
 
 services:
   namenode:
-    image: bde2020/hadoop-namenode:2.0.0-hadoop3.1.3-java8
+    image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8
     container_name: namenode
+    restart: always
     ports:
       - 9870:9870
+      - 9000:9000
     volumes:
       - hadoop_namenode:/hadoop/dfs/name
     environment:
@@ -14,8 +16,9 @@ services:
       - ./hadoop.env
 
   datanode:
-    image: bde2020/hadoop-datanode:2.0.0-hadoop3.1.3-java8
+    image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
     container_name: datanode
+    restart: always
     volumes:
       - hadoop_datanode:/hadoop/dfs/data
     environment:
@@ -24,26 +27,29 @@ services:
       - ./hadoop.env
 
   resourcemanager:
-    image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.1.3-java8
+    image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.2.1-java8
     container_name: resourcemanager
+    restart: always
     environment:
-      SERVICE_PRECONDITION: "namenode:9870 datanode:9864"
+      SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864"
     env_file:
       - ./hadoop.env
 
   nodemanager1:
-    image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.1.3-java8
+    image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.2.1-java8
     container_name: nodemanager
+    restart: always
     environment:
-      SERVICE_PRECONDITION: "namenode:9870 datanode:9864 resourcemanager:8088"
+      SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864 resourcemanager:8088"
     env_file:
       - ./hadoop.env
 
   historyserver:
-    image: bde2020/hadoop-historyserver:2.0.0-hadoop3.1.3-java8
+    image: bde2020/hadoop-historyserver:2.0.0-hadoop3.2.1-java8
     container_name: historyserver
+    restart: always
     environment:
-      SERVICE_PRECONDITION: "namenode:9870 datanode:9864 resourcemanager:8088"
+      SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864 resourcemanager:8088"
     volumes:
       - hadoop_historyserver:/hadoop/yarn/timeline
     env_file:

diff --git a/hadoop.env b/hadoop.env
@@ -38,6 +38,6 @@ MAPRED_CONF_mapreduce_map_memory_mb=4096
 MAPRED_CONF_mapreduce_reduce_memory_mb=8192
 MAPRED_CONF_mapreduce_map_java_opts=-Xmx3072m
 MAPRED_CONF_mapreduce_reduce_java_opts=-Xmx6144m
-MAPRED_CONF_yarn_app_mapreduce_am_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.3/
-MAPRED_CONF_mapreduce_map_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.3/
-MAPRED_CONF_mapreduce_reduce_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.1.3/
+MAPRED_CONF_yarn_app_mapreduce_am_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/
+MAPRED_CONF_mapreduce_map_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/
+MAPRED_CONF_mapreduce_reduce_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/
diff --git a/historyserver/Dockerfile b/historyserver/Dockerfile
@@ -1,4 +1,4 @@
-FROM bde2020/hadoop-base
+FROM bde2020/hadoop-base:2.0.0-hadoop3.2.1-java8
 
 MAINTAINER Ivan Ermilov <[email protected]>
 

diff --git a/historyserver/run.sh b/historyserver/run.sh
@@ -1,3 +1,3 @@
 #!/bin/bash
 
-$HADOOP_PREFIX/bin/yarn --config $HADOOP_CONF_DIR historyserver
+$HADOOP_HOME/bin/yarn --config $HADOOP_CONF_DIR historyserver
diff --git a/namenode/Dockerfile b/namenode/Dockerfile
@@ -1,4 +1,4 @@
-FROM bde2020/hadoop-base
+FROM bde2020/hadoop-base:2.0.0-hadoop3.2.1-java8
 
 MAINTAINER Ivan Ermilov <[email protected]>
 

diff --git a/namenode/run.sh b/namenode/run.sh
@@ -13,7 +13,7 @@ fi
 
 if [ "`ls -A $namedir`" == "" ]; then
   echo "Formatting namenode name directory: $namedir"
-  $HADOOP_PREFIX/bin/hdfs --config $HADOOP_CONF_DIR namenode -format $CLUSTER_NAME 
+  $HADOOP_HOME/bin/hdfs --config $HADOOP_CONF_DIR namenode -format $CLUSTER_NAME
 fi
 
-$HADOOP_PREFIX/bin/hdfs --config $HADOOP_CONF_DIR namenode
+$HADOOP_HOME/bin/hdfs --config $HADOOP_CONF_DIR namenode
diff --git a/nodemanager/Dockerfile b/nodemanager/Dockerfile
@@ -1,4 +1,4 @@
-FROM bde2020/hadoop-base
+FROM bde2020/hadoop-base:2.0.0-hadoop3.2.1-java8
 
 MAINTAINER Ivan Ermilov <[email protected]>
 

diff --git a/nodemanager/run.sh b/nodemanager/run.sh
@@ -1,3 +1,3 @@
 #!/bin/bash
 
-$HADOOP_PREFIX/bin/yarn --config $HADOOP_CONF_DIR nodemanager
+$HADOOP_HOME/bin/yarn --config $HADOOP_CONF_DIR nodemanager
diff --git a/resourcemanager/Dockerfile b/resourcemanager/Dockerfile
@@ -1,4 +1,4 @@
-FROM bde2020/hadoop-base
+FROM bde2020/hadoop-base:2.0.0-hadoop3.2.1-java8
 
 MAINTAINER Ivan Ermilov <[email protected]>
 

diff --git a/resourcemanager/run.sh b/resourcemanager/run.sh
@@ -1,3 +1,3 @@
 #!/bin/bash
 
-$HADOOP_PREFIX/bin/yarn --config $HADOOP_CONF_DIR resourcemanager
+$HADOOP_HOME/bin/yarn --config $HADOOP_CONF_DIR resourcemanager
diff --git a/submit/Dockerfile b/submit/Dockerfile
@@ -1,4 +1,4 @@
-FROM bde2020/hadoop-base
+FROM bde2020/hadoop-base:2.0.0-hadoop3.2.1-java8
 
 MAINTAINER Ivan Ermilov <[email protected]>
 

diff --git a/submit/run.sh b/submit/run.sh
@@ -1,3 +1,3 @@
 #!/bin/bash
 
-$HADOOP_PREFIX/bin/hadoop jar $JAR_FILEPATH $CLASS_TO_RUN $PARAMS 
+$HADOOP_HOME/bin/hadoop jar $JAR_FILEPATH $CLASS_TO_RUN $PARAMS
-Original file line number
+Diff line change
@@ Expand Up / @@ -6,4 +6,4 @@ if [ ! -d $datadir ]; then @@
       exit 2
     fi
-    $HADOOP_PREFIX/bin/hdfs --config $HADOOP_CONF_DIR datanode
+    $HADOOP_HOME/bin/hdfs --config $HADOOP_CONF_DIR datanode