add hadoop helper scripts

bash/db_cluster/couchbase_cluster.sh → data_layer/couchbase/couchbase_cluster.sh

@@ -1,11 +1,15 @@
-#!/bin/bash -ex
+#!/bin/bash -e
 ##-------------------------------------------------------------------
-## File - couchbase_cluster.sh
-## Author : Denny <[email protected]>
+## @copyright 2016 DennyZhang.com
+## Licensed under MIT
+## https://raw.githubusercontent.com/DennyZhang/devops_public/master/LICENSE
+##
+## File : couchbase_cluster.sh
+## Author : Denny <[email protected]>
 ## Description :
 ## --
-## Created : <2015-06-22>
-## Updated: Time-stamp: <2016-06-23 15:19:41>
+## Created : <2016-06-04>
+## Updated: Time-stamp: <2016-06-23 15:59:40>
 ##-------------------------------------------------------------------
 cb_json="/tmp/$$.json"
 function shell_exit() {

data_layer/hadoop/format_hdfs.sh

@@ -0,0 +1,24 @@
+#!/bin/bash -e
+##-------------------------------------------------------------------
+## @copyright 2016 DennyZhang.com
+## Licensed under MIT
+## https://raw.githubusercontent.com/DennyZhang/devops_public/master/LICENSE
+##
+## File : format_hdfs.sh
+## Author : Denny <[email protected]>
+## Description :
+## --
+## Created : <2016-06-04>
+## Updated: Time-stamp: <2016-06-23 15:59:56>
+##-------------------------------------------------------------------
+flagfile=${1?}
+
+if [ -f "$flagfile" ]; then
+    log "Error: $flagfile exists, which indicates it's already initialized"
+    exit 1
+fi
+
+su -s /bin/bash hdfs -c "hdfs namenode -format"
+
+touch "$flagfile"
+## File : format_hdfs.sh ends

data_layer/hadoop/hadoop_jps.sh

@@ -0,0 +1,20 @@
+#!/bin/bash -e
+##-------------------------------------------------------------------
+## @copyright 2016 DennyZhang.com
+## Licensed under MIT
+## https://raw.githubusercontent.com/DennyZhang/devops_public/master/LICENSE
+##
+## File : hadoop_jps.sh
+## Author : Denny <[email protected]>
+## Description :
+## --
+## Created : <2016-06-04>
+## Updated: Time-stamp: <2016-06-23 16:00:05>
+##-------------------------------------------------------------------
+user_list="hdfs yarn zookeeper mapred oozie hive hbase"
+for hadoop_user in $user_list; do
+    if grep "$hadoop_user" /etc/passwd 1>/dev/null 2>&1; then
+        sudo -u "$hadoop_user" jps | grep -v Jps
+    fi
+done
+## File : hadoop_jps.sh ends

data_layer/hadoop/hdfs_createdir.sh

@@ -0,0 +1,31 @@
+#!/bin/bash -e
+##-------------------------------------------------------------------
+## @copyright 2016 DennyZhang.com
+## Licensed under MIT
+## https://raw.githubusercontent.com/DennyZhang/devops_public/master/LICENSE
+##
+## File : hdfs_createdir.sh
+## Author : Denny <[email protected]>
+## Description :
+## --
+## Created : <2016-06-04>
+## Updated: Time-stamp: <2016-06-23 16:00:09>
+##-------------------------------------------------------------------
+flagfile=${1?}
+shift
+dir_to_create=$*
+
+if [ -f "$flagfile" ]; then
+    log "Error: $flagfile exists, which indicates it's already initialized"
+    exit 1
+fi
+
+hadoop_user="hdfs"
+# create folders for hadoop-mapreduce-historyserver
+for dir in $dir_to_create; do
+    su -s /bin/bash "$hadoop_user" -c "hdfs dfs -mkdir -p ${dir}"
+    su -s /bin/bash "$hadoop_user" -c "hdfs dfs -chmod -R 777 ${dir}"
+done
+
+touch "$flagfile"
+## File : hdfs_createdir.sh ends

data_layer/hadoop/mapreduce_example.sh

@@ -0,0 +1,50 @@
+#!/bin/bash -e
+##-------------------------------------------------------------------
+## @copyright 2016 DennyZhang.com
+## Licensed under MIT
+## https://raw.githubusercontent.com/DennyZhang/devops_public/master/LICENSE
+##
+## File : mapreduce_example.sh
+## Author : Denny <[email protected]>
+## Description :
+## --
+## Created : <2016-06-04>
+## Updated: Time-stamp: <2016-06-23 16:00:13>
+##-------------------------------------------------------------------
+function log() {
+    local msg=$*
+    date_timestamp=$(date +['%Y-%m-%d %H:%M:%S'])
+    echo -ne "$date_timestamp $msg\n"
+
+    if [ -n "$LOG_FILE" ]; then
+        echo -ne "$date_timestamp $msg\n" >> "$LOG_FILE"
+    fi
+}
+
+mapreduce_sample_jar="/usr/hdp/2.3.4.7-4/hadoop-mapreduce/hadoop-mapreduce-examples-2.7.1.2.3.4.7-4.jar"
+output_id="output_$$_$(date +'%Y-%m-%d_%H-%M-%S')"
+log "Clean up directories: hdfs:/user/hdfs/test/input"
+su -s /bin/bash hdfs -c "hdfs dfs -rm -r /user/hdfs/test/input" || true
+
+log "Make source directory in hdfs: mkdir -p /user/hdfs/test/input"
+su -s /bin/bash hdfs -c "hdfs dfs -mkdir -p /user/hdfs/test/input"
+
+log "Data input: /etc/hadoop/conf"
+su -s /bin/bash hdfs -c "hdfs dfs -put /etc/hadoop/conf /user/hdfs/test/input"
+
+log "List data: hdfs:/user/hdfs/test/input/"
+su -s /bin/bash hdfs -c "hdfs dfs -ls /user/hdfs/test/input"
+
+# run mapr job
+su -s /bin/bash hdfs -c "yarn jar $mapreduce_sample_jar grep /user/hdfs/test/input/conf $output_id 'dfs[a-z.]+'"
+
+rm -rf /tmp/output/
+
+# get output
+su -s /bin/bash hdfs -c "hdfs dfs -get $output_id /tmp/output"
+
+log "Show result"
+cat /tmp/output/*
+
+log "Sample MapReduce Test pass"
+## File : mapreduce_example.sh ends