adding Bigtop app

aws-blog-bigtop-application-emr/bigtop-deploy/puppet/modules/elasticsearch/manifests/init.pp

@@ -0,0 +1,62 @@
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class elasticsearch {
+  class client {
+
+    # Obtaining configuration parameters from cluster configuration files.
+    $clusterId = generate ("/bin/bash", "-c", "cat /mnt/var/lib/instance-controller/extraInstanceData.json |grep \"jobFlowId\" | cut -d':' -f2 | tr -d '\"' |tr -d ',' |tr -d ' '")
+    $region = generate ("/bin/bash", "-c", "cat /mnt/var/lib/instance-controller/extraInstanceData.json |grep 'region\"' | cut -d':' -f2 | tr -d '\"' |tr -d ',' |tr -d ' '")
+    $isMaster = generate ("/bin/bash", "-c", "cat /mnt/var/lib/info/instance.json | grep \"isMaster\" | cut -d':' -f2 | tr -d '\"' |tr -d ',' |tr -d ' '")
+    # Setting default port configuration
+    $elasticsearch_port = '9200'
+
+    include common
+
+  }
+
+
+  class common () {
+    # elasticsearch package installation. This line will run a 'yum install' command.
+    package { ["elasticsearch"]: ensure => latest, }
+
+    # elasticsearch service configuration from elasticsearch.yml template
+    file {
+      "/etc/elasticsearch/elasticsearch.yml":
+      content => template("elasticsearch/elasticsearch.yml"),
+    }
+
+    # elasticsearch aws plugin installation. 
+    exec { "install aws plugin":
+      command => "/usr/share/elasticsearch/bin/plugin -install elasticsearch/elasticsearch-cloud-aws/2.6.0",
+      onlyif  => "test ! -d /usr/share/elasticsearch/plugins/cloud-aws", 
+      path    => ['/usr/bin', 'usr/sbin', '/bin', '/sbin'],
+      require => [Package["elasticsearch"]],
+    }
+
+    # upstart init script installed from puppet template 
+    file {
+      "/etc/init/elasticsearch.conf":
+      content => template("elasticsearch/elasticsearch.conf"),
+    }
+
+    # elasticsearch service handling. service will restart on any change on file 'elasticsearch.yml'
+    service { "elasticsearch":
+      ensure => running,
+      require => Package["elasticsearch"],
+      subscribe => File["/etc/elasticsearch/elasticsearch.yml"],
+      hasrestart => true,
+      hasstatus => true,
+    }
+
+  }
+}

aws-blog-bigtop-application-emr/bigtop-deploy/puppet/modules/elasticsearch/templates/elasticsearch.conf

@@ -0,0 +1,22 @@
+description "Upstart job for Elasticsearch"
+author "hvivani"
+
+start on filesystem or runlevel [2345]
+stop on shutdown
+
+#respawn unlimited times with 5 seconds time interval
+#respawn limit 0 5
+respawn
+
+#service listening in a port and writing output to log file
+script
+   exec /usr/share/elasticsearch/bin/elasticsearch -Des.default.path.conf=/etc/elasticsearch >> /mnt/var/log/elasticsearch.log
+end script
+
+pre-start script
+    echo "[`date`] Elasticsearch Starting" >> /mnt/var/log/elasticsearch.log
+end script
+
+pre-stop script
+    echo "[`date`] Elasticsearch Stopping" >> /mnt/var/log/elasticsearch.log
+end script

aws-blog-bigtop-application-emr/bigtop-deploy/puppet/modules/elasticsearch/templates/elasticsearch.yml

@@ -0,0 +1,7 @@
+http.port: <%= @elasticsearch_port %>
+node.master: <%= @isMaster %>
+node.data: true
+cluster.name: <%= @clusterId %>
+discovery.type: ec2
+cloud.aws.region: <%= @region %>
+discovery.ec2.tag.aws:elasticmapreduce:job-flow-id: <%= @clusterId %>

aws-blog-bigtop-application-emr/bigtop-packages/src/common/elasticsearch/do-component-build

@@ -0,0 +1,20 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+. `dirname $0`/bigtop.bom
+
+mvn clean install -DskipTests -Dhadoop.version=$HADOOP_VERSION "$@"

aws-blog-bigtop-application-emr/bigtop-packages/src/common/elasticsearch/install_elasticsearch.sh

@@ -0,0 +1,141 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+usage() {
+  echo "
+usage: $0 <options>
+  Required not-so-options:
+     --distro-dir=DIR path to distro specific files (debian/RPM)
+     --build-dir=DIR             path to Tachyon dist.dir
+     --prefix=PREFIX             path to install into
+ 
+  Optional options:
+     --bin-dir=DIR               path to install bin
+     --data-dir=DIR              path to install local Tachyon data
+     ... [ see source for more similar options ]
+  "
+  exit 1
+}
+
+OPTS=$(getopt \
+  -n $0 \
+  -o '' \
+  -l 'prefix:' \
+  -l 'distro-dir:' \
+  -l 'bin-dir:' \
+  -l 'libexec-dir:' \
+  -l 'var-dir:' \
+  -l 'lib-dir:' \
+  -l 'data-dir:' \
+  -l 'build-dir:' -- "$@")
+
+if [ $? != 0 ] ; then
+    usage
+fi
+
+eval set -- "$OPTS"
+while true ; do
+    case "$1" in
+        --prefix)
+        PREFIX=$2 ; shift 2
+        ;;
+        --distro-dir)
+        DISTRO_DIR=$2 ; shift 2
+        ;;
+        --build-dir)
+        BUILD_DIR=$2 ; shift 2
+        ;;
+        --libexec-dir)
+        LIBEXEC_DIR=$2 ; shift 2
+        ;;
+        --lib-dir)
+        LIB_DIR=$2 ; shift 2
+        ;;
+        --bin-dir)
+        BIN_DIR=$2 ; shift 2
+        ;;
+        --var-dir)
+        VAR_DIR=$2 ; shift 2
+        ;;
+        --data-dir)
+        DATA_DIR=$2 ; shift 2
+        ;;
+        --)
+        shift ; break
+        ;;
+        *)
+        echo "Unknown option: $1"
+        usage
+        exit 1
+        ;;
+    esac
+done
+
+for var in PREFIX BUILD_DIR DISTRO_DIR ; do
+  if [ -z "$(eval "echo \$$var")" ]; then
+    echo Missing param: $var
+    usage
+  fi
+done
+
+LIB_DIR=${LIB_DIR:-/usr/share/elasticsearch}
+LIBEXEC_DIR=${INSTALLED_LIB_DIR:-/usr/libexec}
+BIN_DIR=${BIN_DIR:-/usr/bin}
+
+install -d -m 0755 $PREFIX/$LIB_DIR
+install -d -m 0755 $PREFIX/$LIB_DIR/plugins
+install -d -m 0755 $PREFIX/$LIB_DIR/bin
+install -d -m 0755 $PREFIX/$LIB_DIR/libexec
+install -d -m 0755 $PREFIX/$LIB_DIR/lib
+install -d -m 0755 $PREFIX/$LIB_DIR/lib/sigar
+install -d -m 0755 $PREFIX/$LIB_DIR/logs
+install -d -m 0755 $PREFIX/$DATA_DIR
+install -d -m 0755 $PREFIX/$DATA_DIR/elasticsearch
+install -d -m 0755 $PREFIX/etc
+install -d -m 0755 $PREFIX/etc/elasticsearch
+install -d -m 0755 $PREFIX/etc/sysconfig
+install -d -m 0755 $PREFIX/etc/sysconfig/elasticsearch
+install -d -m 0755 $PREFIX/etc/init.d
+install -d -m 0755 $PREFIX/usr/lib/systemd/system
+install -d -m 0755 $PREFIX/usr/lib/sysctl.d
+install -d -m 0755 $PREFIX/usr/lib/tmpfiles.d
+install -d -m 0755 $PREFIX/$VAR_DIR/log/elasticsearch
+install -d -m 0755 $PREFIX/var/run/elasticsearch
+install -d -m 0755 $PREFIX/var/lib/elasticsearch
+install -d -m 0755 $PREFIX/var/log/elasticsearch
+
+#cp -ra ${BUILD_DIR}/dist/lib/*.*ar $PREFIX/${LIB_DIR}/lib/
+#echo "==============================TARGET files============================="
+#ls -l target/
+cp -a lib/sigar/libsigar*.so $PREFIX/${LIB_DIR}/lib/sigar/
+cp -a target/lib/a*.jar $PREFIX/${LIB_DIR}/lib/
+cp -a target/elasticsearch*.jar $PREFIX/${LIB_DIR}/lib/
+cp -a target/lib/groovy*.jar $PREFIX/${LIB_DIR}/lib/
+cp -a target/lib/j*.jar $PREFIX/${LIB_DIR}/lib/
+cp -a target/lib/log*.jar $PREFIX/${LIB_DIR}/lib/
+cp -a target/lib/lucene*.jar $PREFIX/${LIB_DIR}/lib/
+cp -a target/lib/spatial*.jar $PREFIX/${LIB_DIR}/lib/
+
+cp -a target/bin/elasticsearch $PREFIX/${LIB_DIR}/bin/
+cp -a target/bin/elasticsearch.in.sh $PREFIX/${LIB_DIR}/bin/
+cp -a target/bin/plugin $PREFIX/${LIB_DIR}/bin/
+
+cp -a $DISTRO_DIR/logging.yml $PREFIX/etc/elasticsearch
+
+

aws-blog-bigtop-application-emr/bigtop-packages/src/common/elasticsearch/logging.yml

@@ -0,0 +1,56 @@
+# you can override this using by setting a system property, for example -Des.logger.level=DEBUG
+es.logger.level: INFO
+rootLogger: ${es.logger.level}, console, file
+logger:
+  # log action execution errors for easier debugging
+  action: DEBUG
+  # reduce the logging for aws, too much is logged under the default INFO
+  com.amazonaws: WARN
+
+  # gateway
+  #gateway: DEBUG
+  #index.gateway: DEBUG
+
+  # peer shard recovery
+  #indices.recovery: DEBUG
+
+  # discovery
+  #discovery: TRACE
+
+  index.search.slowlog: TRACE, index_search_slow_log_file
+  index.indexing.slowlog: TRACE, index_indexing_slow_log_file
+
+additivity:
+  index.search.slowlog: false
+  index.indexing.slowlog: false
+
+appender:
+  console:
+    type: console
+    layout:
+      type: consolePattern
+      conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"
+
+  file:
+    type: dailyRollingFile
+    file: ${path.logs}/${cluster.name}.log
+    datePattern: "'.'yyyy-MM-dd"
+    layout:
+      type: pattern
+      conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"
+
+  index_search_slow_log_file:
+    type: dailyRollingFile
+    file: ${path.logs}/${cluster.name}_index_search_slowlog.log
+    datePattern: "'.'yyyy-MM-dd"
+    layout:
+      type: pattern
+      conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"
+
+  index_indexing_slow_log_file:
+    type: dailyRollingFile
+    file: ${path.logs}/${cluster.name}_index_indexing_slowlog.log
+    datePattern: "'.'yyyy-MM-dd"
+    layout:
+      type: pattern
+      conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"