Applied local changes

cluster/ansible/files/tmp/certs/rootCA.pem

@@ -1,22 +1,21 @@
 -----BEGIN CERTIFICATE-----
-MIIDnTCCAoWgAwIBAgIJAPlNMt/eegoAMA0GCSqGSIb3DQEBBQUAMGUxCzAJBgNV
+MIIDgzCCAmugAwIBAgIJAMARjGxQy9rlMA0GCSqGSIb3DQEBCwUAMFgxCzAJBgNV
 BAYTAkRFMQ4wDAYDVQQHDAVXZWRlbDERMA8GA1UECgwIT3BlbkNvcmUxETAPBgNV
-BAsMCEludGVybmFsMSAwHgYDVQQDDBdpbnRlcm5hbC5sYXJzZ2VvcmdlLmNvbTAe
-Fw0xNjAzMDcxOTA1NDFaFw0xODEyMDIxOTA1NDFaMGUxCzAJBgNVBAYTAkRFMQ4w
-DAYDVQQHDAVXZWRlbDERMA8GA1UECgwIT3BlbkNvcmUxETAPBgNVBAsMCEludGVy
-bmFsMSAwHgYDVQQDDBdpbnRlcm5hbC5sYXJzZ2VvcmdlLmNvbTCCASIwDQYJKoZI
-hvcNAQEBBQADggEPADCCAQoCggEBAKm2CSJWd1koOGoA0uYTMQ9049D7ohSHrl55
-IsF3EriJHvZKmfNRDHA3fG2Tb8/CBelPOG+qsxPVx3C8zjCaINSJxwfBai20AtNV
-vJGU94RxexF7e680vzhoKTqOGI3Hcr+b5jE9JTJBwmfHpCGl96nWiOv/+Vpp9RSU
-euEj9cuHO9Lk6XFOHBMXwOUWylL5sDxUOR1c1FlBBUkJM8lXJoJn3DaGzfsDuZqD
-zjslkioqLjGY3N8kF2ZAq3p6DdqRiWLvjzUudAfyatfLuF5Og0P4JS6R4L7QWMr2
-8RKzP+2bWamuK71HU6A4+BzlY0+4gta6h0kEHqskTes++FDPOOkCAwEAAaNQME4w
-HQYDVR0OBBYEFIwKD1NHop5jyGD7o6VEqyWHAdN7MB8GA1UdIwQYMBaAFIwKD1NH
-op5jyGD7o6VEqyWHAdN7MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEB
-AJPDEwgNtM5GZ9I8oNvzcPgjuh7YNoO/VuuxouRhRyBJ1tdtNDF2QvYneKbYBixT
-O7ddDMthu5SWdTtF7HuSCoqssJt5TtCjI8VCasPhNnl+HDkdS2DH/LgQA3THTRRK
-OqYaAi0Ok6MuhNLYZCEtz4FlvM2qaXRq+mY73uHfvWn39Z3uCX1X6cMYyoGQGEqw
-5w/Gh8wrc0sP6bX5inZWgw96efjtaSPbSyl5YUv7cd+CXG2wccdLMmywhv/eisCN
-N8ZeJ9z1FAnrR51oqLrSeKheMxxHwNkBI2TMEo8pAxxykDdQfZAkT4SBMgvH5Aa/
-9UJ0egQvrsQMOofJImQLuH0=
+BAsMCEludGVybmFsMRMwEQYDVQQDDApoYmFzZS5ib29rMB4XDTE3MDIxOTA5NDMw
+MVoXDTE5MTExNjA5NDMwMVowWDELMAkGA1UEBhMCREUxDjAMBgNVBAcMBVdlZGVs
+MREwDwYDVQQKDAhPcGVuQ29yZTERMA8GA1UECwwISW50ZXJuYWwxEzARBgNVBAMM
+CmhiYXNlLmJvb2swggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDKURPJ
++MukIiscitfqqAf4CWmyXo0r+HdfRdFAMj2iCpf5JsERcNJ5TlsBPy7Gdqt1SLpG
+S+MeCEQJfiohoT5TpyCkev8L6BHQUZKLwE8elLI9vMfl7wmZkJKCXqHj2iE/DvuM
+7gCQ/5pGbbLUu9QzlXmW8SaEbnQYPQzOASTV50OSgE59L7qmwX/x5EaQtTvWW1Nb
+h4uEtAswImi5QP6qSngVdIzGC0c300upJtJBv/0PrFt1aSys8KYML5SzRSsYnCHd
+9cMQZ/KqEtfDrbrXVBkkMPKhFWAwFGtqG7kjAOneoTk4r1Xqfs/chGYlU6SS63ae
+0BWBVn5ZV3h0WXXnAgMBAAGjUDBOMB0GA1UdDgQWBBRkdm7WjNSbrDpJSCLEriJp
+kJQ3iDAfBgNVHSMEGDAWgBRkdm7WjNSbrDpJSCLEriJpkJQ3iDAMBgNVHRMEBTAD
+AQH/MA0GCSqGSIb3DQEBCwUAA4IBAQCTrcQ70/104hJb3Qhmoqh3LfAAS+HnSY7g
+Y17h+JxYLDxFuPoUH1GBAItKRrv619r+NrmJE7nKpklgHKGIdU6BWZLx18/1NK2+
+1XDoEeT9kth+LRa/udcFhRu2ifC7LAVrLeq3Cj92BKB5DuAW/lKrs3b6jbzP4LwW
+6gzYhfr6lshDkYYfp9ktXlLkM3R8bI9IWyXUfhTs/73sqgADwqKqLXWMB6l0dyWb
+Sfs8/oNcSYsbmfSpNotuAWQPdnjFTMuEUSTOwQBCV8JdhDtyOqM7SsGgzU905Ur8
+ndlOzu66mrv+X4A7n6s1r33sGBRYuw2X51pob+C/h2XygPMMM153
 -----END CERTIFICATE-----

cluster/ansible/group_vars/all

@@ -28,11 +28,11 @@ CLUSTER_DOMAIN: hbase.book
 # for example.
 # Setting this to false makes the playboo upload the tarballs to the remote machines from the machine that runs ansible.
 # In this case you need to provide tarballs for the versions configured below in the files/downloads/ directory.
-REMOTE_FETCH_TARBALLS: true
+REMOTE_FETCH_TARBALLS: false
 
 # The versions of software that will be installed by the playbooks
 HADOOP_VERSION: 2.7.3
-HBASE_VERSION: 1.2.4
+HBASE_VERSION: 1.3.0
 HUE_VERSION: 3.10.0
 ZOOKEEPER_VERSION: 3.4.9
 SPARK_VERSION: 1.6.1
@@ -44,6 +44,7 @@ CONFIG_BASE: "/etc" # config files will be placed in subdirectories of this fold
 VAR_RUN_BASE: "/var/opt"
 HADOOP_DATA: "/data/hadoop/hdfs" # the Hadoop data directory, all HDFS data will be stored in this directory
 ZOOKEEPER_DATA: "/data/zookeeper" # Zookeeper data will be stored here
+HBASE_TEMP: "/data/tmp"
 
 TMP_DIR: "/tmp"
 LOG_DIR: "/var/log/hadoop"
@@ -55,7 +56,6 @@ PID_DIR: "/var/run"
 SSL_STORE_PASSWORD: sslsekret
 SSL_KEY_PASSWORD: sslsekret
 
-
 # ==== Don't change anything below this line - really! ====
 
 OS_USERS: [ 'hdfs', 'yarn', 'mapred', 'hbase', 'zookeeper', 'hue', 'hadoop' ]
@@ -66,16 +66,15 @@ SSL_SERVICES:
   - { owner: 'hbase', path: 'hbase' }
   - { owner: 'zookeeper', path: 'zookeeper' }
 
+KRB_ADMIN: "hbasebook"
 KRB_REALM: "{{ CLUSTER_DOMAIN | upper }}"
-KRB_SERVER: master-1.hbase.book
+KRB_SERVER: "{{ groups['kdc_server'][0] }}"
 KRB_PRINCIPALS: [ 'hdfs', 'HTTP', 'yarn', 'hbase', 'zookeeper' ]
 KRB_SERVICES_WITH_HTTP: [ 'hdfs', 'yarn', 'hbase', 'zookeeper' ]
 
-
 # Paths
 JAVA_HOME: /etc/alternatives/jre_1.7.0
 
-
 BASE_PACKAGES_RH_PRE7: [ 'snappy', 'ntp', 'ntpdate', 'java-1.7.0-openjdk', 'java-1.7.0-openjdk-devel', 'libselinux-python', 'unzip', 'tar' ]
 BASE_PACKAGES_RH_7: [ 'snappy', 'ntp', 'ntpdate', 'java-1.7.0-openjdk', 'java-1.7.0-openjdk-devel', 'libselinux-python', 'unzip', 'tar' ]
 BASE_PACKAGES_DEBIAN: [ 'libsnappy-dev', 'ntp', 'ntpdate', 'openjdk-7-jdk', 'python-selinux', 'unzip', 'tar' ]

cluster/ansible/host_vars/master-1

@@ -1,3 +1,5 @@
 ---
 
-ZOOKEEPER_ID: 1
+ZOOKEEPER_ID: 1
+
+namenode: true

cluster/ansible/roles/common/files/hosts

@@ -1,7 +1,9 @@
 127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
 ::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
-10.10.0.10  master-1.hbase.book master-1
-10.10.0.11  master-2.hbase.book master-2
-10.10.0.20  worker-1.hbase.book worker-1
-10.10.0.21  worker-2.hbase.book worker-2
-10.10.0.22  worker-3.hbase.book worker-3
+
+10.0.20.1   master-1.hbase.book master-1
+10.0.20.2   master-2.hbase.book master-2
+10.0.20.3   master-3.hbase.book master-3
+10.0.20.10  worker-1.hbase.book worker-1
+10.0.20.11  worker-2.hbase.book worker-2
+10.0.20.12  worker-3.hbase.book worker-3

cluster/ansible/roles/common/tasks/main.yml

@@ -9,18 +9,18 @@
 # ==== Stop firewall ====
 
 # RedHat <= 6
-#- name: Stop IP Tables
-#  service: name={{ item }} state=stopped enabled=no
-#  with_items:
-#    - iptables
-#  when: (ansible_os_family == 'RedHat' and ansible_distribution_major_version <= "6") or ansible_os_family == 'Debian'
+- name: Stop IP Tables
+  service: name={{ item }} state=stopped enabled=no
+  with_items:
+    - iptables
+  when: (ansible_os_family == 'RedHat' and ansible_distribution_major_version <= "6") or ansible_os_family == 'Debian'
 
 # RedHat > 6
-#- name: Stop Firewalld
-#  service: name={{ item }} state=stopped enabled=no
-#  with_items:
-#    - firewalld
-#  when: ansible_os_family == 'RedHat' and ansible_distribution_major_version > "6"
+- name: Stop Firewalld
+  service: name={{ item }} state=stopped enabled=no
+  with_items:
+    - firewalld
+  when: ansible_os_family == 'RedHat' and ansible_distribution_major_version > "6"
 
 # ==== Stop firewall ====
 

cluster/ansible/roles/hbase/tasks/main.yml

@@ -4,17 +4,16 @@
 # Notes:
 #   - Executed on all servers in the cluster
 
-
-
-- name: Create Hadoop related directories (secure mode)
+- name: Create HBase related directories (secure mode)
   file: path={{ item.path }} owner={{ item.owner }} group=hadoop mode={{ item.mode }} recurse=yes state=directory
   with_items:
     - { path: '{{ VAR_RUN_BASE }}/hbase/logs', owner: 'hbase', mode: 755 }
     - { path: '{{ VAR_RUN_BASE }}/hbase/pid', owner: 'hbase', mode: 755 }
     - { path: '{{ HBASE_CONFIG }}', owner: 'hbase', mode: 775 }
+    - { path: '{{ HBASE_TEMP }}', owner: 'hdfs', mode: 775 }
   when: SECURITY_ENABLED == true
 
-- name: Create Hadoop related directories (non-secure mode)
+- name: Create HBase related directories (non-secure mode)
   file: path={{ item.path }} owner={{ item.owner }} group=hadoop mode={{ item.mode }} recurse=yes state=directory
   with_items:
     - { path: '{{ VAR_RUN_BASE }}/hbase/logs', owner: 'hadoop', mode: 775 }
@@ -50,8 +49,7 @@
     - { src: "etc/hbase/conf/hbase-site.xml.j2", dest: "{{ HBASE_CONFIG }}/hbase-site.xml" }
     - { src: "etc/hbase/conf/hadoop-metrics2-hbase.properties.j2", dest: "{{ HBASE_CONFIG }}/hadoop-metrics2-hbase.properties" }
     - { src: "etc/hbase/conf/hbase-policy.xml.j2", dest: "{{ HBASE_CONFIG }}/hbase-policy.xml" }
-    - { src: "etc/hbase/conf/hbase.jaas.j2", dest: "{{ HBASE_CONFIG }}/hbase.jaas" }
-    - { src: "etc/hbase/conf/zk-jaas.conf.j2", dest: "{{ HBASE_CONFIG }}/zk-jaas.conf" }
+    - { src: "etc/hbase/conf/hbase-jaas.conf.j2", dest: "{{ HBASE_CONFIG }}/hbase-jaas.conf" }
 
 # Also: hbase-policy.xml, log4j.properties, hadoop-metrics2-hbase.properties
 

cluster/ansible/roles/hbase/templates/etc/hbase/conf/hbase-env.sh.j2

@@ -25,12 +25,6 @@
 # so try to keep things idempotent unless you want to take an even deeper look
 # into the startup scripts (bin/hbase, etc.)
 
-{%- if SECURITY_ENABLED -%}
-export HBASE_OPTS="-Djava.security.auth.login.config={{ HBASE_CONFIG}}/zk-jaas.conf"
-export HBASE_MASTER_OPTS="-Djava.security.auth.login.config={{ HBASE_CONFIG}}/zk-jaas.conf"
-export HBASE_REGIONSERVER_OPTS="-Djava.security.auth.login.config={{ HBASE_CONFIG}}/hbase.jaas"
-{% endif %}
-
 # The java implementation to use.  Java 1.7+ required.
 # export JAVA_HOME=/usr/java/jdk1.6.0/
 export JAVA_HOME={{ JAVA_HOME }}
@@ -151,5 +145,10 @@ export HBASE_MANAGES_ZK=false
 # The reason for changing default to RFA is to avoid the boundary case of filling out disk space as 
 # DRFA doesn't put any cap on the log size. Please refer to HBase-5655 for more context.
 
+{%- if SECURITY_ENABLED -%}
+export HBASE_OPTS="$HBASE_OPTS -Djava.security.auth.login.config={{ HBASE_CONFIG }}/hbase-jaas.conf"
+export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Djava.security.auth.login.config={{ HBASE_CONFIG }}/hbase-jaas.conf"
+export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Djava.security.auth.login.config={{ HBASE_CONFIG }}/hbase-jaas.conf"
+{% endif %}
 
 

cluster/ansible/roles/hbase/templates/etc/hbase/conf/hbase-site.xml.j2

@@ -35,7 +35,7 @@
   </property>
   <property>
     <name>hbase.tmp.dir</name>
-    <value>/data/tmp/hbase-${user.name}</value>
+    <value>{{ HBASE_TEMP }}/hbase-${user.name}</value>
   </property>
   <property>
     <name>OFF-hbase.hlog.split.skip.errors</name>

cluster/ansible/roles/kdc-server/tasks/createprincs.yml

@@ -13,10 +13,10 @@
 - debug: var=principal_list.stdout_lines
 
 - name: Add UPNs
-  command: kadmin.local -q 'addprinc -pw krbsekret larsgeorge/admin@{{ KRB_REALM }}'
-  when: principal_list.stdout.find('larsgeorge/admin@{{ KRB_REALM }}') == -1
-- command: kadmin.local -q 'addprinc -pw krbsekret larsgeorge@{{ KRB_REALM }}'
-  when: principal_list.stdout.find('larsgeorge@{{ KRB_REALM }}') == -1
+  command: kadmin.local -q 'addprinc -pw krbsekret {{ KRB_ADMIN }}/admin@{{ KRB_REALM }}'
+  when: principal_list.stdout.find('{{ KRB_ADMIN }}/admin@{{ KRB_REALM }}') == -1
+- command: kadmin.local -q 'addprinc -pw krbsekret {{ KRB_ADMIN }}@{{ KRB_REALM }}'
+  when: principal_list.stdout.find('{{ KRB_ADMIN }}@{{ KRB_REALM }}') == -1
 - command: kadmin.local -q 'addprinc -pw krbsekret krbtgt/{{ KRB_REALM }}@{{ KRB_REALM }}'
   when: principal_list.stdout.find('krbtgt/{{ KRB_REALM }}@{{ KRB_REALM }}') == -1