Merge pull request cdapio#169 from caskdata/feature/sql-connectors-re…

…cipe

Move SQL connectors to their own recipe

recipes/_sql_connectors.rb

@@ -0,0 +1,70 @@
+#
+# Cookbook Name:: hadoop
+# Recipe:: _sql_connectors
+#
+# Copyright © 2015 Cask Data, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+include_recipe 'hadoop::repo'
+
+pkgs = []
+jars = []
+
+# rubocop: disable Metrics/BlockNesting
+if node['hadoop'].key?('sql_connector')
+  case node['hadoop']['sql_connector']
+  when 'mysql'
+    if node['platform_family'] == 'rhel' && node['platform_version'].to_i == '5'
+      Chef::Log.warn('You must download and install JDBC connectors, manually')
+      pkgs = nil
+    else
+      if node['platform_family'] == 'debian' && node['hadoop']['distribution'] != 'hdp'
+        pkgs = ['libmysql-java']
+      else
+        pkgs = ['mysql-connector-java']
+      end
+    end
+    jars = pkgs
+  when 'postgresql'
+    if node['platform_family'] == 'rhel'
+      if node['platform_version'].to_i == '5'
+        Chef::Log.warn('You must download and install JDBC connectors, manually')
+        pkgs = nil
+      else
+        pkgs = ['postgresql-jdbc']
+      end
+      jars = pkgs
+    else # Assume debian
+      pkgs = ['libpostgresql-jdbc-java']
+      jars = ['postgresql-jdbc4']
+    end
+  ### TODO: Oracle support
+  when 'oracle'
+    Chef::Log.warn('You must download and install JDBC connectors, manually')
+    pkgs = nil
+    jars = pkgs
+  else
+    Chef::Log.info('No JDBC driver necessary')
+  end
+end
+# rubocop: enable Metrics/BlockNesting
+
+pkgs.each do |p|
+  package p do
+    action :install
+  end
+end
+
+node.default['hadoop']['sql_jars'] = jars

recipes/hive.rb

@@ -25,52 +25,6 @@
 
 hive_conf_dir = "/etc/hive/#{node['hive']['conf_dir']}"
 
-hive_data_dir =
-  if node['hadoop']['distribution'] == 'hdp' && (node['hadoop']['distribution_version'].to_s == '2' || \
-                                                 node['hadoop']['distribution_version'].to_f == 2.2)
-    '/usr/hdp/current/hive-client/lib'
-  else
-    '/usr/lib/hive/lib'
-  end
-
-java_share_dir = '/usr/share/java'
-
-case node['platform_family']
-when 'debian'
-  pkgs = %w(
-    mysql-connector-java
-    libpostgresql-jdbc-java
-  )
-  jars = %w(
-    mysql-connector-java
-    postgresql-jdbc4
-  )
-when 'rhel'
-  case node['platform_version'].to_i
-  when 6
-    pkgs = %w(
-      mysql-connector-java
-      postgresql-jdbc
-    )
-    jars = pkgs
-  else
-    Chef::Log.warn('You must download and install JDBC connectors')
-    pkgs = nil
-  end
-end
-
-pkgs.each do |pkg|
-  package pkg do
-    action :install
-  end
-end
-
-jars.each do |jar|
-  link "#{hive_data_dir}/#{jar}.jar" do
-    to "#{java_share_dir}/#{jar}.jar"
-  end
-end
-
 directory hive_conf_dir do
   mode '0755'
   owner 'root'

recipes/hive_metastore.rb

@@ -38,6 +38,33 @@
   end
 end
 
+hive_data_dir =
+  if node['hadoop']['distribution'] == 'hdp' && (node['hadoop']['distribution_version'].to_s == '2' || \
+                                                 node['hadoop']['distribution_version'].to_f == 2.2)
+    '/usr/hdp/current/hive-client/lib'
+  else
+    '/usr/lib/hive/lib'
+  end
+
+hive_sql =
+  if node['hive'].key?('hive_site') && node['hive']['hive_site'].key?('javax.jdo.option.ConnectionURL')
+    node['hive']['hive_site']['javax.jdo.option.ConnectionURL'].split(':')[1]
+  else
+    'derby'
+  end
+
+node.default['hadoop']['sql_connector'] = hive_sql
+include_recipe 'hadoop::_sql_connectors'
+
+java_share_dir = '/usr/share/java'
+jars = node['hadoop']['sql_jars']
+
+jars.each do |jar|
+  link "#{hive_data_dir}/#{jar}.jar" do
+    to "#{java_share_dir}/#{jar}.jar"
+  end
+end
+
 # Hive HDFS directories
 dfs = node['hadoop']['core_site']['fs.defaultFS']
 warehouse_dir =

recipes/oozie.rb

@@ -40,37 +40,19 @@
 
 oozie_conf_dir = "/etc/oozie/#{node['oozie']['conf_dir']}"
 oozie_data_dir = '/var/lib/oozie'
-java_share_dir = '/usr/share/java'
 
-case node['platform_family']
-when 'debian'
-  pkgs = %w(
-    mysql-connector-java
-    libpostgresql-jdbc-java
-  )
-  jars = %w(
-    mysql-connector-java
-    postgresql-jdbc4
-  )
-when 'rhel'
-  case node['platform_version'].to_i
-  when 6
-    pkgs = %w(
-      mysql-connector-java
-      postgresql-jdbc
-    )
-    jars = pkgs
+oozie_sql =
+  if node['oozie'].key?('oozie_site') && node['oozie']['oozie_site'].key?('oozie.service.JPAService.jdbc.url')
+    node['oozie']['oozie_site']['oozie.service.JPAService.jdbc.url'].split(':')[1]
   else
-    Chef::Log.warn('You must download and install JDBC connectors')
-    pkgs = nil
+    'derby'
   end
-end
 
-pkgs.each do |p|
-  package p do
-    action :install
-  end
-end
+node.default['hadoop']['sql_connector'] = oozie_sql
+include_recipe 'hadoop::_sql_connectors'
+
+java_share_dir = '/usr/share/java'
+jars = node['hadoop']['sql_jars']
 
 jars.each do |jar|
   link "#{oozie_data_dir}/#{jar}.jar" do

spec/_sql_connectors_spec.rb

@@ -0,0 +1,39 @@
+require 'spec_helper'
+
+describe 'hadoop::_sql_connectors' do
+  context 'Using MySQL on Centos 6.6 x86_64' do
+    let(:chef_run) do
+      ChefSpec::SoloRunner.new(platform: 'centos', version: 6.6) do |node|
+        node.default['hadoop']['sql_connector'] = 'mysql'
+      end.converge(described_recipe)
+    end
+
+    it 'install mysql-connector-java package' do
+      expect(chef_run).to install_package('mysql-connector-java')
+    end
+  end
+
+  context 'using PostgreSQL on CentOS 6.6 x86_64' do
+    let(:chef_run) do
+      ChefSpec::SoloRunner.new(platform: 'centos', version: 6.6) do |node|
+        node.default['hadoop']['sql_connector'] = 'postgresql'
+      end.converge(described_recipe)
+    end
+
+    it 'install postgresql-jdbc package' do
+      expect(chef_run).to install_package('postgresql-jdbc')
+    end
+  end
+
+  context 'Using PostgreSQL on Ubuntu 12.04' do
+    let(:chef_run) do
+      ChefSpec::SoloRunner.new(platform: 'ubuntu', version: 12.04) do |node|
+        node.default['hadoop']['sql_connector'] = 'postgresql'
+      end.converge(described_recipe)
+    end
+
+    it 'install libpostgresql-jdbc-java package' do
+      expect(chef_run).to install_package('libpostgresql-jdbc-java')
+    end
+  end
+end

spec/hive_metastore_spec.rb

@@ -19,6 +19,12 @@
       expect(chef_run).to run_ruby_block("package-#{pkg}")
     end
 
+    %w(mysql-connector-java postgresql-jdbc).each do |p|
+      it "does not install #{p} package" do
+        expect(chef_run).not_to install_package(p)
+      end
+    end
+
     it "creates #{pkg} service resource, but does not run it" do
       expect(chef_run).to_not disable_service(pkg)
       expect(chef_run).to_not enable_service(pkg)
@@ -36,4 +42,68 @@
       expect(chef_run).not_to run_execute('hive-hdfs-warehousedir')
     end
   end
+
+  context 'using MySQL on HDP 2.2' do
+    let(:chef_run) do
+      ChefSpec::SoloRunner.new(platform: 'centos', version: 6.6) do |node|
+        node.override['hadoop']['distribution'] = 'hdp'
+        node.override['hadoop']['distribution_version'] = '2.2.4.2'
+        node.override['hive']['hive_site']['javax.jdo.option.ConnectionURL'] = 'jdbc:mysql:localhost/hive'
+        node.automatic['domain'] = 'example.com'
+        node.default['hive']['hive_env']['hive_log_dir'] = '/data/log/hive'
+        node.default['hive']['hive_site']['hive.exec.local.scratchdir'] = '/tmp/hive/scratch'
+        stub_command('test -L /var/log/hive').and_return(false)
+        stub_command('update-alternatives --display hive-conf | grep best | awk \'{print $5}\' | grep /etc/hive/conf.chef').and_return(false)
+        stub_command(%r{/sys/kernel/mm/(.*)transparent_hugepage/defrag}).and_return(false)
+      end.converge(described_recipe)
+    end
+
+    it 'link mysql-connector-java.jar' do
+      link = chef_run.link('/usr/hdp/current/hive-client/lib/mysql-connector-java.jar')
+      expect(link).to link_to('/usr/share/java/mysql-connector-java.jar')
+    end
+  end
+
+  context 'using PostgreSQL on Ubuntu 12.04' do
+    let(:chef_run) do
+      ChefSpec::SoloRunner.new(platform: 'ubuntu', version: 12.04) do |node|
+        node.override['hive']['hive_site']['javax.jdo.option.ConnectionURL'] = 'jdbc:postgresql:localhost/hive'
+        node.automatic['domain'] = 'example.com'
+        stub_command('update-alternatives --display hive-conf | grep best | awk \'{print $5}\' | grep /etc/hive/conf.chef').and_return(false)
+        stub_command(%r{/sys/kernel/mm/(.*)transparent_hugepage/defrag}).and_return(false)
+      end.converge(described_recipe)
+    end
+    pkg = 'hive-metastore'
+
+    it "does not install #{pkg} package" do
+      expect(chef_run).not_to install_package(pkg)
+    end
+
+    it "runs package-#{pkg} ruby_block" do
+      expect(chef_run).to run_ruby_block("package-#{pkg}")
+    end
+
+    it 'link postgresql-jdbc4.jar' do
+      link = chef_run.link('/usr/lib/hive/lib/postgresql-jdbc4.jar')
+      expect(link).to link_to('/usr/share/java/postgresql-jdbc4.jar')
+    end
+  end
+
+  context 'using PostgreSQL on Ubuntu 12.04 HDP 2.2' do
+    let(:chef_run) do
+      ChefSpec::SoloRunner.new(platform: 'ubuntu', version: 12.04) do |node|
+        node.override['hadoop']['distribution'] = 'hdp'
+        node.override['hadoop']['distribution_version'] = '2.2.4.2'
+        node.override['hive']['hive_site']['javax.jdo.option.ConnectionURL'] = 'jdbc:postgresql:localhost/hive'
+        node.automatic['domain'] = 'example.com'
+        stub_command('update-alternatives --display hive-conf | grep best | awk \'{print $5}\' | grep /etc/hive/conf.chef').and_return(false)
+        stub_command(%r{/sys/kernel/mm/(.*)transparent_hugepage/defrag}).and_return(false)
+      end.converge(described_recipe)
+    end
+
+    it 'link postgresql-jdbc4.jar' do
+      link = chef_run.link('/usr/hdp/current/hive-client/lib/postgresql-jdbc4.jar')
+      expect(link).to link_to('/usr/share/java/postgresql-jdbc4.jar')
+    end
+  end
 end