Adds Hiera implementation within Packstack

Packstack configures Hiera as data backend. Packstack puppet templates are now
using hiera() and hiera_array() functions to fetch data from hiera backend.

Packstack generates a defaults.yaml file in the /var/tmp/packstack directory.

Firewall rules for each openstack components are inserted into the hiera
backend as hash and created by the create_resources function.

Change-Id: Iab553a71264b0fc0f26d33a6304b545ad302f664
Fixes: rhbz#1145223
Signed-off-by: Gael Chamoulaud <[email protected]>

packstack/installer/basedefs.py

@@ -43,6 +43,8 @@
 DIR_LOG = VAR_DIR
 PUPPET_MANIFEST_RELATIVE = "manifests"
 PUPPET_MANIFEST_DIR = os.path.join(VAR_DIR, PUPPET_MANIFEST_RELATIVE)
+HIERADATA_FILE_RELATIVE = "hieradata"
+HIERADATA_DIR = os.path.join(VAR_DIR, HIERADATA_FILE_RELATIVE)
 
 FILE_INSTALLER_LOG = "setup.log"
 

packstack/modules/ospluginutils.py

@@ -2,6 +2,7 @@
 import logging
 import os
 import re
+import yaml
 
 from packstack.installer import basedefs
 from packstack.installer.setup_controller import Controller
@@ -11,6 +12,7 @@
 
 PUPPET_DIR = os.path.join(basedefs.DIR_PROJECT_DIR, "puppet")
 PUPPET_TEMPLATE_DIR = os.path.join(PUPPET_DIR, "templates")
+HIERA_DEFAULTS_YAML = os.path.join(basedefs.HIERADATA_DIR, "defaults.yaml")
 
 
 class NovaConfig(object):
@@ -80,6 +82,19 @@ def appendManifestFile(manifest_name, data, marker=''):
     manifestfiles.addFile(manifest_name, marker, data)
 
 
+def generateHieraDataFile():
+    os.mkdir(basedefs.HIERADATA_DIR, 0700)
+    with open(HIERA_DEFAULTS_YAML, 'w') as outfile:
+        outfile.write(yaml.dump(controller.CONF,
+                                explicit_start=True,
+                                default_flow_style=False))
+
+
+def createFirewallResources(hiera_key, default_value='{}'):
+    hiera_function = "hiera('%s', %s)" % (hiera_key, default_value)
+    return "create_resources(packstack::firewall, %s)\n\n" % hiera_function
+
+
 def gethostlist(CONF):
     hosts = []
     for key, value in CONF.items():

packstack/plugins/amqp_002.py

@@ -15,7 +15,8 @@
 
 from packstack.modules.common import filtered_hosts
 from packstack.modules.ospluginutils import (getManifestTemplate,
-                                             appendManifestFile)
+                                             appendManifestFile,
+                                             createFirewallResources)
 
 
 #------------------ oVirt installer initialization ------------------
@@ -219,7 +220,7 @@ def initSequences(controller):
 def create_manifest(config, messages):
     server = utils.ScriptRunner(config['CONFIG_AMQP_HOST'])
     if config['CONFIG_AMQP_ENABLE_SSL'] == 'y':
-        config['CONFIG_AMQP_ENABLE_SSL'] = 'true'
+        config['CONFIG_AMQP_ENABLE_SSL'] = True
         config['CONFIG_AMQP_PROTOCOL'] = 'ssl'
         config['CONFIG_AMQP_CLIENTS_PORT'] = "5671"
         if config['CONFIG_AMQP_SSL_SELF_SIGNED'] == 'y':
@@ -234,10 +235,10 @@ def create_manifest(config, messages):
         # Set default values
         config['CONFIG_AMQP_CLIENTS_PORT'] = "5672"
         config['CONFIG_AMQP_SSL_PORT'] = "5671"
-        config['CONFIG_AMQP_SSL_CERT_FILE'] = ""
-        config['CONFIG_AMQP_SSL_KEY_FILE'] = ""
-        config['CONFIG_AMQP_NSS_CERTDB_PW'] = ""
-        config['CONFIG_AMQP_ENABLE_SSL'] = 'false'
+        config['CONFIG_AMQP_SSL_CERT_FILE'] = ''
+        config['CONFIG_AMQP_SSL_KEY_FILE'] = ''
+        config['CONFIG_AMQP_NSS_CERTDB_PW'] = ''
+        config['CONFIG_AMQP_ENABLE_SSL'] = False
         config['CONFIG_AMQP_PROTOCOL'] = 'tcp'
 
     if config['CONFIG_AMQP_ENABLE_AUTH'] == 'n':
@@ -247,14 +248,17 @@ def create_manifest(config, messages):
     manifestfile = "%s_amqp.pp" % config['CONFIG_AMQP_HOST']
     manifestdata = getManifestTemplate('amqp.pp')
 
+    fw_details = dict()
     # All hosts should be able to talk to amqp
-    config['FIREWALL_SERVICE_NAME'] = "amqp"
-    config['FIREWALL_PORTS'] = "['5671', '5672']"
-    config['FIREWALL_CHAIN'] = "INPUT"
-    config['FIREWALL_PROTOCOL'] = 'tcp'
     for host in filtered_hosts(config, exclude=False):
-        config['FIREWALL_ALLOWED'] = "'%s'" % host
-        config['FIREWALL_SERVICE_ID'] = "amqp_%s" % host
-        manifestdata += getManifestTemplate("firewall.pp")
+        key = "amqp_%s" % host
+        fw_details.setdefault(key, {})
+        fw_details[key]['host'] = "%s" % host
+        fw_details[key]['service_name'] = "amqp"
+        fw_details[key]['chain'] = "INPUT"
+        fw_details[key]['ports'] = ['5671', '5672']
+        fw_details[key]['proto'] = "tcp"
+    config['FIREWALL_AMQP_RULES'] = fw_details
 
+    manifestdata += createFirewallResources('FIREWALL_AMQP_RULES')
     appendManifestFile(manifestfile, manifestdata, 'pre')

packstack/plugins/ceilometer_800.py

@@ -13,7 +13,8 @@
 from packstack.installer import processors
 from packstack.modules.shortcuts import get_mq
 from packstack.modules.ospluginutils import (getManifestTemplate,
-                                             appendManifestFile)
+                                             appendManifestFile,
+                                             createFirewallResources)
 
 
 #------------------ oVirt installer initialization ------------------
@@ -112,13 +113,17 @@ def create_manifest(config, messages):
     manifestdata = getManifestTemplate(get_mq(config, "ceilometer"))
     manifestdata += getManifestTemplate("ceilometer.pp")
 
-    config['FIREWALL_ALLOWED'] = "'ALL'"
-    config['FIREWALL_SERVICE_NAME'] = 'ceilometer-api'
-    config['FIREWALL_SERVICE_ID'] = 'ceilometer_api'
-    config['FIREWALL_PORTS'] = "'8777'"
-    config['FIREWALL_CHAIN'] = "INPUT"
-    config['FIREWALL_PROTOCOL'] = 'tcp'
-    manifestdata += getManifestTemplate("firewall.pp")
+    fw_details = dict()
+    key = "ceilometer_api"
+    fw_details.setdefault(key, {})
+    fw_details[key]['host'] = "ALL"
+    fw_details[key]['service_name'] = "ceilometer-api"
+    fw_details[key]['chain'] = "INPUT"
+    fw_details[key]['ports'] = ['8777']
+    fw_details[key]['proto'] = "tcp"
+    config['FIREWALL_CEILOMETER_RULES'] = fw_details
+    manifestdata += createFirewallResources('FIREWALL_CEILOMETER_RULES')
+
     # Add a template that creates a group for nova because the ceilometer
     # class needs it
     if config['CONFIG_NOVA_INSTALL'] == 'n':
@@ -129,11 +134,18 @@ def create_manifest(config, messages):
 def create_mongodb_manifest(config, messages):
     manifestfile = "%s_mongodb.pp" % config['CONFIG_MONGODB_HOST']
     manifestdata = getManifestTemplate("mongodb.pp")
-    config['FIREWALL_ALLOWED'] = "'%s'" % config['CONFIG_CONTROLLER_HOST']
-    config['FIREWALL_SERVICE_NAME'] = 'mongodb-server'
-    config['FIREWALL_PORTS'] = "'27017'"
-    config['FIREWALL_PROTOCOL'] = 'tcp'
-    manifestdata += getManifestTemplate("firewall.pp")
+
+    fw_details = dict()
+    key = "mongodb_server"
+    fw_details.setdefault(key, {})
+    fw_details[key]['host'] = "%s" % config['CONFIG_CONTROLLER_HOST']
+    fw_details[key]['service_name'] = "mongodb-server"
+    fw_details[key]['chain'] = "INPUT"
+    fw_details[key]['ports'] = ['27017']
+    fw_details[key]['proto'] = "tcp"
+    config['FIREWALL_MONGODB_RULES'] = fw_details
+
+    manifestdata += createFirewallResources('FIREWALL_MONGODB_RULES')
     appendManifestFile(manifestfile, manifestdata, 'pre')
 
 

packstack/plugins/cinder_250.py

@@ -20,7 +20,8 @@
 
 from packstack.modules.shortcuts import get_mq
 from packstack.modules.ospluginutils import (getManifestTemplate,
-                                             appendManifestFile)
+                                             appendManifestFile,
+                                             createFirewallResources)
 
 from packstack.installer import exceptions
 from packstack.installer import output_messages
@@ -125,10 +126,10 @@ def initConfig(controller):
                        "domain:/vol-name "),
              "PROMPT": ("Enter a single or comma separated list of gluster "
                         "volume shares to use with Cinder"),
-             "OPTION_LIST": ["^'([\d]{1,3}\.){3}[\d]{1,3}:/.*'",
-                             "^'[a-zA-Z0-9][\-\.\w]*:/.*'"],
+             "OPTION_LIST": ["^([\d]{1,3}\.){3}[\d]{1,3}:/.*",
+                             "^[a-zA-Z0-9][\-\.\w]*:/.*"],
              "VALIDATORS": [validators.validate_multi_regexp],
-             "PROCESSORS": [processors.process_add_quotes_around_values],
+             "PROCESSORS": [],
              "DEFAULT_VALUE": "",
              "MASK_INPUT": False,
              "LOOSE_VALIDATION": True,
@@ -144,9 +145,9 @@ def initConfig(controller):
                        "mount, eg: ip-address:/export-name "),
              "PROMPT": ("Enter a single or comma seprated list of NFS exports "
                         "to use with Cinder"),
-             "OPTION_LIST": ["^'([\d]{1,3}\.){3}[\d]{1,3}:/.*'"],
+             "OPTION_LIST": ["^([\d]{1,3}\.){3}[\d]{1,3}:/.*"],
              "VALIDATORS": [validators.validate_multi_regexp],
-             "PROCESSORS": [processors.process_add_quotes_around_values],
+             "PROCESSORS": [],
              "DEFAULT_VALUE": "",
              "MASK_INPUT": False,
              "LOOSE_VALIDATION": True,
@@ -592,10 +593,16 @@ def initSequences(controller):
     if config['CONFIG_CINDER_INSTALL'] != 'y':
         return
 
-    config['CONFIG_CINDER_BACKEND'] = str(
+    config['CONFIG_CINDER_BACKEND'] = (
         [i.strip() for i in config['CONFIG_CINDER_BACKEND'].split(',') if i]
     )
 
+    for key in ('CONFIG_CINDER_NETAPP_VOLUME_LIST',
+                'CONFIG_CINDER_GLUSTER_MOUNTS',
+                'CONFIG_CINDER_NFS_MOUNTS'):
+        if key in config:
+            config[key] = [i.strip() for i in config[key].split(',') if i]
+
     cinder_steps = [
         {'title': 'Adding Cinder Keystone manifest entries',
          'functions': [create_keystone_manifest]},
@@ -711,8 +718,7 @@ def create_manifest(config, messages):
     manifestfile = "%s_cinder.pp" % config['CONFIG_STORAGE_HOST']
     manifestdata += getManifestTemplate("cinder.pp")
 
-    backends = config['CONFIG_CINDER_BACKEND'].strip('[]')
-    backends = [i.strip('\' ') for i in backends.split(',')]
+    backends = config['CONFIG_CINDER_BACKEND']
     if 'netapp' in backends:
         backends.remove('netapp')
         puppet_cdot_iscsi = "cinder_netapp_cdot_iscsi.pp"
@@ -740,24 +746,36 @@ def create_manifest(config, messages):
     if config['CONFIG_SWIFT_INSTALL'] == 'y':
         manifestdata += getManifestTemplate('cinder_backup.pp')
 
-    config['FIREWALL_SERVICE_NAME'] = "cinder"
-    config['FIREWALL_PORTS'] = "['3260']"
-    config['FIREWALL_CHAIN'] = "INPUT"
-    config['FIREWALL_PROTOCOL'] = 'tcp'
-    if (config['CONFIG_NOVA_INSTALL'] == 'y' and
+    fw_details = dict()
+    for host in split_hosts(config['CONFIG_COMPUTE_HOSTS']):
+        if (config['CONFIG_NOVA_INSTALL'] == 'y' and
             config['CONFIG_VMWARE_BACKEND'] == 'n'):
-        for host in split_hosts(config['CONFIG_COMPUTE_HOSTS']):
-            config['FIREWALL_ALLOWED'] = "'%s'" % host
-            config['FIREWALL_SERVICE_ID'] = "cinder_%s" % host
-            manifestdata += getManifestTemplate("firewall.pp")
-    else:
-        config['FIREWALL_ALLOWED'] = "'ALL'"
-        config['FIREWALL_SERVICE_ID'] = "cinder_ALL"
-        manifestdata += getManifestTemplate("firewall.pp")
+            key = "cinder_%s" % host
+            fw_details.setdefault(key, {})
+            fw_details[key]['host'] = "%s" % host
+        else:
+            key = "cinder_all"
+            fw_details.setdefault(key, {})
+            fw_details[key]['host'] = "ALL"
+
+        fw_details[key]['service_name'] = "cinder"
+        fw_details[key]['chain'] = "INPUT"
+        fw_details[key]['ports'] = ['3260']
+        fw_details[key]['proto'] = "tcp"
+
+    config['FIREWALL_CINDER_RULES'] = fw_details
+    manifestdata += createFirewallResources('FIREWALL_CINDER_RULES')
+
     # cinder API should be open for everyone
-    config['FIREWALL_SERVICE_NAME'] = "cinder-api"
-    config['FIREWALL_ALLOWED'] = "'ALL'"
-    config['FIREWALL_SERVICE_ID'] = "cinder_API"
-    config['FIREWALL_PORTS'] = "['8776']"
-    manifestdata += getManifestTemplate("firewall.pp")
+    fw_details = dict()
+    key = "cinder_api"
+    fw_details.setdefault(key, {})
+    fw_details[key]['host'] = "ALL"
+    fw_details[key]['service_name'] = "cinder-api"
+    fw_details[key]['chain'] = "INPUT"
+    fw_details[key]['ports'] = ['8776']
+    fw_details[key]['proto'] = "tcp"
+    config['FIREWALL_CINDER_API_RULES'] = fw_details
+    manifestdata += createFirewallResources('FIREWALL_CINDER_API_RULES')
+
     appendManifestFile(manifestfile, manifestdata)

packstack/plugins/dashboard_500.py

@@ -123,11 +123,11 @@ def create_manifest(config, messages):
     manifestfile = "%s_horizon.pp" % horizon_host
 
     proto = "http"
-    config["CONFIG_HORIZON_PORT"] = "'80'"
+    config["CONFIG_HORIZON_PORT"] = 80
     sslmanifestdata = ''
     if config["CONFIG_HORIZON_SSL"] == 'y':
-        config["CONFIG_HORIZON_SSL"] = 'true'
-        config["CONFIG_HORIZON_PORT"] = "'443'"
+        config["CONFIG_HORIZON_SSL"] = True
+        config["CONFIG_HORIZON_PORT"] = 443
         proto = "https"
 
         # Are we using the users cert/key files
@@ -160,16 +160,16 @@ def create_manifest(config, messages):
                 "/etc/httpd/conf.d/ssl.conf on %s to use a CA signed cert."
                 % (utils.COLORS['red'], utils.COLORS['nocolor'], horizon_host))
     else:
-        config["CONFIG_HORIZON_SSL"] = 'false'
+        config["CONFIG_HORIZON_SSL"] = False
 
-    config["CONFIG_HORIZON_NEUTRON_LB"] = 'false'
-    config["CONFIG_HORIZON_NEUTRON_FW"] = 'false'
+    config["CONFIG_HORIZON_NEUTRON_LB"] = False
+    config["CONFIG_HORIZON_NEUTRON_FW"] = False
 
     if config['CONFIG_NEUTRON_INSTALL'] == 'y':
         if config["CONFIG_LBAAS_INSTALL"] == 'y':
-            config["CONFIG_HORIZON_NEUTRON_LB"] = 'true'
+            config["CONFIG_HORIZON_NEUTRON_LB"] = True
         if config["CONFIG_NEUTRON_FWAAS"] == 'y':
-            config["CONFIG_HORIZON_NEUTRON_FW"] = 'true'
+            config["CONFIG_HORIZON_NEUTRON_FW"] = True
 
     manifestdata = getManifestTemplate("horizon.pp")
     appendManifestFile(manifestfile, manifestdata)

packstack/plugins/glance_200.py

@@ -15,7 +15,8 @@
 
 from packstack.modules.shortcuts import get_mq
 from packstack.modules.ospluginutils import (getManifestTemplate,
-                                             appendManifestFile)
+                                             appendManifestFile,
+                                             createFirewallResources)
 
 #------------------ oVirt installer initialization ------------------
 
@@ -126,15 +127,15 @@ def create_manifest(config, messages):
         mq_template = get_mq(config, "glance_ceilometer")
         manifestdata += getManifestTemplate(mq_template)
 
-    manifestdata += getManifestTemplate(
-        'glance_%s.pp' % config['CONFIG_GLANCE_BACKEND'])
-
-    config['FIREWALL_SERVICE_NAME'] = "glance"
-    config['FIREWALL_PORTS'] = "'9292'"
-    config['FIREWALL_CHAIN'] = "INPUT"
-    config['FIREWALL_PROTOCOL'] = 'tcp'
-    config['FIREWALL_ALLOWED'] = "'ALL'"
-    config['FIREWALL_SERVICE_ID'] = "glance_API"
-    manifestdata += getManifestTemplate("firewall.pp")
-
+    fw_details = dict()
+    key = "glance_api"
+    fw_details.setdefault(key, {})
+    fw_details[key]['host'] = "ALL"
+    fw_details[key]['service_name'] = "glance"
+    fw_details[key]['chain'] = "INPUT"
+    fw_details[key]['ports'] = ['9292']
+    fw_details[key]['proto'] = "tcp"
+    config['FIREWALL_GLANCE_RULES'] = fw_details
+
+    manifestdata += createFirewallResources('FIREWALL_GLANCE_RULES')
     appendManifestFile(manifestfile, manifestdata)