saas.py

#!/usr/bin/env python

ODOO_VERSION = 9
SUPERUSER_ID = 1

import ConfigParser
import argparse
import contextlib
import fcntl
import json
import os
import psycopg2
import requests
import resource
import signal
import subprocess
import traceback
import xmlrpclib


def log(*args):
    print ''
    print 'saas.py >>> ' + ', '.join([str(a) for a in args])

# ----------------------------------------------------------
# Options
# ----------------------------------------------------------

parser = argparse.ArgumentParser(description='''Control script to manage saas system.
It\'s assumed, that you have configured webserver (e.g. nginx). Check docs/port_80.rst for details.
''',
                                 formatter_class=argparse.RawTextHelpFormatter,
                                 epilog='''
------------------------
Local usage:

   python saas.py
   sudo bash -c "python saas.py --print-local-hosts >> /etc/hosts"
''')

settings_group = parser.add_argument_group('Common settings')
settings_group.add_argument('--suffix', dest='suffix', default=ODOO_VERSION, help='suffix for names')
settings_group.add_argument('--odoo-script', dest='odoo_script', help='Path to openerp-server', default='./openerp-server')
settings_group.add_argument('--odoo-config', dest='odoo_config', help='Path to odoo configuration file')
settings_group.add_argument('--odoo-data-dir', dest='odoo_data_dir', help='Path to odoo data dir', default=None)
settings_group.add_argument('--odoo-xmlrpc-port', dest='xmlrpc_port', default=None)
settings_group.add_argument('--admin-password', dest='admin_password', help='Password for admin user. It\'s used for all databases.', default='admin')
#settings_group.add_argument('--db_user', dest='db_user', help='database user name')
settings_group.add_argument('-s', '--simulate', dest='simulate', action='store_true', help='Don\'t make actual changes. Just show what script is going to do.')


portal_group = parser.add_argument_group('Portal creation')
portal_group.add_argument('--portal-create', dest='portal_create', help='Create SaaS Portal database', action='store_true')
portal_group.add_argument('--portal-db-name', dest='portal_db_name', default='saas-portal-{suffix}.local')

server_group = parser.add_argument_group('Server creation')
server_group.add_argument('--server-create', dest='server_create', help='Create SaaS Server database', action='store_true')
server_group.add_argument('--server-db-name', dest='server_db_name', default='server-1.saas-portal-{suffix}.local')

plan_group = parser.add_argument_group('Plan creation')
plan_group.add_argument('--plan-create', dest='plan_create', help='Create Plan', action='store_true')
plan_group.add_argument('--plan-name', dest='plan_name', default='Plan')
plan_group.add_argument('--plan-template-db-name', dest='plan_template_db_name', default='template-1.saas-portal-{suffix}.local')
plan_group.add_argument('--plan-clients', dest='plan_clients', default='client-%i.saas-portal-{suffix}.local', help='Template for new client databases')

other_group = parser.add_argument_group('Other')
other_group.add_argument('--print-local-hosts', dest='print_local_hosts', action='store_true', help='Print hosts rules for local usage.')
other_group.add_argument('--run', dest='run', action='store_true', help='Run server')
other_group.add_argument('--cleanup', dest='cleanup', action='store_true', help='Drop all saas databases. Use along with --simulate to check which database would be deleted')

args = vars(parser.parse_args())

# format vars
suffix = args['suffix']
for a in args:
    if type(args[a]) == str:
        args[a] = args[a].format(suffix=suffix)


def get_odoo_config():
    res = {}
    if not args.get('odoo_config'):
        return res
    p = ConfigParser.ConfigParser()
    log('Read odoo config', args.get('odoo_config'))
    p.read(args.get('odoo_config'))
    for (name, value) in p.items('options'):
        if value == 'True' or value == 'true':
            value = True
        if value == 'False' or value == 'false':
            value = False
        res[name] = value
    return res

odoo_config = get_odoo_config()

datadir = args.get('odoo_data_dir') or odoo_config.get('data_dir')
xmlrpc_port = args.get('xmlrpc_port') or odoo_config.get('xmlrpc_port') or '8069'


# ----------------------------------------------------------
# Main
# ----------------------------------------------------------
def main():
    if args.get('print_local_hosts'):
        host_line = '127.0.0.1 %s'
        print ''
        print '# generated by odoo-saas-tools'
        for host in ['portal_db_name', 'server_db_name', 'plan_template_db_name']:
            print host_line % args.get(host)
        for i in range(1, 11):
            print host_line % (args.get('plan_clients').replace('%i', '%03i' % i))
        return

    if args.get('simulate'):
        log('SIMULATION MODE')

    if args.get('cleanup'):
        cleanup()

    # create databases
    if args.get('portal_create'):
        createdb(args.get('portal_db_name'), ['auth_signup', 'saas_portal', 'saas_portal_start', 'saas_portal_sale_online'])

    if args.get('server_create'):
        createdb(args.get('server_db_name'), ['saas_server'])

    # run odoo to make updates via rpc
    cmd = get_cmd()
    pid = spawn_cmd(cmd)
    error = None
    try:
        wait_net_service('127.0.0.1', int(xmlrpc_port), 10)

        if args.get('portal_create'):
            rpc_init_db(args.get('portal_db_name'), new_admin_password=args.get('admin_password'))
            rpc_init_portal(args.get('portal_db_name'))

        if args.get('server_create'):
            rpc_init_db(args.get('server_db_name'), new_admin_password=args.get('admin_password'))
            rpc_init_server(args.get('server_db_name'))
            rpc_add_server_to_portal(args.get('portal_db_name'))

        if args.get('plan_create'):
            rpc_create_plan(args.get('portal_db_name'))

    except Exception, e:
        error = e
        traceback.print_exc()
    kill(pid)

    if error:
        return

    if args.get('run'):
        if args.get('portal_create'):
            print '\n\n\n\n\n     ------ ====== THE SAAS SYSTEM IS READY ===== -----     \n\n\n\n\n'

        cmd = get_cmd()
        exec_cmd(cmd)


# ----------------------------------------------------------
# Tools
# ----------------------------------------------------------
def createdb(dbname, install_modules=['base'], without_demo=True):
    pg_dropdb(dbname)
    pg_createdb(dbname, without_demo=without_demo)

    cmd = get_cmd()
    cmd += ['-d', dbname]
    cmd += ['-i', ','.join(install_modules)]
    if without_demo:
        cmd += ['--without-demo=all']

    cmd += ['--stop-after-init']
    exec_cmd(cmd)


def dropdb(dbname):
    pg_dropdb(dbname)
    # cleanup filestore
    #paths = [os.path.join(datadir, pn, 'filestore', dbname) for pn in 'OpenERP Odoo'.split()]
    paths = [os.path.join(datadir, 'filestore', dbname)]
    exec_cmd(['rm', '-rf'] + paths)


def cleanup():
    for dbname in find_databases(args.get('portal_db_name')):
        dropdb(dbname)


# ----------------------------------------------------------
# RPC Tools
# ----------------------------------------------------------
def rpc_auth(dbname, admin_username='admin', admin_password='admin'):
    main_url = 'http://%s' % dbname
    if args.get('simulate'):
        return None, None, None, None

    # Authenticate
    common = xmlrpclib.ServerProxy('{}/xmlrpc/2/common'.format(main_url))
    admin_uid = common.authenticate(dbname, admin_username, admin_password, {})
    models = xmlrpclib.ServerProxy('{}/xmlrpc/2/object'.format(main_url))

    return dbname, models, admin_uid, admin_password


def rpc_execute_kw(auth, model, method, rpc_args=[], rpc_kwargs={}):
    dbname, models, admin_uid, admin_password = auth
    log('RPC Execute', model, method, rpc_args, rpc_kwargs)
    if args.get('simulate'):
        return
    return models.execute_kw(dbname, admin_uid, admin_password,
                             model, method, rpc_args, rpc_kwargs)


def rpc_init_db(dbname, new_admin_password=None):
    if new_admin_password:
        auth = rpc_auth(dbname)
        rpc_execute_kw(auth, 'res.users', 'write', [[SUPERUSER_ID], {
            'password': new_admin_password,
        }])


def rpc_init_portal(dbname):
    # * open Settings/Configuration/SaaS Portal Settings
    #   * set *Base SaaS domain*, e.g. **odoo.local**
    #   * click Apply (do it even if you didn't make changes)
    auth = rpc_auth(dbname, admin_password=args.get('admin_password'))
    rpc_execute_kw(auth, 'ir.config_parameter', 'set_param', ['saas_portal.base_saas_domain', dbname])

    # Allow external users to sign up
    rpc_execute_kw(auth, 'ir.config_parameter', 'set_param', ['auth_signup.allow_uninvited', repr(True)])


def rpc_init_server(server_db_name, new_admin_password=None):
    # Update OAuth Provider urls
    auth = rpc_auth(server_db_name, admin_password=args.get('admin_password'))
    portal_db_name = args.get('portal_db_name')
    oauth_provider = rpc_xmlid_to_object(auth, 'saas_server.saas_oauth_provider', 'auth.oauth.provider')
    vals = {
        'auth_endpoint': oauth_provider.get('auth_endpoint').replace('odoo.local', portal_db_name),
        'validation_endpoint': oauth_provider.get('validation_endpoint').replace('odoo.local', portal_db_name),
    }
    oauth_provider = rpc_execute_kw(auth, 'auth.oauth.provider', 'write', [[oauth_provider.get('id')], vals])


def rpc_add_server_to_portal(portal_db_name):
    # 5. Register Server Database in Main Database
    #    * open SaaS/SaaS/Servers
    #      * click [Create]
    #      * set Database Name, e.g. **s1.odoo.local**
    #      * fix autogenerated Database UUID to actual one (see previous section)
    #      * click [Save]
    auth = rpc_auth(portal_db_name, admin_password=args.get('admin_password'))
    server_db_name = args.get('server_db_name')
    uuid = rpc_get_uuid(server_db_name)
    rpc_execute_kw(auth, 'saas_portal.server', 'create', [{'name': server_db_name, 'client_id': uuid}])


def rpc_get_uuid(dbname):
    auth = rpc_auth(dbname, admin_password=args.get('admin_password'))
    res = rpc_execute_kw(auth, 'ir.config_parameter', 'get_param', ['database.uuid'])
    return res


def rpc_xmlid_to_object(auth, xmlid, model):
    res_id = rpc_execute_kw(auth, 'ir.model.data', 'xmlid_to_res_id', [xmlid])
    return rpc_execute_kw(auth, model, 'read', [res_id])


def rpc_create_plan(portal_db_name):
    plan_name = args.get('plan_name')
    plan_template_db_name = args.get('plan_template_db_name')
    plan_clients = args.get('plan_clients')

    auth = rpc_auth(portal_db_name, admin_password=args.get('admin_password'))

    # 6. Create Plan
    #    * open Saas/SaaS/Plans
    #      * click [Create]
    #      * set Plan's name, e.g. "POS + ECommerce"
    #      * set SaaS Server
    #      * set Template DB: type name, e.g. **t1.odoo.local**, and click *Create "__t1.odoo.local__"*
    #      * click [Save]

    res = rpc_execute_kw(auth, 'saas_portal.server', 'search', [[]])
    # use last created server
    log('search server', res)
    server_id = res[0]

    template_id = rpc_execute_kw(auth, 'saas_portal.database', 'create', [{'name': plan_template_db_name}])

    plan_id = rpc_execute_kw(auth, 'saas_portal.plan', 'create', [{'name': plan_name, 'server_id': server_id, 'template_id': template_id, 'dbname_template': plan_clients}])

    #      * click [Create Template DB].
    #      * wait couple minutes while Database is being created.
    dropdb(plan_template_db_name)
    rpc_execute_kw(auth, 'saas_portal.plan', 'create_template', [[plan_id]])


# some functions below were taken from runbot module: https://github.com/odoo/odoo-extra/tree/master/runbot
# ----------------------------------------------------------
# DB Tools
# ----------------------------------------------------------
def pg_createdb(dbname, without_demo=True):
    log('Creating empty database %s' % dbname)
    if args.get('simulate'):
        return
    with local_pgadmin_cursor() as local_cr:
        local_cr.execute("""CREATE DATABASE "%s" TEMPLATE template0 LC_COLLATE 'C' ENCODING 'unicode'""" % dbname)


def pg_dropdb(dbname):
    log('Dropping  database %s' % dbname)
    if args.get('simulate'):
        return
    with local_pgadmin_cursor() as local_cr:
        local_cr.execute('DROP DATABASE IF EXISTS "%s"' % dbname)


def find_databases(root_database):
    with local_pgadmin_cursor() as local_cr:
        local_cr.execute("SELECT datname FROM pg_database WHERE  datname ilike '%%.{root}' OR datname='{root}'".format(root=root_database))
        res = local_cr.fetchall()
    return [row[0] for row in res]


@contextlib.contextmanager
def local_pgadmin_cursor():
    cnx = None
    try:
        cnx = psycopg2.connect("dbname=postgres")
        cnx.autocommit = True  # required for admin commands
        yield cnx.cursor()
    finally:
        if cnx:
            cnx.close()


# ----------------------------------------------------------
# OS Tools
# ----------------------------------------------------------
def get_cmd():
    cmd = [
        args.get('odoo_script'),
        "--xmlrpc-port=%s" % xmlrpc_port,
    ]
    if args.get('odoo_config'):
        cmd += ['--config=%s' % args.get('odoo_config')]

    return cmd


def exec_cmd(cmd):
    log('EXEC: ', ' '.join(cmd))
    if args.get('simulate'):
        return
    os.system(' '.join(cmd))


def spawn_cmd(cmd, cpu_limit=None, shell=False):
    log('Spawn',  ' '.join(cmd))
    if args.get('simulate'):
        return

    def preexec_fn():
        os.setsid()
        if cpu_limit:
            # set soft cpulimit
            soft, hard = resource.getrlimit(resource.RLIMIT_CPU)
            r = resource.getrusage(resource.RUSAGE_SELF)
            cpu_time = r.ru_utime + r.ru_stime
            resource.setrlimit(resource.RLIMIT_CPU, (cpu_time + cpu_limit, hard))
        # close parent files
        os.closerange(3, os.sysconf("SC_OPEN_MAX"))
        #lock(lock_path)
    #out=open(log_path,"w")
    #_logger.debug("spawn: %s stdout: %s", ' '.join(cmd), log_path)
    p = subprocess.Popen(cmd,
                         #stdout=out,
                         #stderr=out,
                         preexec_fn=preexec_fn,
                         shell=shell)
    log('Spawn pid: %s' % p.pid)
    return p.pid


def kill(pid):
    log('KILL', pid)
    if args.get('simulate'):
        return
    try:
        os.killpg(pid, signal.SIGKILL)
    except OSError:
        pass


# http://code.activestate.com/recipes/576655-wait-for-network-service-to-appear/
def wait_net_service(server, port, timeout=None):
    """ Wait for network service to appear
        @param timeout: in seconds, if None or 0 wait forever
        @return: True of False, if timeout is None may return only True or
                 throw unhandled network exception
    """
    log('Waiting for port', server, port)

    if args.get('simulate'):
        return

    import socket

    s = socket.socket()
    if timeout:
        from time import time as now
        # time module is needed to calc timeout shared between two exceptions
        end = now() + timeout

    while True:
        try:
            if timeout:
                next_timeout = end - now()
                if next_timeout < 0:
                    return False
                else:
                    s.settimeout(next_timeout)

            s.connect((server, port))

        except socket.timeout, err:
            # this exception occurs only if timeout is set
            if timeout:
                log('Port timeout')
                return False

        except socket.error, err:
            pass
        else:
            s.close()
            return True


if __name__ == "__main__":
    main()