add support for redirecting output of cloud-init, cloud-config and cl…

…oud-final

ChangeLog

@@ -27,3 +27,5 @@
    this is to ensure they don't get run off the 'get-console-ouptut' buffer
  - user_scripts run via cloud-final and thus semaphore renamed from
    user_scripts to config_user_scripts
+ - add support for redirecting output of cloud-init, cloud-config, cloud-final
+   via the config file, or user data config file

cloud-init-cfg.py

@@ -35,14 +35,15 @@ def main():
     #   read cloud config jobs from config (builtin -> system)
     #   and run all in order
 
-    modlist = "cloud_config"
+    modename = "config"
+
     if len(sys.argv) < 2:
         Usage(sys.stderr)
         sys.exit(1)
     if sys.argv[1] == "all":
         name = "all"
         if len(sys.argv) > 2:
-            modlist = sys.argv[2]
+            modename = sys.argv[2]
     else:
         freq = None
         run_args = []
@@ -54,30 +55,37 @@ def main():
         if len(sys.argv) > 3:
             run_args=sys.argv[3:]
 
-    cloudinit.logging_set_from_cfg_file()
-    log = logging.getLogger()
-    log.info("cloud-init-cfg %s" % sys.argv[1:])
-
     cfg_path = cloudinit.get_ipath_cur("cloud_config")
     cfg_env_name = cloudinit.cfg_env_name
     if os.environ.has_key(cfg_env_name):
         cfg_path = os.environ[cfg_env_name]
 
     cc = CC.CloudConfig(cfg_path)
 
+    try:
+        (outfmt, errfmt) = CC.get_output_cfg(cc.cfg,modename)
+        CC.redirect_output(outfmt, errfmt)
+    except Exception, e:
+        err("Failed to get and set output config: %s\n" % e)
+
+    cloudinit.logging_set_from_cfg(cc.cfg)
+    log = logging.getLogger()
+    log.info("cloud-init-cfg %s" % sys.argv[1:])
+
     module_list = [ ]
     if name == "all":
-        modlist_cfg_name = "%s_modules" % modlist
+        modlist_cfg_name = "cloud_%s_modules" % modename
+        print modlist_cfg_name
         module_list = CC.read_cc_modules(cc.cfg,modlist_cfg_name)
         if not len(module_list):
-            err("no modules to run in cloud_config [%s]" % modlist,log)
+            err("no modules to run in cloud_config [%s]" % modename,log)
             sys.exit(0)
     else:
         module_list.append( [ name, freq ] + run_args )
 
     failures = CC.run_cc_modules(cc,module_list,log)
     if len(failures):
-        err("errors running cloud_config [%s]: %s" % (modlist,failures), log)
+        err("errors running cloud_config [%s]: %s" % (modename,failures), log)
     sys.exit(len(failures))
 
 def err(msg,log=None):

cloud-init.py

@@ -27,8 +27,8 @@
 import logging
 import errno
 
-def warn(str):
-    sys.stderr.write(str)
+def warn(wstr):
+    sys.stderr.write(wstr)
 
 def main():
     cmds = ( "start", "start-local" )
@@ -40,7 +40,7 @@ def main():
         sys.stderr.write("bad command %s. use one of %s\n" % (cmd, cmds))
         sys.exit(1)
 
-    now = time.strftime("%a, %d %b %Y %H:%M:%S %z")
+    now = time.strftime("%a, %d %b %Y %H:%M:%S %z",time.gmtime())
     try:
        uptimef=open("/proc/uptime")
        uptime=uptimef.read().split(" ")[0]
@@ -49,24 +49,31 @@ def main():
        warn("unable to open /proc/uptime\n")
        uptime = "na"
 
+    source_type = "all"
+    if cmd == "start-local":
+        source_type = "local"
+
+    try:
+        cfg = cloudinit.get_base_cfg()
+        (outfmt, errfmt) = CC.get_output_cfg(cfg,"init")
+        CC.redirect_output(outfmt, errfmt)
+    except Exception, e:
+        warn("Failed to get and set output config: %s\n" % e)
+
     msg = "cloud-init %s running: %s. up %s seconds" % (cmd, now, uptime)
     sys.stderr.write(msg + "\n")
     sys.stderr.flush()
 
-    source_type = "all"
-    if cmd == "start-local":
-        source_type = "local"
+    cloudinit.logging_set_from_cfg(cfg)
+    log = logging.getLogger()
+    log.info(msg)
 
     try:
         cloudinit.initfs()
     except Exception, e:
         warn("failed to initfs, likely bad things to come: %s\n" % str(e))
 
 
-    cloudinit.logging_set_from_cfg_file()
-    log = logging.getLogger()
-    log.info(msg)
-
     # cache is not instance specific, so it has to be purged
     # but we want 'start' to benefit from a cache if
     # a previous start-local populated one
@@ -104,6 +111,18 @@ def main():
 
     cfg_path = cloudinit.get_ipath_cur("cloud_config")
     cc = CC.CloudConfig(cfg_path, cloud)
+
+    # if the output config changed, update output and err
+    try:
+        outfmt_orig = outfmt
+        errfmt_orig = errfmt
+        (outfmt, errfmt) = CC.get_output_cfg(cc.cfg,"init")
+        if outfmt_orig != outfmt or errfmt_orig != errfmt:
+            warn("stdout, stderr changing to (%s,%s)" % (outfmt,errfmt))
+            CC.redirect_output(outfmt, errfmt)
+    except Exception, e:
+        warn("Failed to get and set output config: %s\n" % e)
+
     module_list = CC.read_cc_modules(cc.cfg,"cloud_init_modules")
 
     failures = []

cloudinit/CloudConfig/__init__.py

@@ -21,6 +21,8 @@
 import cloudinit.util as util
 import sys
 import traceback
+import os
+import subprocess
 
 per_instance="once-per-instance"
 per_always="always"
@@ -103,3 +105,110 @@ def run_cc_modules(cc,module_list,log):
             failures.append(name)
 
     return(failures)
+
+# always returns well formated values 
+# cfg is expected to have an entry 'output' in it, which is a dictionary
+# that includes entries for 'init', 'config', 'final' or 'all'
+#   init: /var/log/cloud.out
+#   config: [ ">> /var/log/cloud-config.out", /var/log/cloud-config.err ]
+#   final:
+#     output: "| logger -p"
+#     error: "> /dev/null"
+# this returns the specific 'mode' entry, cleanly formatted, with value
+# None if if none is given
+def get_output_cfg(cfg, mode="init"):
+    ret = [ None, None ]
+    if not 'output' in cfg: return ret
+
+    outcfg = cfg['output']
+    if mode in outcfg:
+        modecfg = outcfg[mode]
+    else:
+        if 'all' not in outcfg: return ret
+        # if there is a 'all' item in the output list
+        # then it applies to all users of this (init, config, final)
+        modecfg = outcfg['all']
+
+    # if value is a string, it specifies stdout
+    if isinstance(modecfg,str):
+        ret = [ modecfg, None ]
+
+    # if its a list, then we expect (stdout, stderr)
+    if isinstance(modecfg,list):
+        if len(modecfg) > 0: ret[0] = modecfg[0]
+        if len(modecfg) > 1:
+            ret[1] = modecfg[1]
+
+    # if it is a dictionary, expect 'out' and 'error'
+    # items, which indicate out and error
+    if isinstance(modecfg, dict):
+        if 'output' in modecfg:
+            ret[0] = modecfg['output']
+        if 'error' in modecfg:
+            ret[1] = modecfg['error']
+
+    # if err's entry == "&1", then make it same as stdout
+    # as in shell syntax of "echo foo >/dev/null 2>&1"
+    if ret[1] == "&1": ret[1] = ret[0]
+
+    swlist = [ ">>", ">", "|" ]
+    for i in range(len(ret)):
+        if not ret[i]: continue
+        val = ret[i].lstrip()
+        found = False
+        for s in swlist:
+            if val.startswith(s):
+                val = "%s %s" % (s,val[len(s):].strip())
+                found = True
+                break
+        if not found:
+            # default behavior is append
+            val = "%s %s" % ( ">>", val.strip())
+        ret[i] = val
+
+    return(ret)
+
+
+# redirect_output(outfmt, errfmt, orig_out, orig_err)
+#  replace orig_out and orig_err with filehandles specified in outfmt or errfmt
+#  fmt can be:
+#   > FILEPATH
+#   >> FILEPATH
+#   | program [ arg1 [ arg2 [ ... ] ] ]
+#
+#   with a '|', arguments are passed to shell, so one level of
+#   shell escape is required.  
+def redirect_output(outfmt,errfmt, o_out=sys.stdout, o_err=sys.stderr):
+    if outfmt:
+        (mode, arg) = outfmt.split(" ",1)
+        if mode == ">" or mode == ">>":
+            owith = "ab"
+            if mode == ">": owith = "wb"
+            new_fp = open(arg, owith)
+        elif mode == "|":
+            proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE)
+            new_fp = proc.stdin
+        else:
+            raise TypeError("invalid type for outfmt: %s" % outfmt)
+
+        if o_out:
+            os.dup2(new_fp.fileno(), o_out.fileno())
+        if errfmt == outfmt:
+            os.dup2(new_fp.fileno(), o_err.fileno())
+            return
+
+    if errfmt:
+        (mode, arg) = errfmt.split(" ",1)
+        if mode == ">" or mode == ">>":
+            owith = "ab"
+            if mode == ">": owith = "wb"
+            new_fp = open(arg, owith)
+        elif mode == "|":
+            proc = subprocess.Popen(arg, shell=True, stdin=subprocess.PIPE)
+            new_fp = proc.stdin
+        else:
+            raise TypeError("invalid type for outfmt: %s" % outfmt)
+
+        if o_err:
+            os.dup2(new_fp.fileno(), o_err.fileno())
+    return

cloudinit/__init__.py

@@ -73,7 +73,7 @@ def emit(self,record): pass
 def logging_set_from_cfg_file(cfg_file=system_config):
     logging_set_from_cfg(util.get_base_cfg(cfg_file,cfg_builtin,parsed_cfgs))
 
-def logging_set_from_cfg(cfg, logfile=None):
+def logging_set_from_cfg(cfg):
     log_cfgs = []
     logcfg=util.get_cfg_option_str(cfg, "log_cfg", False)
     if logcfg:
@@ -530,5 +530,8 @@ def get_ipath_cur(name=None):
 def get_cpath(name=None):
     return("%s%s" % (varlibdir, pathmap[name]))
 
+def get_base_cfg():
+    return(util.get_base_cfg(system_config,cfg_builtin,parsed_cfgs))
+
 class DataSourceNotFoundException(Exception):
     pass

doc/examples/cloud-config.txt

@@ -292,3 +292,21 @@ resize_rootfs: True
 # this message is written by cloud-final when the system is finished
 # its first boot
 final_message: "The system is finally up, after $UPTIME seconds"
+
+# configure where output will go
+# 'output' entry is a dict with 'init', 'config', 'final' or 'all'
+# entries.  Each one defines where 
+#  cloud-init, cloud-config, cloud-config-final or all output will go
+# each entry in the dict can be a string, list or dict.
+#  if it is a string, it refers to stdout
+#  if it is a list, entry 0 is stdout, entry 1 is stderr
+#  if it is a dict, it is expected to have 'output' and 'error' fields
+# default is to write to console only
+# the special entry "&1" for an error means "same location as stdout"
+#  (Note, that '&1' has meaning in yaml, so it must be quoted)
+output:
+ init: "> /var/log/my-cloud-init.log"
+ config: [ ">> /tmp/foo.out", "> /tmp/foo.err" ]
+ final:
+   output: "| tee /tmp/final.stdout | tee /tmp/bar.stdout"
+   error: "&1"

upstart/cloud-config.conf

@@ -5,4 +5,4 @@ start on (filesystem and started rsyslog)
 console output
 task
 
-exec cloud-init-cfg all cloud_config
+exec cloud-init-cfg all config

upstart/cloud-final.conf

@@ -7,4 +7,4 @@ start on (stopped rc RUNLEVEL=[2345] and stopped cloud-config)
 console output
 task
 
-exec cloud-init-cfg all cloud_final
+exec cloud-init-cfg all final