merging with upstream

bin/flashbake

@@ -7,7 +7,7 @@ from optparse import OptionParser
 import os
 from os.path import join, dirname, exists, realpath, abspath
 
-VERSION='0.23.2'
+VERSION='0.25'
 
 def handlebadplugin(plugin_error):
     logging.debug('Plugin error, %s.' % plugin_error)
@@ -78,13 +78,17 @@ if __name__ == "__main__":
         logging.info("Running from source tree; adjusting path")
         sys.path.insert(0, realpath(dirname(source_tree_flashbake)))
         try:
+            import flashbake
+            import flashbake.git
             from flashbake.commit import commit, parsecontrol
             from flashbake.context import buildmessagefile
             from flashbake.plugins import PluginError, PLUGIN_ERRORS
         finally:
             del sys.path[0]
     else:
         logging.debug("Assuming path is correct")
+        import flashbake
+        import flashbake.git
         from flashbake.commit import commit, parsecontrol
         from flashbake.context import buildmessagefile
         from flashbake.plugins import PluginError, PLUGIN_ERRORS
@@ -144,6 +148,9 @@ if __name__ == "__main__":
                 message_file.close()
                 os.remove(msg_filename)            
             sys.exit(0)
+        except (flashbake.git.VCError, flashbake.ConfigError), error:
+            logging.error('Error: %s' % str(error))
+            sys.exit(1)
         except PluginError, error:
             handlebadplugin(error)
             sys.exit(1)
@@ -159,6 +166,9 @@ if __name__ == "__main__":
     try:
         (hot_files, control_config) = parsecontrol(project_dir, control_file, control_config, hot_files)
         commit(control_config, hot_files, quiet_period, options.dryrun)
+    except (flashbake.git.VCError, flashbake.ConfigError), error:
+        logging.error('Error: %s' % str(error))
+        sys.exit(1)
     except PluginError, error:
         handlebadplugin(error)
         sys.exit(1)

bin/test

@@ -20,9 +20,13 @@ if __name__ == "__main__":
     try:
         from flashbake.commit import commit, parsecontrol
         from flashbake.context import buildmessagefile
-        import test
+        import test.config
+        import test.files
     finally:
         del sys.path[0]
 
-    suite = unittest.TestLoader().loadTestsFromTestCase(test.ConfigTestCase)
+    # combine classes into single suite
+    config_suite = unittest.TestLoader().loadTestsFromTestCase(test.config.ConfigTestCase)
+    files_suite = unittest.TestLoader().loadTestsFromTestCase(test.files.FilesTestCase)
+    suite = unittest.TestSuite([config_suite, files_suite])
     unittest.TextTestRunner(verbosity=2).run(suite)

flashbake/__init__.py

@@ -9,8 +9,13 @@
 import commands
 import glob
 from types import *
+import flashbake.plugins
 from flashbake.plugins import PluginError, PLUGIN_ERRORS
 
+
+class ConfigError(Exception):
+    pass
+
 class ControlConfig:
     """ Accumulates options from a control file for use by the core modules as
         well as for the plugins.  Also handles boot strapping the configured
@@ -29,7 +34,11 @@ def __init__(self):
         self.prop_types['smtp_port'] = int
 
         self.plugin_names = list()
-        self.plugins = list()
+        self.msg_plugins = list()
+
+        self.file_plugins = list()
+
+        self.git_path = None
 
     def init(self):
         """ Do any property clean up, after parsing but before use """
@@ -41,12 +50,14 @@ def init(self):
 
         if len(self.plugin_names) == 0:
             logging.debug('No plugins configured, enabling the stock set.')
-            self.addplugins(['flashbake.plugins.timezone:TimeZone',
-                    'flashbake.plugins.weather:Weather',
-                    'flashbake.plugins.uptime:UpTime',
-                    'flashbake.plugins.feed:Feed'])
+            raise ConfigError('No plugins configured!')
 
-        self.initplugins()
+        for plugin_name in self.plugin_names:
+            plugin = self.initplugin(plugin_name)
+            if isinstance(plugin, flashbake.plugins.AbstractMessagePlugin):
+                self.msg_plugins.append(plugin)
+            if isinstance(plugin, flashbake.plugins.AbstractFilePlugin):
+                self.file_plugins.append(plugin)
 
     def sharedproperty(self, name, type = None):
         """ Declare a shared property, this way multiple plugins can share some
@@ -60,62 +71,61 @@ def sharedproperty(self, name, type = None):
             value = self.extra_props[name]
             del self.extra_props[name]
 
-        # TODO handle ValueError
-        # TODO handle bad type
-        if type != None:
-            value = type(value)
-        self.__dict__[name] = value
+            if type != None:
+                try:
+                    value = type(value)
+                except:
+                    raise ConfigError('Problem parsing %s for option %s'
+                            % (name, value))
+            self.__dict__[name] = value
 
     def addplugins(self, plugin_names):
-        # TODO use a comprehension to ensure uniqueness
-        self.plugin_names = self.plugin_names + plugin_names
-
-    def initplugins(self):
-        for plugin_name in self.plugin_names:
-            plugin = self.initplugin(plugin_name)
-            self.plugins.append(plugin)
+        # use a comprehension to ensure uniqueness
+        [self.__add_last(inbound_name) for inbound_name in plugin_names]
 
     def initplugin(self, plugin_spec):
         """ Initialize a plugin, including vetting that it meets the correct
             protocol; not private so it can be used in testing. """
-        if plugin_spec.find(':') > 0:
-            tokens = plugin_spec.split(':')
-            module_name = tokens[0]
-            plugin_name = tokens[1]
-        else:
-            module_name = plugin_spec
-            plugin_name = None
+        if plugin_spec.find(':') < 0:
+            logging.debug('Plugin spec not validly formed, %s.' % plugin_spec)
+            raise PluginError(PLUGIN_ERRORS.invalid_plugin, plugin_spec)
+
+        tokens = plugin_spec.split(':')
+        module_name = tokens[0]
+        plugin_name = tokens[1]
 
         try:
             __import__(module_name)
         except ImportError:
             logging.warn('Invalid module, %s' % plugin_name)
             raise PluginError(PLUGIN_ERRORS.unknown_plugin, plugin_spec)
 
-
-        if plugin_name == None:
-            plugin = sys.modules[module_name]
-            self.__checkattr(plugin_spec, plugin, 'connectable', bool)
-            self.__checkattr(plugin_spec, plugin, 'addcontext', FunctionType)
-
-            if 'init' in plugin.__dict__ and isinstance(plugin.__dict__['init'], FunctionType):
-                plugin.init(self)
-        else:
-            try:
-                # TODO re-visit pkg_resources, EntryPoint
-                plugin_class = self.__forname(module_name, plugin_name)
-                plugin = plugin_class(plugin_spec)
-            except:
-                logging.debug('Couldn\'t load class %s' % plugin_spec)
-                raise PluginError(PLUGIN_ERRORS.unknown_plugin, plugin_spec)
+        try:
+            # TODO re-visit pkg_resources, EntryPoint
+            plugin_class = self.__forname(module_name, plugin_name)
+            plugin = plugin_class(plugin_spec)
+        except:
+            logging.debug('Couldn\'t load class %s' % plugin_spec)
+            raise PluginError(PLUGIN_ERRORS.unknown_plugin, plugin_spec)
+        is_message_plugin = isinstance(plugin, flashbake.plugins.AbstractMessagePlugin)
+        is_file_plugin = isinstance(plugin, flashbake.plugins.AbstractFilePlugin)
+        if not is_message_plugin and not is_file_plugin:
+            raise PluginError(PLUGIN_ERRORS.invalid_type, plugin_spec)
+        if is_message_plugin:
             self.__checkattr(plugin_spec, plugin, 'connectable', bool)
             self.__checkattr(plugin_spec, plugin, 'addcontext', MethodType)
+        if is_file_plugin:
+            self.__checkattr(plugin_spec, plugin, 'processfiles', MethodType)
 
-            plugin.init(self)
-
+        plugin.init(self)
 
         return plugin
 
+    def __add_last(self, plugin_name):
+        if plugin_name in self.plugin_names:
+            self.plugin_names.remove(plugin_name) 
+        self.plugin_names.append(plugin_name)
+
     def __checkattr(self, plugin_spec, plugin, name, expected_type):
         try:
             attrib = eval('plugin.%s' % name)
@@ -140,46 +150,56 @@ class HotFiles:
     status and the dot-control file.
     """
     def __init__(self, project_dir):
-        self.project_dir = project_dir
+        self.project_dir = os.path.realpath(project_dir)
         self.linked_files = dict()
+        self.outside_files = set()
         self.control_files = set()
         self.not_exists = set()
         self.to_add = set()
 
     def addfile(self, filename):
+        to_expand = os.path.join(self.project_dir, filename)
         file_exists = False
         logging.debug('%s: %s'
-               % (filename,
-                  glob.glob(os.path.join(self.project_dir, filename))))
+               % (filename, glob.glob(to_expand)))
         if sys.hexversion < 0x2050000:
-            file_iter = glob.glob(os.path.join(self.project_dir, filename))
+            glob_iter = glob.glob(to_expand)
         else:
-            file_iter = glob.iglob(os.path.join(self.project_dir, filename))
+            glob_iter = glob.iglob(to_expand)
 
-        for real_file in file_iter:
+        for expanded_file in glob_iter:
+            # track whether iglob iterates at all, if it does not, then the line
+            # didn't expand to anything meaningful
             if not file_exists:
                 file_exists = True
-            link = self.checklink(real_file)
+
+            # skip the file if some previous glob hit it
+            if (expanded_file in self.outside_files
+                    or expanded_file in self.linked_files.keys()):
+                continue
+
+            # the commit code expects a relative path
+            rel_file = self.__make_rel(expanded_file)
+
+            # skip the file if some previous glob hit it
+            if rel_file in self.control_files:
+                continue
+
+            # checking this after removing the expanded project directory
+            # catches absolute paths to files outside the project directory
+            if rel_file == expanded_file:
+                self.outside_files.add(expanded_file)
+                continue
+
+            link = self.__check_link(expanded_file)
 
             if link == None:
-                self.control_files.add(real_file)
+                self.control_files.add(rel_file)
             else:
-                self.linked_files[real_file] = link
+                self.linked_files[expanded_file] = link
 
         if not file_exists:
-            self.not_exists.add(filename)
-
-    def checklink(self, filename):
-        # TODO: only check for links between the file and the project directory
-        if os.path.islink(filename):
-           return filename
-        directory = os.path.dirname(filename)
-
-        while (len(directory) > 0 and directory != '/'):
-            if os.path.islink(directory):
-                return directory
-            directory = os.path.dirname(directory)
-        return None
+            self.putabsent(filename)
 
     def contains(self, filename):
         return filename in self.control_files
@@ -193,35 +213,71 @@ def putabsent(self, filename):
     def putneedsadd(self, filename):
         self.to_add.add(filename)
 
-    def warnlinks(self):
+    def warnproblems(self):
         # print warnings for linked files
-        for (filename, link) in self.linked_files.iteritems():
+        for filename in self.linked_files.keys():
             logging.info('%s is a link or its directory path contains a link.' % filename)
-
-    def addorphans(self, control_config):
+        # print warnings for files outside the project
+        for filename in self.outside_files:
+            logging.info('%s is outside the project directory.' % filename)
+        # print warnings for files outside the project
+        for filename in self.not_exists:
+            logging.info('%s does not exist.' % filename)
+
+    def addorphans(self, git_obj, control_config):
         if len(self.to_add) == 0:
             return
 
         message_file = context.buildmessagefile(control_config)
 
-        add_template = 'git add "%s"'
-        git_commit = 'git commit -F %(msg_filename)s %(filenames)s'
-        file_template = ' "%s"'
-        to_commit = ''
+        to_commit = list()
         for orphan in self.to_add:
             logging.debug('Adding %s.' % orphan)
-            add_output = commands.getoutput(add_template % orphan)
-            to_commit += file_template % orphan
+            add_output = git_obj.add(orphan)
+            logging.debug('Add output, %s' % orphan)
+            to_commit.append(orphan)
 
         logging.info('Adding new files, %s.' % to_commit)
         # consolidate the commit to be friendly to how git normally works
-        git_commit = git_commit % {'msg_filename' : message_file, 'filenames' : to_commit}
-        logging.debug(git_commit)
         if not control_config.dryrun:
-            commit_output = commands.getoutput(git_commit)
-            logging.debug(commit_output)
+            commit_output = git_obj.commit(message_file, to_commit)
+            logging.debug('Commit output, %s' % commit_output)
 
         os.remove(message_file)
 
     def needsnotice(self):
-        return len(self.not_exists) > 0 or len(self.linked_files) > 0
+        return (len(self.not_exists) > 0
+               or len(self.linked_files) > 0
+               or len(self.outside_files) > 0) 
+
+    def __check_link(self, filename):
+        # add, above, makes sure filename is always relative
+        if os.path.islink(filename):
+           return filename
+        directory = os.path.dirname(filename)
+
+        while (len(directory) > 0):
+            # stop at the project directory, if it is in the path
+            if directory == self.project_dir:
+                break
+            # stop at root, as a safety check though it should not happen
+            if directory == os.sep:
+                break
+            if os.path.islink(directory):
+                return directory
+            directory = os.path.dirname(directory)
+        return None
+
+    def __make_rel(self, filepath):
+        return self.__drop_prefix(self.project_dir, filepath)
+
+    def __drop_prefix(self, prefix, filepath):
+        if not filepath.startswith(prefix):
+            return filepath
+
+        if not prefix.endswith(os.sep):
+            prefix += os.sep
+        if sys.hexversion < 0x2060000:
+            return filepath.replace(prefix, "")
+        else:
+            return os.path.relpath(filepath, prefix)