Bug 1695312 - Add the ability for dispatch to ad-hoc load command m…

…odules that aren't already loaded r=firefox-build-system-reviewers,glandium

This is really just shuffling a bunch of things around. None of the
'load_*' member functions of the `Mach` class actually needed to be
member functions. They can all be static so that they can be used
anywhere. That combined with moving all the other 'mach_command' logic
to a different file, allows us to load the module for any command so
that we can successfully dispatch it.

Differential Revision: https://phabricator.services.mozilla.com/D184060

python/mach/docs/driver.rst

@@ -25,8 +25,8 @@ providers. e.g.:
 See http://pythonhosted.org/setuptools/setuptools.html#dynamic-discovery-of-services-and-plugins
 for more information on creating an entry point. To search for entry
 point plugins, you can call
-:py:meth:`mach.main.Mach.load_commands_from_entry_point`. e.g.:
+:py:meth:`mach.command_util.load_commands_from_entry_point`. e.g.:
 
 .. code-block:: python
 
-   mach.load_commands_from_entry_point("mach.external.providers")
+   load_commands_from_entry_point("mach.external.providers")

python/mach/mach/main.py

@@ -7,24 +7,19 @@
 
 import argparse
 import codecs
-import errno
-import imp
 import logging
 import os
 import sys
 import traceback
-import uuid
-from collections.abc import Iterable
 from pathlib import Path
-from typing import Dict, List, Optional, Union
+from typing import List, Optional
 
 from mach.site import CommandSiteManager
 
 from .base import (
     CommandContext,
     FailedCommandError,
     MachError,
-    MissingFileError,
     NoCommandError,
     UnknownCommandError,
     UnrecognizedArgumentError,
@@ -103,17 +98,6 @@
 The %s command does not accept the arguments: %s
 """.lstrip()
 
-INVALID_ENTRY_POINT = r"""
-Entry points should return a list of command providers or directories
-containing command providers. The following entry point is invalid:
-
-    %s
-
-You are seeing this because there is an error in an external module attempting
-to implement a mach command. Please fix the error, or uninstall the module from
-your system.
-""".lstrip()
-
 
 class ArgumentParser(argparse.ArgumentParser):
     """Custom implementation argument parser to make things look pretty."""
@@ -174,18 +158,6 @@ def __setattr__(self, key, value):
         setattr(object.__getattribute__(self, "_context"), key, value)
 
 
-class MachCommandReference:
-    """A reference to a mach command.
-
-    Holds the metadata for a mach command.
-    """
-
-    module: Path
-
-    def __init__(self, module: Union[str, Path]):
-        self.module = Path(module)
-
-
 class Mach(object):
     """Main mach driver type.
 
@@ -246,93 +218,6 @@ def __init__(
         self.log_manager.register_structured_logger(self.logger)
         self.populate_context_handler = None
 
-    def load_commands_from_directory(self, path: Path):
-        """Scan for mach commands from modules in a directory.
-
-        This takes a path to a directory, loads the .py files in it, and
-        registers and found mach command providers with this mach instance.
-        """
-        for f in sorted(path.iterdir()):
-            if not f.suffix == ".py" or f.name == "__init__.py":
-                continue
-
-            full_path = path / f
-            module_name = f"mach.commands.{str(f)[0:-3]}"
-
-            self.load_commands_from_file(full_path, module_name=module_name)
-
-    def load_commands_from_file(self, path: Union[str, Path], module_name=None):
-        """Scan for mach commands from a file.
-
-        This takes a path to a file and loads it as a Python module under the
-        module name specified. If no name is specified, a random one will be
-        chosen.
-        """
-        if module_name is None:
-            # Ensure parent module is present otherwise we'll (likely) get
-            # an error due to unknown parent.
-            if "mach.commands" not in sys.modules:
-                mod = imp.new_module("mach.commands")
-                sys.modules["mach.commands"] = mod
-
-            module_name = f"mach.commands.{uuid.uuid4().hex}"
-
-        try:
-            imp.load_source(module_name, str(path))
-        except IOError as e:
-            if e.errno != errno.ENOENT:
-                raise
-
-            raise MissingFileError(f"{path} does not exist")
-
-    def load_commands_from_spec(
-        self, spec: Dict[str, MachCommandReference], topsrcdir: str, missing_ok=False
-    ):
-        """Load mach commands based on the given spec.
-
-        Takes a dictionary mapping command names to their metadata.
-        """
-        modules = set(spec[command].module for command in spec)
-
-        for path in modules:
-            try:
-                self.load_commands_from_file(topsrcdir / path)
-            except MissingFileError:
-                if not missing_ok:
-                    raise
-
-    def load_commands_from_entry_point(self, group="mach.providers"):
-        """Scan installed packages for mach command provider entry points. An
-        entry point is a function that returns a list of paths to files or
-        directories containing command providers.
-
-        This takes an optional group argument which specifies the entry point
-        group to use. If not specified, it defaults to 'mach.providers'.
-        """
-        try:
-            import pkg_resources
-        except ImportError:
-            print(
-                "Could not find setuptools, ignoring command entry points",
-                file=sys.stderr,
-            )
-            return
-
-        for entry in pkg_resources.iter_entry_points(group=group, name=None):
-            paths = entry.load()()
-            if not isinstance(paths, Iterable):
-                print(INVALID_ENTRY_POINT % entry)
-                sys.exit(1)
-
-            for path in paths:
-                path = Path(path)
-                if path.is_file():
-                    self.load_commands_from_file(path)
-                elif path.is_dir():
-                    self.load_commands_from_directory(path)
-                else:
-                    print(f"command provider '{path}' does not exist")
-
     def define_category(self, name, title, description, priority=50):
         """Provide a description for a named command category."""
 
@@ -648,7 +533,7 @@ def find_in_dir(base: Path):
         self.settings.load_files(list(files))
 
 
-def get_argument_parser(context=None, action=CommandAction):
+def get_argument_parser(context=None, action=CommandAction, topsrcdir=None):
     """Returns an argument parser for the command-line interface."""
 
     parser = ArgumentParser(
@@ -740,6 +625,6 @@ def get_argument_parser(context=None, action=CommandAction):
             "command", action=CommandAction, registrar=Registrar, context=context
         )
     else:
-        parser.add_argument("command", action=action)
+        parser.add_argument("command", topsrcdir=topsrcdir, action=action)
 
     return parser

python/mach/mach/registrar.py

@@ -164,7 +164,17 @@ def dispatch(self, name, context, argv=None, subcommand=None, **kwargs):
 
         Commands can use this to call other commands.
         """
-        handler = self.command_handlers[name]
+        from mach.command_util import load_command_module_from_command_name
+
+        handler = self.command_handlers.get(name)
+
+        if not handler:
+            load_command_module_from_command_name(name, context.topdir)
+            handler = self.command_handlers.get(name)
+            if not handler:
+                raise MachError(
+                    f"Mach was not able to load the module for the '{name}' command."
+                )
 
         if subcommand:
             handler = handler.subcommand_handlers[subcommand]

python/mach/mach/test/conftest.py

@@ -17,6 +17,7 @@
     # TODO io.StringIO causes failures with Python 2 (needs to be sorted out)
     from io import StringIO
 
+from mach.command_util import load_commands_from_entry_point, load_commands_from_file
 from mach.main import Mach
 
 PROVIDER_DIR = Path(__file__).resolve().parent / "providers"
@@ -43,10 +44,10 @@ def inner(
                 provider_files = [provider_files]
 
             for path in provider_files:
-                m.load_commands_from_file(PROVIDER_DIR / path)
+                load_commands_from_file(PROVIDER_DIR / path)
 
         if entry_point:
-            m.load_commands_from_entry_point(entry_point)
+            load_commands_from_entry_point(entry_point)
 
         return m
 

python/mach/mach/test/test_conditions.py

@@ -8,6 +8,7 @@
 from mozunit import main
 
 from mach.base import MachError
+from mach.command_util import load_commands_from_file
 from mach.main import Mach
 from mach.registrar import Registrar
 from mach.test.conftest import PROVIDER_DIR, TestBase
@@ -81,7 +82,7 @@ def test_invalid_type(self):
         m.define_category("testing", "Mach unittest", "Testing for mach core", 10)
         self.assertRaises(
             MachError,
-            m.load_commands_from_file,
+            load_commands_from_file,
             PROVIDER_DIR / "conditions_invalid.py",
         )
 

testing/tools/mach_test_package_initialize.py

@@ -165,7 +165,7 @@ def bootstrap(test_package_root):
 
     sys.path[0:0] = [os.path.join(test_package_root, path) for path in SEARCH_PATHS]
     import mach.main
-    from mach.main import MachCommandReference
+    from mach.command_util import MachCommandReference, load_commands_from_spec
 
     # Centralized registry of available mach commands
     MACH_COMMANDS = {
@@ -244,6 +244,6 @@ def populate_context(context, key=None):
 
     # Depending on which test zips were extracted,
     # the command module might not exist
-    mach.load_commands_from_spec(MACH_COMMANDS, test_package_root, missing_ok=True)
+    load_commands_from_spec(MACH_COMMANDS, test_package_root, missing_ok=True)
 
     return mach