Merge plugins feature branch into main. (conda#11960)

* Add Plugin Mechanism (conda#11435)

Initial plugin framework implementation. Adds registration decorator `@conda.plugins.register` and hook `conda_subcommands` this allows simple subcommand registration:

```
import conda.plugins

@conda.plugins.register
def conda_subcommands():
    yield conda.plugins.CondaSubcommand(
        name="my-subcommand",
        summary="...",
        action=<function>,
    )
```

Co-authored-by: Filipe Lains <[email protected]>
Co-authored-by: Katherine Kinnaman <[email protected]>
Co-authored-by: Jannis Leidel <[email protected]>
Co-authored-by: Travis Hathaway <[email protected]>
Co-authored-by: Ken Odegard <[email protected]>
Co-authored-by: Bianca Henderson <[email protected]>
Co-authored-by: Daniel Holth <[email protected]>

conda/base/context.py

@@ -3,13 +3,15 @@
 
 from collections import OrderedDict
 
+import functools
 from errno import ENOENT
 from functools import lru_cache
 from logging import getLogger
 from typing import Optional
 import os
 from os.path import abspath, basename, expanduser, isdir, isfile, join, split as path_split
 import platform
+import pluggy
 import sys
 import struct
 from contextlib import contextmanager
@@ -61,6 +63,8 @@
 
 from .. import CONDA_SOURCE_ROOT
 
+from .. import plugins
+
 try:
     os.getcwd()
 except OSError as e:
@@ -150,6 +154,14 @@ def ssl_verify_validation(value):
     return True
 
 
+@functools.lru_cache(maxsize=None)  # FUTURE: Python 3.9+, replace w/ functools.cache
+def get_plugin_manager():
+    pm = pluggy.PluginManager('conda')
+    pm.add_hookspecs(plugins)
+    pm.load_setuptools_entrypoints('conda')
+    return pm
+
+
 class Context(Configuration):
 
     add_pip_as_python_dependency = ParameterLoader(PrimitiveParameter(True))
@@ -401,6 +413,9 @@ def __init__(self, search_path=None, argparse_args=None):
 
         super().__init__(search_path=search_path, app_name=APP_NAME, argparse_args=argparse_args)
 
+        # Add plugin support
+        self._plugin_manager = get_plugin_manager()
+
     def post_build_validation(self):
         errors = []
         if self.client_ssl_cert_key and not self.client_ssl_cert:

conda/cli/conda_argparse.py

@@ -22,9 +22,12 @@
 from .. import __version__
 from ..auxlib.ish import dals
 from ..auxlib.compat import isiterable
+from ..base import context
 from ..base.constants import COMPATIBLE_SHELLS, CONDA_HOMEPAGE_URL, DepsModifier, \
     UpdateModifier, SolverChoice
 from ..common.constants import NULL
+from ..common.io import dashlist
+from ..exceptions import PluginError
 
 log = getLogger(__name__)
 
@@ -113,6 +116,42 @@ def __init__(self, *args, **kwargs):
         if self.description:
             self.description += "\n\nOptions:\n"
 
+        pm = context.get_plugin_manager()
+        self._subcommands = sorted(
+            (
+                subcommand
+                for subcommands in pm.hook.conda_subcommands()
+                for subcommand in subcommands
+            ),
+            key=lambda subcommand: subcommand.name,
+        )
+
+        # Check for conflicts
+        seen = set()
+        conflicts = [
+            subcommand
+            for subcommand in self._subcommands
+            if subcommand.name in seen or seen.add(subcommand.name)
+        ]
+        if conflicts:
+            raise PluginError(
+                dals(
+                    f"""
+                    Conflicting entries found for the following subcommands:
+                    {dashlist(conflicts)}
+                    Multiple conda plugins are registering these subcommands via the
+                    `conda_subcommands` hook; please make sure that
+                    you do not have any incompatible plugins installed.
+                    """
+                )
+            )
+
+        if self._subcommands:
+            self.epilog = 'conda commands available from other packages:' + ''.join(
+                f'\n {subcommand.name} - {subcommand.summary}'
+                for subcommand in self._subcommands
+            )
+
     def _get_action_from_name(self, name):
         """Given a name, get the Action instance registered with this parser.
         If only it were made available in the ArgumentError object. It is
@@ -148,6 +187,18 @@ def error(self, message):
                         self.print_help()
                         sys.exit(0)
                     else:
+                        # Run the subcommand from plugins
+                        for subcommand in self._subcommands:
+                            if cmd == subcommand.name:
+                                sys.exit(subcommand.action(sys.argv[2:]))
+                        # Run the subcommand from executables; legacy path
+                        warnings.warn(
+                            (
+                                "Loading conda subcommands via executables is "
+                                "pending deprecation in favor of the plugin system. "
+                            ),
+                            PendingDeprecationWarning,
+                        )
                         executable = find_executable('conda-' + cmd)
                         if not executable:
                             from ..exceptions import CommandNotFoundError
@@ -166,7 +217,7 @@ def print_help(self):
             other_commands = find_commands()
             if other_commands:
                 builder = ['']
-                builder.append("conda commands available from other packages:")
+                builder.append("conda commands available from other packages (legacy):")
                 builder.extend('  %s' % cmd for cmd in sorted(other_commands))
                 print('\n'.join(builder))
 

conda/cli/main.py

@@ -31,23 +31,10 @@
 
     conda <command> -h
 """
+from .conda_argparse import generate_parser
 
 import sys
 
-PARSER = None
-
-
-def generate_parser():
-    # Generally using `global` is an anti-pattern.  But it's the lightest-weight way to memoize
-    # or do a singleton.  I'd normally use the `@memoize` decorator here, but I don't want
-    # to copy in the code or take the import hit.
-    global PARSER
-    if PARSER is not None:
-        return PARSER
-    from .conda_argparse import generate_parser
-    PARSER = generate_parser()
-    return PARSER
-
 
 def init_loggers(context=None):
     from logging import CRITICAL, getLogger

conda/exceptions.py

@@ -1039,6 +1039,10 @@ def __init__(self, msg, *args, **kwargs):
         super().__init__(msg, *args, **kwargs)
 
 
+class PluginError(CondaError):
+    pass
+
+
 def maybe_raise(error, context):
     if isinstance(error, CondaMultiError):
         groups = groupby(lambda e: isinstance(e, ClobberError), error.errors)

conda/plugins/__init__.py

@@ -0,0 +1,38 @@
+# Copyright (C) 2012 Anaconda, Inc
+# SPDX-License-Identifier: BSD-3-Clause
+from __future__ import annotations
+
+import pluggy
+
+from typing import Callable, NamedTuple
+from collections.abc import Iterable
+
+
+_hookspec = pluggy.HookspecMarker("conda")
+register = pluggy.HookimplMarker("conda")
+
+
+class CondaSubcommand(NamedTuple):
+    """
+    Conda subcommand entry.
+
+    :param name: Subcommand name (e.g., ``conda my-subcommand-name``).
+    :param summary: Subcommand summary, will be shown in ``conda --help``.
+    :param action: Callable that will be run when the subcommand is invoked.
+    """
+    name: str
+    summary: str
+    action: Callable[
+        [list[str]],  # arguments
+        int | None,  # return code
+    ]
+
+
+@_hookspec
+def conda_subcommands() -> Iterable[CondaSubcommand]:
+    """
+    Register external subcommands in conda.
+
+    :return: An iterable of subcommand entries.
+    """
+    ...

docs/requirements.txt

@@ -1,6 +1,8 @@
 linkify-it-py==1.0.1
 myst-parser==0.15.2
 Pillow==8.3.2
+pluggy==1.0.0
+pylint==2.11.1
 PyYAML==5.4.1
 requests==2.26.0
 ruamel.yaml==0.17.16
@@ -17,4 +19,3 @@ sphinxcontrib-plantuml==0.21
 sphinxcontrib-programoutput==0.17
 sphinxcontrib-qthelp==1.0.3
 sphinxcontrib-serializinghtml==1.1.5
-pylint==2.11.1

docs/source/conf.py

@@ -44,6 +44,7 @@
     "sphinxarg.ext",
     "sphinxcontrib.programoutput",
     "sphinx.ext.autodoc",
+    "sphinx.ext.autosectionlabel",
     "sphinx.ext.napoleon",
     "sphinx.ext.autosummary",
     "sphinx.ext.graphviz",

docs/source/dev-guide/index.rst

@@ -10,6 +10,7 @@ Developer guide
    writing-tests/index.rst
    releasing
    ../architecture
+   plugin-api/index
    deep-dive-install
    deep-dive-activation
    deep-dive-context

docs/source/dev-guide/plugin-api/index.rst

@@ -0,0 +1,132 @@
+=======
+Plugins
+=======
+
+As of version ``4.14.0``, ``conda`` has support for user plugins, enabling extension and/or
+alterations to some of its functionality.
+
+An overview of ``pluggy``
+-------------------------
+
+Plugins in ``conda`` are implemented with the use of Pluggy_, a Python framework used by
+other projects, such as ``pytest``, ``tox``, and ``devpi``. ``pluggy`` provides the ability to
+extend and modify the behavior of ``conda`` via function hooking, which results in plugin
+systems that are discoverable with the use of `Python package entrypoints`_.
+
+At its core, creating and implementing custom plugins with the use of ``pluggy`` can
+be broken down into two parts:
+
+1. Define the hooks you want to register
+2. Register your plugin under the ``conda`` entrypoint namespace
+
+If you would like more information about ``pluggy``, please refer to their documentation_
+for a full description of its features.
+
+
+Basic hook and entry point examples
+-----------------------------------
+
+Hook
+~~~~
+
+Below is an example of a very basic plugin "hook":
+
+.. code-block:: python
+   :caption: my_plugin.py
+
+   import conda.plugins
+
+
+   @conda.plugins.register
+   def conda_subcommands():
+       ...
+
+
+Packaging / entry point namespace
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The ``pyproject.toml`` file shown below is an example of a way to define and build
+a package out of the custom plugin hook shown above:
+
+.. code-block::
+   :caption: pyproject.toml
+
+   [build-system]
+   requires = ["setuptools", "setuptools-scm"]
+   build-backend = "setuptools.build_meta"
+
+   [project]
+   name = "my-conda-plugin"
+   version = "1.0.0"
+   description = "My conda plugin"
+   requires-python = ">=3.7"
+   dependencies = ["conda"]
+
+   [project.entry-points."conda"]
+   my-conda-plugin = "my_plugin"
+
+
+The ``setup.py`` file below is an alternative to the ``pyproject.toml`` file shown
+above; its main difference is the ``entry_points`` argument that is provided to the
+``setup()`` function:
+
+.. code-block:: python
+   :caption: setup.py
+
+   from setuptools import setup
+
+   setup(
+       name="my-conda-plugin",
+       install_requires="conda",
+       entry_points={"conda": ["my-conda-plugin = my_plugin"]},
+       py_modules=["my_plugin"],
+   )
+
+
+A note on licensing
+-------------------
+
+When licensing plugins, we recommend using licenses such as BSD-3_, MIT_, and
+`Apache License 2.0`_. Some ``import`` statements may possibly require the GPLv3_
+license, which ensures that the software being licensed is open source.
+
+Ultimately, the authors of the plugins can decide which license is best for their particular
+use case. Be sure to credit the original author of the plugin, and keep in mind that
+licenses can be altered depending on the situation.
+
+For more information on which license to use for your custom plugin, please reference
+the `"Choose an Open Source License"`_ site.
+
+
+Tutorials
+---------
+
+.. py:module:: conda.plugins
+
+
+.. toctree::
+   :maxdepth: 1
+
+   subcommand_guide
+
+
+API reference
+-------------
+
+.. py:module:: conda.plugins
+
+
+.. toctree::
+   :maxdepth: 1
+
+   subcommands
+
+
+.. _Pluggy: https://pluggy.readthedocs.io/en/stable/
+.. _documentation: https://pluggy.readthedocs.io/en/stable/
+.. _`Python package entrypoints`: https://packaging.python.org/en/latest/specifications/entry-points/
+.. _BSD-3: https://opensource.org/licenses/BSD-3-Clause
+.. _MIT: https://opensource.org/licenses/MIT
+.. _`Apache License 2.0`: https://www.apache.org/licenses/LICENSE-2.0
+.. _GPLv3: https://www.gnu.org/licenses/gpl-3.0.en.html
+.. _`"Choose an Open Source License"`: https://choosealicense.com/