Bug 1808732 - Move various dependencies out of the mach site to eit…

…her the specific site they're needed in (like `build, or `lint`), and/or to `common` r=firefox-build-system-reviewers,glandium

Now that we selectively load command modules, and activate the command
virtualenv much earlier in the mach process, a lot of the module
dependencies specified in `mach.txt` are no longer necessary there. With
their removal from `mach.txt` they will no longer be automatically
inherited by every site, which reduces potential dependency conflicts
for specific sites.

The `common` site still effectively has the same set of dependencies.
This is the default site that all commands use unless otherwise
specified. Most commands use this site, and going through every command
and seeing if a dependency is or isn't needed, then deciding if or if
not to create a new site for that command made sense was too time
consuming to do here.

Essentially the idea here going forward is that if you're trying to
add/update a new dependency to a command that is currently defaulting to
the `common` site and there is a conflict with one of the dependencies
in `common` you can move your command to a new site specifically for
your command, and you will have the minimal possible set of dependencies
a mach command can have, improving the odds that you can add the
module(s) you need for your command.

Depends on D180500

Differential Revision: https://phabricator.services.mozilla.com/D180501

python/mozbuild/mozbuild/artifact_commands.py

@@ -136,7 +136,11 @@ def _make_artifacts(
     return artifacts
 
 
-@ArtifactSubCommand("artifact", "install", "Install a good pre-built artifact.")
+@ArtifactSubCommand(
+    "artifact",
+    "install",
+    "Install a good pre-built artifact.",
+)
 @CommandArgument(
     "source",
     metavar="SRC",
@@ -204,7 +208,10 @@ def artifact_clear_cache(command_context, tree=None, job=None, verbose=False):
     return 0
 
 
-@SubCommand("artifact", "toolchain")
+@SubCommand(
+    "artifact",
+    "toolchain",
+)
 @CommandArgument("--verbose", "-v", action="store_true", help="Print verbose output.")
 @CommandArgument(
     "--cache-dir",

python/mozbuild/mozbuild/mach_commands.py

@@ -28,7 +28,6 @@
     SettingsProvider,
     SubCommand,
 )
-from voluptuous import All, Boolean, Required, Schema
 
 import mozbuild.settings  # noqa need @SettingsProvider hook to execute
 from mozbuild.base import (
@@ -112,6 +111,8 @@ def watch(command_context, verbose=False):
 
 
 def _cargo_config_yaml_schema():
+    from voluptuous import All, Boolean, Required, Schema
+
     def starts_with_cargo(s):
         if s.startswith("cargo-"):
             return s

python/sites/build.txt

@@ -1 +1,58 @@
 pth:third_party/python/vsdownload
+vendored:testing/web-platform/tests/tools/third_party/h2
+vendored:testing/web-platform/tests/tools/third_party/hpack
+vendored:testing/web-platform/tests/tools/third_party/html5lib
+vendored:testing/web-platform/tests/tools/third_party/hyperframe
+vendored:testing/web-platform/tests/tools/third_party/pywebsocket3
+vendored:testing/web-platform/tests/tools/third_party/webencodings
+vendored:testing/web-platform/tests/tools/wptrunner
+vendored:testing/web-platform/tests/tools/wptserve
+vendored:third_party/python/Jinja2
+vendored:third_party/python/PyYAML/lib/
+vendored:third_party/python/aiohttp
+vendored:third_party/python/appdirs
+vendored:third_party/python/arrow
+vendored:third_party/python/async_timeout
+vendored:third_party/python/binaryornot
+vendored:third_party/python/chardet
+vendored:third_party/python/compare_locales
+vendored:third_party/python/cookiecutter
+vendored:third_party/python/diskcache
+vendored:third_party/python/dlmanager
+vendored:third_party/python/ecdsa
+vendored:third_party/python/fluent.migrate
+vendored:third_party/python/fluent.syntax
+vendored:third_party/python/giturlparse
+vendored:third_party/python/glean_parser
+vendored:third_party/python/gyp/pylib
+vendored:third_party/python/importlib_metadata
+vendored:third_party/python/importlib_resources
+vendored:third_party/python/jinja2_time
+vendored:third_party/python/json-e
+vendored:third_party/python/jsonschema
+vendored:third_party/python/mohawk
+vendored:third_party/python/mozilla_repo_urls
+vendored:third_party/python/multidict
+vendored:third_party/python/pathspec
+vendored:third_party/python/pkgutil_resolve_name
+vendored:third_party/python/ply
+vendored:third_party/python/pyasn1
+vendored:third_party/python/pyasn1_modules
+vendored:third_party/python/pylru
+vendored:third_party/python/pyrsistent
+vendored:third_party/python/python_dateutil
+vendored:third_party/python/python_slugify
+vendored:third_party/python/redo
+vendored:third_party/python/requests_unixsocket
+vendored:third_party/python/rsa
+vendored:third_party/python/slugid
+vendored:third_party/python/taskcluster
+vendored:third_party/python/taskcluster_taskgraph
+vendored:third_party/python/taskcluster_urls
+vendored:third_party/python/text_unidecode
+vendored:third_party/python/toml
+vendored:third_party/python/typing_extensions
+vendored:third_party/python/voluptuous
+vendored:third_party/python/yamllint
+vendored:third_party/python/yarl
+vendored:third_party/python/zipp

python/sites/common.txt

@@ -1 +1,63 @@
-
+vendored:testing/web-platform/tests/tools/third_party/h2
+vendored:testing/web-platform/tests/tools/third_party/hpack
+vendored:testing/web-platform/tests/tools/third_party/html5lib
+vendored:testing/web-platform/tests/tools/third_party/hyperframe
+vendored:testing/web-platform/tests/tools/third_party/pywebsocket3
+vendored:testing/web-platform/tests/tools/third_party/webencodings
+vendored:testing/web-platform/tests/tools/wptrunner
+vendored:testing/web-platform/tests/tools/wptserve
+vendored:third_party/python/MarkupSafe/src
+vendored:third_party/python/PyYAML/lib/
+vendored:third_party/python/aiohttp
+vendored:third_party/python/appdirs
+vendored:third_party/python/arrow
+vendored:third_party/python/async_timeout
+vendored:third_party/python/binaryornot
+vendored:third_party/python/cbor2
+vendored:third_party/python/chardet
+vendored:third_party/python/compare_locales
+vendored:third_party/python/cookiecutter
+vendored:third_party/python/cookies
+vendored:third_party/python/cram
+vendored:third_party/python/diskcache
+vendored:third_party/python/dlmanager
+vendored:third_party/python/ecdsa
+vendored:third_party/python/esprima
+vendored:third_party/python/fluent.migrate
+vendored:third_party/python/fluent.syntax
+vendored:third_party/python/giturlparse
+vendored:third_party/python/glean_parser
+vendored:third_party/python/gyp/pylib
+vendored:third_party/python/importlib_metadata
+vendored:third_party/python/importlib_resources
+vendored:third_party/python/jinja2_time
+vendored:third_party/python/json-e
+vendored:third_party/python/jsonschema
+vendored:third_party/python/mohawk
+vendored:third_party/python/mozilla_repo_urls
+vendored:third_party/python/multidict
+vendored:third_party/python/pathspec
+vendored:third_party/python/pkgutil_resolve_name
+vendored:third_party/python/ply
+vendored:third_party/python/pyasn1
+vendored:third_party/python/pyasn1_modules
+vendored:third_party/python/pylru
+vendored:third_party/python/pyparsing
+vendored:third_party/python/pyrsistent
+vendored:third_party/python/python_dateutil
+vendored:third_party/python/python_slugify
+vendored:third_party/python/redo
+vendored:third_party/python/requests_unixsocket
+vendored:third_party/python/responses
+vendored:third_party/python/rsa
+vendored:third_party/python/slugid
+vendored:third_party/python/taskcluster
+vendored:third_party/python/taskcluster_taskgraph
+vendored:third_party/python/taskcluster_urls
+vendored:third_party/python/text_unidecode
+vendored:third_party/python/toml
+vendored:third_party/python/typing_extensions
+vendored:third_party/python/voluptuous
+vendored:third_party/python/yamllint
+vendored:third_party/python/yarl
+vendored:third_party/python/zipp

python/sites/docs.txt

@@ -1,44 +1,37 @@
 pth:tools/lint/eslint/
-pypi:alabaster==0.7.13
 pypi:Babel==2.12.1
+pypi:Markdown==3.4.3
+pypi:Pygments==2.14.0
+pypi:Sphinx==5.1.1
+pypi:alabaster==0.7.13
 pypi:backports-abc==0.5
 pypi:boto3==1.26.80
 pypi:botocore==1.29.80
-pypi:colorama==0.4.5
 pypi:commonmark==0.9.1
 pypi:docutils==0.17.1
 pypi:fluent.pygments==1.0
-pypi:fluent.syntax==0.19.0
 pypi:idna==2.10
 pypi:imagesize==1.4.1
-pypi:importlib-resources==5.12.0
 pypi:jmespath==1.0.1
-pypi:jsonschema==4.17.3
 pypi:livereload==2.6.3
-pypi:Markdown==3.4.3
-pypi:MarkupSafe==2.0.1
+pypi:mdit-py-plugins==0.3.5
 pypi:mock==5.0.1
 pypi:mots==0.8.0
 pypi:myst-parser==1.0
-pypi:mdit-py-plugins==0.3.5
 pypi:parsimonious==0.10.0
-pypi:pyasn1==0.4.8
 pypi:pyasn1-modules==0.2.8
-pypi:Pygments==2.14.0
-pypi:pyparsing==2.4.7
+pypi:pyasn1==0.4.8
 # for autodoc
 pypi:pytest==7.2.1
 pypi:python-dateutil==2.8.2
 pypi:python-frontmatter==1.0.0
 pypi:pytz==2022.7.1
 # for autodoc
 pypi:pywatchman==1.4.1
-pypi:redo==2.0.3
 pypi:s3transfer==0.6.0
 pypi:scandir==1.10.0
 pypi:singledispatch==4.0.0
 pypi:snowballstemmer==2.2.0
-pypi:Sphinx==5.1.1
 pypi:sphinx-copybutton==0.5.1
 pypi:sphinx-design==0.3.0
 pypi:sphinx-js==3.2.1
@@ -53,16 +46,42 @@ pypi:sphinxcontrib-qthelp==1.0.3
 pypi:sphinxcontrib-serializinghtml==1.1.5
 pypi:tornado==6.0.4
 vendored:third_party/python/Jinja2
+vendored:third_party/python/MarkupSafe/src
+vendored:third_party/python/PyYAML/lib/
+vendored:third_party/python/aiohttp
+vendored:third_party/python/appdirs
+vendored:third_party/python/arrow
+vendored:third_party/python/async_timeout
+vendored:third_party/python/binaryornot
 vendored:third_party/python/certifi
 vendored:third_party/python/chardet
+vendored:third_party/python/cookiecutter
+vendored:third_party/python/fluent.syntax
+vendored:third_party/python/giturlparse
 vendored:third_party/python/importlib_metadata
 vendored:third_party/python/importlib_resources
+vendored:third_party/python/jinja2_time
+vendored:third_party/python/json-e
 vendored:third_party/python/jsonschema
+vendored:third_party/python/mohawk
+vendored:third_party/python/mozilla_repo_urls
 vendored:third_party/python/packaging
 vendored:third_party/python/pkgutil_resolve_name
+vendored:third_party/python/pyparsing
+vendored:third_party/python/python_dateutil
+vendored:third_party/python/python_slugify
+vendored:third_party/python/redo
 vendored:third_party/python/requests
+vendored:third_party/python/requests_unixsocket
 vendored:third_party/python/rsa
 vendored:third_party/python/sentry_sdk
 vendored:third_party/python/six
+vendored:third_party/python/slugid
+vendored:third_party/python/taskcluster
+vendored:third_party/python/taskcluster_taskgraph
+vendored:third_party/python/taskcluster_urls
+vendored:third_party/python/text_unidecode
+vendored:third_party/python/toml
 vendored:third_party/python/urllib3
-vendored:third_party/python/zipp
+vendored:third_party/python/voluptuous
+vendored:third_party/python/zipp

python/sites/lint.txt

@@ -1,2 +1,35 @@
-
-
+vendored:third_party/python/PyYAML/lib/
+vendored:third_party/python/aiohttp
+vendored:third_party/python/appdirs
+vendored:third_party/python/arrow
+vendored:third_party/python/async_timeout
+vendored:third_party/python/binaryornot
+vendored:third_party/python/chardet
+vendored:third_party/python/compare_locales
+vendored:third_party/python/cookiecutter
+vendored:third_party/python/esprima
+vendored:third_party/python/fluent.syntax
+vendored:third_party/python/giturlparse
+vendored:third_party/python/jinja2_time
+vendored:third_party/python/json-e
+vendored:third_party/python/jsonschema
+vendored:third_party/python/mohawk
+vendored:third_party/python/mozilla_repo_urls
+vendored:third_party/python/multidict
+vendored:third_party/python/pathspec
+vendored:third_party/python/pyparsing
+vendored:third_party/python/pyrsistent
+vendored:third_party/python/python_dateutil
+vendored:third_party/python/python_slugify
+vendored:third_party/python/redo
+vendored:third_party/python/requests_unixsocket
+vendored:third_party/python/slugid
+vendored:third_party/python/taskcluster
+vendored:third_party/python/taskcluster_taskgraph
+vendored:third_party/python/taskcluster_urls
+vendored:third_party/python/text_unidecode
+vendored:third_party/python/toml
+vendored:third_party/python/typing_extensions
+vendored:third_party/python/voluptuous
+vendored:third_party/python/yamllint
+vendored:third_party/python/yarl