ansible-galaxy - support resolvelib versions >= 0.5.3, < 0.9.0 (ansib…

…le#77649)

* ansible-galaxy - support resolvelib versions >= 0.5.3, <= 0.8.1

Test incompatibilities are removed for resolvelib >= 0.6.0

Test against the latest 0.8.x version and fix requirements

* Fix tests - use a venv for testing the range of resolvelib versions

* Update temporary hardcoded fallback for ansible-test

* Update hardcoded upperbound for sanity tests

* Make error check more flexible

changelogs/fragments/77649-support-recent-resolvelib-versions.yml

@@ -0,0 +1,3 @@
+minor_changes:
+  - ansible-galaxy - Support resolvelib versions 0.6.x, 0.7.x, and 0.8.x.
+    The full range of supported versions is now >= 0.5.3, < 0.9.0.

lib/ansible/galaxy/dependency_resolution/providers.py

@@ -41,7 +41,7 @@ class AbstractProvider:  # type: ignore[no-redef]
 
 # TODO: add python requirements to ansible-test's ansible-core distribution info and remove the hardcoded lowerbound/upperbound fallback
 RESOLVELIB_LOWERBOUND = SemanticVersion("0.5.3")
-RESOLVELIB_UPPERBOUND = SemanticVersion("0.6.0")
+RESOLVELIB_UPPERBOUND = SemanticVersion("0.9.0")
 RESOLVELIB_VERSION = SemanticVersion.from_loose_version(LooseVersion(resolvelib_version))
 
 
@@ -73,11 +73,11 @@ def __len__(self):
         return len(self._candidates)
 
 
-class CollectionDependencyProvider(AbstractProvider):
+class CollectionDependencyProviderBase(AbstractProvider):
     """Delegate providing a requirement interface for the resolver."""
 
     def __init__(
-            self,  # type: CollectionDependencyProvider
+            self,  # type: CollectionDependencyProviderBase
             apis,  # type: MultiGalaxyAPIProxy
             concrete_artifacts_manager=None,  # type: ConcreteArtifactsManager
             user_requirements=None,  # type: t.Iterable[Requirement]
@@ -180,19 +180,17 @@ def identify(self, requirement_or_candidate):
         """
         return requirement_or_candidate.canonical_package_id
 
-    def get_preference(
-            self,  # type: CollectionDependencyProvider
-            resolution,  # type: t.Optional[Candidate]
-            candidates,  # type: list[Candidate]
-            information,  # type: list[t.NamedTuple]
-    ):  # type: (...) -> t.Union[float, int]
+    def get_preference(self, *args, **kwargs):
+        # type: (t.Any, t.Any) -> t.Union[float, int]
         """Return sort key function return value for given requirement.
 
         This result should be based on preference that is defined as
         "I think this requirement should be resolved first".
         The lower the return value is, the more preferred this
         group of arguments is.
 
+        resolvelib >=0.5.3, <0.7.0
+
         :param resolution: Currently pinned candidate, or ``None``.
 
         :param candidates: A list of possible candidates.
@@ -208,6 +206,35 @@ def get_preference(
             (dependend on) the requirement, or `None`
             to indicate a root requirement.
 
+        resolvelib >=0.7.0, < 0.8.0
+
+        :param identifier: The value returned by ``identify()``.
+
+        :param resolutions: Mapping of identifier, candidate pairs.
+
+        :param candidates: Possible candidates for the identifer.
+            Mapping of identifier, list of candidate pairs.
+
+        :param information: Requirement information of each package.
+            Mapping of identifier, list of named tuple pairs.
+            The named tuples have the entries ``requirement`` and ``parent``.
+
+        resolvelib >=0.8.0, <= 0.8.1
+
+        :param identifier: The value returned by ``identify()``.
+
+        :param resolutions: Mapping of identifier, candidate pairs.
+
+        :param candidates: Possible candidates for the identifer.
+            Mapping of identifier, list of candidate pairs.
+
+        :param information: Requirement information of each package.
+            Mapping of identifier, list of named tuple pairs.
+            The named tuples have the entries ``requirement`` and ``parent``.
+
+        :param backtrack_causes: Sequence of requirement information that were
+            the requirements that caused the resolver to most recently backtrack.
+
         The preference could depend on a various of issues, including
         (not necessarily in this order):
 
@@ -229,6 +256,10 @@ def get_preference(
         the value is, the more preferred this requirement is (i.e. the
         sorting function is called with ``reverse=False``).
         """
+        raise NotImplementedError
+
+    def _get_preference(self, candidates):
+        # type: (list[Candidate]) -> t.Union[float, int]
         if any(
                 candidate in self._preferred_candidates
                 for candidate in candidates
@@ -238,8 +269,8 @@ def get_preference(
             return float('-inf')
         return len(candidates)
 
-    def find_matches(self, requirements):
-        # type: (list[Requirement]) -> list[Candidate]
+    def find_matches(self, *args, **kwargs):
+        # type: (t.Any, t.Any) -> list[Candidate]
         r"""Find all possible candidates satisfying given requirements.
 
         This tries to get candidates based on the requirements' types.
@@ -251,15 +282,31 @@ def find_matches(self, requirements):
         to find concrete candidates for this requirement. Of theres a
         pre-installed candidate, it's prepended in front of others.
 
+        resolvelib >=0.5.3, <0.6.0
+
         :param requirements: A collection of requirements which all of \
                              the returned candidates must match. \
                              All requirements are guaranteed to have \
                              the same identifier. \
                              The collection is never empty.
 
+        resolvelib >=0.6.0
+
+        :param identifier: The value returned by ``identify()``.
+
+        :param requirements: The requirements all returned candidates must satisfy.
+            Mapping of identifier, iterator of requirement pairs.
+
+        :param incompatibilities: Incompatible versions that must be excluded
+            from the returned list.
+
         :returns: An iterable that orders candidates by preference, \
                   e.g. the most preferred candidate comes first.
         """
+        raise NotImplementedError
+
+    def _find_matches(self, requirements):
+        # type: (list[Requirement]) -> list[Candidate]
         # FIXME: The first requirement may be a Git repo followed by
         # FIXME: its cloned tmp dir. Using only the first one creates
         # FIXME: loops that prevent any further dependency exploration.
@@ -438,3 +485,52 @@ def get_dependencies(self, candidate):
             self._make_req_from_dict({'name': dep_name, 'version': dep_req})
             for dep_name, dep_req in req_map.items()
         ]
+
+
+# Classes to handle resolvelib API changes between minor versions for 0.X
+class CollectionDependencyProvider050(CollectionDependencyProviderBase):
+    def find_matches(self, requirements):  # type: ignore[override]
+        # type: (list[Requirement]) -> list[Candidate]
+        return self._find_matches(requirements)
+
+    def get_preference(self, resolution, candidates, information):  # type: ignore[override]
+        # type: (t.Optional[Candidate], list[Candidate], list[t.NamedTuple]) -> t.Union[float, int]
+        return self._get_preference(candidates)
+
+
+class CollectionDependencyProvider060(CollectionDependencyProviderBase):
+    def find_matches(self, identifier, requirements, incompatibilities):  # type: ignore[override]
+        # type: (str, t.Mapping[str, t.Iterator[Requirement]], t.Mapping[str, t.Iterator[Requirement]]) -> list[Candidate]
+        return [
+            match for match in self._find_matches(list(requirements[identifier]))
+            if not any(match.ver == incompat.ver for incompat in incompatibilities[identifier])
+        ]
+
+    def get_preference(self, resolution, candidates, information):  # type: ignore[override]
+        # type: (t.Optional[Candidate], list[Candidate], list[t.NamedTuple]) -> t.Union[float, int]
+        return self._get_preference(candidates)
+
+
+class CollectionDependencyProvider070(CollectionDependencyProvider060):
+    def get_preference(self, identifier, resolutions, candidates, information):  # type: ignore[override]
+        # type: (str, t.Mapping[str, Candidate], t.Mapping[str, t.Iterator[Candidate]], t.Iterator[t.NamedTuple]) -> t.Union[float, int]
+        return self._get_preference(list(candidates[identifier]))
+
+
+class CollectionDependencyProvider080(CollectionDependencyProvider060):
+    def get_preference(self, identifier, resolutions, candidates, information, backtrack_causes):  # type: ignore[override]
+        # type: (str, t.Mapping[str, Candidate], t.Mapping[str, t.Iterator[Candidate]], t.Iterator[t.NamedTuple], t.Sequence) -> t.Union[float, int]
+        return self._get_preference(list(candidates[identifier]))
+
+
+def _get_provider():  # type () -> CollectionDependencyProviderBase
+    if RESOLVELIB_VERSION >= SemanticVersion("0.8.0"):
+        return CollectionDependencyProvider080
+    if RESOLVELIB_VERSION >= SemanticVersion("0.7.0"):
+        return CollectionDependencyProvider070
+    if RESOLVELIB_VERSION >= SemanticVersion("0.6.0"):
+        return CollectionDependencyProvider060
+    return CollectionDependencyProvider050
+
+
+CollectionDependencyProvider = _get_provider()

requirements.txt

@@ -10,4 +10,6 @@ packaging
 # NOTE: resolvelib 0.x version bumps should be considered major/breaking
 # NOTE: and we should update the upper cap with care, at least until 1.0
 # NOTE: Ref: https://github.com/sarugaku/resolvelib/issues/69
-resolvelib >= 0.5.3, < 0.6.0  # dependency resolver used by ansible-galaxy
+# NOTE: When updating the upper bound, also update the latest version used
+# NOTE: in the ansible-galaxy-collection test suite.
+resolvelib >= 0.5.3, < 0.9.0  # dependency resolver used by ansible-galaxy

test/integration/targets/ansible-galaxy-collection-scm/tasks/main.yml

@@ -23,6 +23,10 @@
   - include_tasks: ./multi_collection_repo_individual.yml
   - include_tasks: ./setup_recursive_scm_dependency.yml
   - include_tasks: ./scm_dependency_deduplication.yml
+  - include_tasks: ./test_supported_resolvelib_versions.yml
+    loop: "{{ supported_resolvelib_versions }}"
+    loop_control:
+      loop_var: resolvelib_version
   - include_tasks: ./download.yml
   - include_tasks: ./setup_collection_bad_version.yml
   - include_tasks: ./test_invalid_version.yml

test/integration/targets/ansible-galaxy-collection-scm/tasks/test_supported_resolvelib_versions.yml

@@ -0,0 +1,25 @@
+- vars:
+    venv_cmd: "{{ ansible_python_interpreter ~ ' -m venv' }}"
+    venv_dest: "{{ galaxy_dir }}/test_venv_{{ resolvelib_version }}"
+  block:
+    - name: install another version of resolvelib that is supported by ansible-galaxy
+      pip:
+        name: resolvelib
+        version: "{{ resolvelib_version }}"
+        state: present
+        virtualenv_command: "{{ venv_cmd }}"
+        virtualenv: "{{ venv_dest }}"
+        virtualenv_site_packages: True
+
+    - include_tasks: ./scm_dependency_deduplication.yml
+      args:
+        apply:
+          environment:
+            PATH: "{{ venv_dest }}/bin:{{ ansible_env.PATH }}"
+            ANSIBLE_CONFIG: '{{ galaxy_dir }}/ansible.cfg'
+
+  always:
+    - name: remove test venv
+      file:
+        path: "{{ venv_dest }}"
+        state: absent

test/integration/targets/ansible-galaxy-collection-scm/vars/main.yml

@@ -3,3 +3,9 @@ alt_install_path: "{{ galaxy_dir }}/other_collections/ansible_collections"
 scm_path: "{{ galaxy_dir }}/development"
 test_repo_path: "{{ galaxy_dir }}/development/ansible_test"
 test_error_repo_path: "{{ galaxy_dir }}/development/error_test"
+
+supported_resolvelib_versions:
+  - "0.5.3"  # Oldest supported
+  - "0.6.0"
+  - "0.7.0"
+  - "0.8.0"

test/integration/targets/ansible-galaxy-collection/tasks/download.yml

@@ -169,3 +169,8 @@
         that:
         - '"Downloading collection ''ansible_test.my_collection:1.0.0'' to" in download_collection.stdout'
         - download_collection_actual.stat.exists
+
+- name: remove test download dir
+  file:
+    path: '{{ galaxy_dir }}/download'
+    state: absent

test/integration/targets/ansible-galaxy-collection/tasks/fail_fast_resolvelib.yml

@@ -0,0 +1,45 @@
+# resolvelib>=0.6.0 added an 'incompatibilities' parameter to find_matches
+# If incompatibilities aren't removed from the viable candidates, this example causes infinite resursion
+- name: test resolvelib removes incompatibilites in find_matches and errors quickly (prevent infinite recursion)
+  block:
+    - name: create collection dir
+      file:
+        dest: "{{ galaxy_dir }}/resolvelib/ns/coll"
+        state: directory
+
+    - name: create galaxy.yml with a dependecy on a galaxy-sourced collection
+      copy:
+        dest: "{{ galaxy_dir }}/resolvelib/ns/coll/galaxy.yml"
+        content: |
+                namespace: ns
+                name: coll
+                authors:
+                  - ansible-core
+                readme: README.md
+                version: "1.0.0"
+                dependencies:
+                  namespace1.name1: "0.0.5"
+
+    - name: build the collection
+      command: ansible-galaxy collection build ns/coll
+      args:
+        chdir: "{{ galaxy_dir }}/resolvelib"
+
+    - name: install a conflicting version of the dep with the tarfile (expected failure)
+      command: ansible-galaxy collection install namespace1.name1:1.0.9 ns-coll-1.0.0.tar.gz -vvvvv -s {{ test_name }} -p collections/
+      args:
+        chdir: "{{ galaxy_dir }}/resolvelib"
+      timeout: 30
+      ignore_errors: yes
+      register: incompatible
+
+    - assert:
+        that:
+          - incompatible.failed
+          - not incompatible.msg.startswith("The command action failed to execute in the expected time frame")
+
+  always:
+    - name: cleanup resolvelib test
+      file:
+        dest: "{{ galaxy_dir }}/resolvelib"
+        state: absent