Use semantic_version for version requirements

Pythons pkg_resources Requirement introduced stricter PEP-440 checking
that is incompatible with ansible versions.

fixes #1349

(cherry picked from commit b1da66a)

CHANGES/1349.misc

@@ -0,0 +1,2 @@
+Switched to using ``semantic_version`` for version comparison, and stopped depending on
+``setuptools``.

pulp_ansible/app/tasks/collections.py

@@ -24,7 +24,6 @@
 from galaxy_importer.collection import sync_collection
 from galaxy_importer.exceptions import ImporterError
 from git import GitCommandError, Repo
-from pkg_resources import Requirement
 from pulpcore.plugin.models import (
     Artifact,
     ContentArtifact,
@@ -50,7 +49,8 @@
     Stage,
 )
 from rest_framework.serializers import ValidationError
-from semantic_version import Version
+from semantic_version import SimpleSpec, Version
+from semantic_version.base import Always
 
 from pulp_ansible.app.constants import PAGE_SIZE
 from pulp_ansible.app.models import (
@@ -74,6 +74,14 @@
 log = logging.getLogger(__name__)
 
 
+# semantic_version.SimpleSpec interpretes "*" as ">=0.0.0"
+class AnsibleSpec(SimpleSpec):
+    def __init__(self, expression):
+        super().__init__(expression)
+        if self.expression == "*":
+            self.clause = Always()
+
+
 async def declarative_content_from_git_repo(remote, url, git_ref=None, metadata_only=False):
     """Returns a DeclarativeContent for the Collection in a Git repository."""
     if git_ref:
@@ -197,25 +205,6 @@ def sync(remote_pk, repository_pk, mirror, optimize):
         ).all().update(version_removed=repo_version)
 
 
-def parse_requirements_entry(requirements_entry):
-    """Parses a `RequirementsFileEntry` and returns a `Requirement` object."""
-    if requirements_entry.version == "*":
-        requirement_version = Requirement.parse("collection")
-    else:
-        # We need specifiers to enforce Requirement object criteria
-        # https://setuptools.readthedocs.io/en/latest/pkg_resources.html#requirements-parsing
-        # https://setuptools.readthedocs.io/en/latest/pkg_resources.html#requirement-methods-and-attributes
-        # If requirements_entry.version is a valid version, adds == specifier to the requirement
-        try:
-            Version(requirements_entry.version)
-            req_to_parse = f"collection=={requirements_entry.version}"
-        except ValueError:
-            req_to_parse = f"collection{requirements_entry.version}"
-
-        requirement_version = Requirement.parse(req_to_parse)
-    return requirement_version
-
-
 def import_collection(
     temp_file_pk,
     repository_pk=None,
@@ -689,7 +678,7 @@ async def _fetch_paginated_collection_metadata(self, name, namespace, requiremen
             else:
                 collection_versions = collection_versions_list["data"]
             for collection_version in collection_versions:
-                if collection_version["version"] in requirement:
+                if Version(collection_version["version"]) in requirement:
                     version_num = collection_version["version"]
                     collection_version_detail_url = f"{collection_url}/versions/{version_num}/"
                     if collection_metadata["deprecated"]:
@@ -734,7 +723,7 @@ async def _read_from_downloaded_metadata(self, name, namespace, requirement):
 
         all_versions_of_collection = self._unpaginated_collection_version_metadata[namespace][name]
         for col_version_metadata in all_versions_of_collection:
-            if col_version_metadata["version"] in requirement:
+            if Version(col_version_metadata["version"]) in requirement:
                 if "git_url" in col_version_metadata and col_version_metadata["git_url"]:
                     tasks.append(
                         loop.create_task(
@@ -759,7 +748,7 @@ async def _read_from_downloaded_metadata(self, name, namespace, requirement):
         await asyncio.gather(*tasks)
 
     async def _fetch_collection_metadata(self, requirements_entry):
-        requirement_version = parse_requirements_entry(requirements_entry)
+        requirement_version = AnsibleSpec(requirements_entry.version)
 
         namespace, name = requirements_entry.name.split(".")
 
@@ -810,7 +799,7 @@ async def _download_unpaginated_metadata(self):
                     except ValidationError:
                         pass
                     else:
-                        excludes = {r.name: parse_requirements_entry(r) for r in excludes_list}
+                        excludes = {r.name: AnsibleSpec(r.version) for r in excludes_list}
                         self.exclude_info.update(excludes)
 
             if not isinstance(col_results, FileNotFoundError):

pulp_ansible/tests/functional/api/collection/test_sync.py

@@ -322,6 +322,32 @@ def test_optimized_sync(
     # TODO CHECK IF SYNC WAS NOT OPTIMIZED
 
 
+@pytest.mark.parallel
+def test_semver_sync(
+    ansible_repo_api_client,
+    ansible_collection_version_api_client,
+    ansible_repo_factory,
+    ansible_collection_remote_factory,
+):
+    remote = ansible_collection_remote_factory(
+        url="https://galaxy.ansible.com",
+        requirements_file="collections:\n  - anil_cm.terraform_provider",
+        sync_dependencies=False,
+        signed_only=False,
+    )
+    repository = ansible_repo_factory(remote=remote.pulp_href)
+    monitor_task(ansible_repo_api_client.sync(repository.pulp_href, {}).task)
+    repository = ansible_repo_api_client.read(repository.pulp_href)
+    content = ansible_collection_version_api_client.list(
+        repository_version=repository.latest_version_href
+    )
+    versions = {item.version for item in content.results}
+    # If this fails check that it is still upstream!
+    # TODO create a better local fixture to sync from.
+    assert "0.0.1-rerelease+meta" in versions
+    assert "0.0.0-rerelease+meta" in versions
+
+
 @pytest.mark.parallel
 def test_last_synced_metadata_time(
     ansible_repo_api_client,

requirements.txt

@@ -5,4 +5,3 @@ jsonschema>=4.9,<4.18
 packaging>=21.3,<22
 pulpcore>=3.21.dev,<3.25
 PyYAML<7.0
-semantic_version>=2.9,<2.11