Fix list of overlapping versions (#17121)

* Failing test

* wip

* Cleanup

* Discard changes to test/integration/command/export/test_export.py

* Cleanup

* Docs

* Sort by name/version always

* Optimize uniqueness by delegating to a simpler sql query

* Update tests, currently it fails for the Conan server case

* Conan server now does not return duplicated references for each revision

* Cleanup

conan/api/subapi/list.py

@@ -150,7 +150,6 @@ def select(self, pattern, package_query=None, remote=None, lru=None, profile=Non
         if search_ref:
             refs = self.conan_api.search.recipes(search_ref, remote=remote)
             refs = pattern.filter_versions(refs)
-            refs = sorted(refs)  # Order alphabetical and older versions first
             pattern.check_refs(refs)
             out.info(f"Found {len(refs)} pkg/version recipes matching {search_ref} in {remote_name}")
         else:

conan/api/subapi/search.py

@@ -16,17 +16,9 @@ def recipes(self, query: str, remote=None):
         if remote:
             refs = app.remote_manager.search_recipes(remote, query)
         else:
-            references = app.cache.search_recipes(query)
-            # For consistency with the remote search, we return references without revisions
-            # user could use further the API to look for the revisions
-            refs = []
-            for r in references:
-                r.revision = None
-                r.timestamp = None
-                if r not in refs:
-                    refs.append(r)
+            refs = app.cache.search_recipes(query)
         ret = []
         for r in refs:
             if not only_none_user_channel or (r.user is None and r.channel is None):
                 ret.append(r)
-        return ret
+        return sorted(ret)

conan/internal/cache/cache.py

@@ -179,10 +179,7 @@ def search_recipes(self, pattern=None, ignorecase=True):
             pattern = translate(pattern)
             pattern = re.compile(pattern, re.IGNORECASE if ignorecase else 0)
 
-        refs = self._db.list_references()
-        if pattern:
-            refs = [r for r in refs if r.partial_match(pattern)]
-        return refs
+        return self._db.list_references(pattern)
 
     def exists_prev(self, pref):
         # Used just by download to skip downloads if prev already exists in cache

conan/internal/cache/db/cache_database.py

@@ -90,9 +90,11 @@ def create_recipe(self, path, ref: RecipeReference):
     def create_package(self, path, ref: PkgReference, build_id):
         self._packages.create(path, ref, build_id=build_id)
 
-    def list_references(self):
-        return [d["ref"]
-                for d in self._recipes.all_references()]
+    def list_references(self, pattern=None):
+        """Returns a list of all RecipeReference in the cache, optionally filtering by pattern.
+         The references have their revision and timestamp attributes unset"""
+        return [ref for ref in self._recipes.all_references()
+                if pattern is None or ref.partial_match(pattern)]
 
     def get_package_revisions_references(self, pref: PkgReference, only_latest_prev=False):
         return [d["pref"]

conan/internal/cache/db/recipes_table.py

@@ -80,18 +80,12 @@ def remove(self, ref: RecipeReference):
 
     # returns all different conan references (name/version@user/channel)
     def all_references(self):
-        query = f'SELECT DISTINCT {self.columns.reference}, ' \
-                    f'{self.columns.rrev}, ' \
-                    f'{self.columns.path} ,' \
-                    f'{self.columns.timestamp}, ' \
-                    f'{self.columns.lru} ' \
-                    f'FROM {self.table_name} ' \
-                    f'ORDER BY {self.columns.timestamp} DESC'
+        query = f'SELECT DISTINCT {self.columns.reference} FROM {self.table_name}'
 
         with self.db_connection() as conn:
             r = conn.execute(query)
-            result = [self._as_dict(self.row_type(*row)) for row in r.fetchall()]
-        return result
+            rows = r.fetchall()
+            return [RecipeReference.loads(row[0]) for row in rows]
 
     def get_recipe(self, ref: RecipeReference):
         query = f'SELECT * FROM {self.table_name} ' \

conans/server/service/v2/search.py

@@ -75,28 +75,26 @@ def _search_recipes(self, pattern=None, ignorecase=True):
         def underscore_to_none(field):
             return field if field != "_" else None
 
+        ret = set()
         if not pattern:
-            ret = []
             for folder in subdirs:
                 fields_dir = [underscore_to_none(d) for d in folder.split("/")]
                 r = RecipeReference(*fields_dir)
                 r.revision = None
-                ret.append(r)
-            return sorted(ret)
+                ret.add(r)
         else:
             # Conan references in main storage
             pattern = str(pattern)
             b_pattern = translate(pattern)
             b_pattern = re.compile(b_pattern, re.IGNORECASE) if ignorecase else re.compile(b_pattern)
-            ret = set()
             for subdir in subdirs:
                 fields_dir = [underscore_to_none(d) for d in subdir.split("/")]
                 new_ref = RecipeReference(*fields_dir)
                 new_ref.revision = None
                 if new_ref.partial_match(b_pattern):
                     ret.add(new_ref)
 
-            return sorted(ret)
+        return sorted(ret)
 
     def search(self, pattern=None, ignorecase=True):
         """ Get all the info about any package

test/integration/command_v2/list_test.py

@@ -921,3 +921,13 @@ def test_list_filter(self, remote):
         assert len(pkg["packages"]) == 2
         settings = pkg["packages"]["d2e97769569ac0a583d72c10a37d5ca26de7c9fa"]["info"]["settings"]
         assert settings == {"arch": "x86", "os": "Windows"}
+
+
+def test_overlapping_versions():
+    tc = TestClient(light=True)
+    tc.save({"conanfile.py": GenConanfile("foo")})
+    tc.run("export . --version=1.0")
+    tc.run("export . --version=1.0.0")
+    tc.run("list * -c -f=json", redirect_stdout="list.json")
+    results = json.loads(tc.load("list.json"))
+    assert len(results["Local Cache"]) == 2

test/integration/conan_api/search_test.py

@@ -1,38 +1,47 @@
+import pytest
+
 from conan.api.conan_api import ConanAPI
+from conan.api.model import Remote
 from conans.model.recipe_ref import RecipeReference
 from conan.test.assets.genconanfile import GenConanfile
-from conan.test.utils.tools import TurboTestClient
+from conan.test.utils.tools import TestClient
 
 
-def test_search_recipes():
+@pytest.mark.parametrize("remote_name", [None, "default"])
+def test_search_recipes(remote_name):
     """
         Test the "api.search.recipes"
     """
-    client = TurboTestClient(default_server_user=True)
-    ref = RecipeReference.loads("foo/1.0")
-    pref1 = client.create(ref, GenConanfile())
-    conanfile_2 = GenConanfile().with_build_msg("change2")
-    pref2 = client.create(ref, conanfile_2)
-    pref3 = client.create(RecipeReference.loads("felipe/1.0"), conanfile_2)
+    client = TestClient(default_server_user=True)
+    client.save({"conanfile.py": GenConanfile()})
+    client.run("create . --name=foo --version=1.0")
+    client.run("create . --name=felipe --version=2.0")
+    client.save({"conanfile.py": GenConanfile().with_build_msg("change")})
+    # Different version&revision, but 1.0 < 2.0, which has an earlier timestamp
+    client.run("create . --name=felipe --version=1.0")
+    # Different revision, newer timestamp
+    client.run("create . --name=foo --version=1.0")
 
-    client.upload_all(pref1.ref, "default")
-    client.upload_all(pref2.ref, "default")
-    client.upload_all(pref3.ref, "default")
+    client.run("upload * -r=default -c")
 
     # Search all the recipes locally and in the remote
     api = ConanAPI(client.cache_folder)
-    for remote in [None, api.remotes.get("default")]:
-        with client.mocked_servers():
-            sot = api.search.recipes(query="f*", remote=remote)
-            assert set(sot) == {RecipeReference.loads("foo/1.0"),
-                                RecipeReference.loads("felipe/1.0")}
+    remote = api.remotes.get(remote_name) if remote_name else None
+
+    with client.mocked_servers():
+        sot = api.search.recipes(query="f*", remote=remote)
+        assert sot == [RecipeReference.loads("felipe/1.0"),
+                       RecipeReference.loads("felipe/2.0"),
+                       RecipeReference.loads("foo/1.0")]
 
-            sot = api.search.recipes(query="fo*", remote=remote)
-            assert set(sot) == {RecipeReference.loads("foo/1.0")}
+        sot = api.search.recipes(query="fo*", remote=remote)
+        assert sot == [RecipeReference.loads("foo/1.0")]
 
-            sot = api.search.recipes(query=None, remote=remote)
-            assert set(sot) == {RecipeReference.loads("foo/1.0"),
-                                RecipeReference.loads("felipe/1.0")}
+        sot = api.search.recipes(query=None, remote=remote)
+        assert sot == [RecipeReference.loads("felipe/1.0"),
+                       RecipeReference.loads("felipe/2.0"),
+                       RecipeReference.loads("foo/1.0")]
 
-            sot = api.search.recipes(query="*i*", remote=remote)
-            assert set(sot) == {RecipeReference.loads("felipe/1.0")}
+        sot = api.search.recipes(query="*i*", remote=remote)
+        assert sot == [RecipeReference.loads("felipe/1.0"),
+                       RecipeReference.loads("felipe/2.0")]