Cache save and restore new functionality

conan/api/subapi/cache.py

@@ -1,11 +1,19 @@
+import json
 import os
+import shutil
+import tarfile
+from io import BytesIO
 
+from conan.api.model import PackagesList
+from conan.api.output import ConanOutput
 from conan.internal.conan_app import ConanApp
 from conan.internal.integrity_check import IntegrityChecker
+from conans.client.cache.cache import ClientCache
 from conans.errors import ConanException
 from conans.model.package_ref import PkgReference
 from conans.model.recipe_ref import RecipeReference
-from conans.util.files import rmdir
+from conans.util.dates import revision_timestamp_now
+from conans.util.files import rmdir, gzopen_without_timestamps
 
 
 class CacheAPI:
@@ -101,6 +109,75 @@ def clean(self, package_list, source=True, build=True, download=True, temp=True)
                 if download:
                     rmdir(pref_layout.download_package())
 
+    def save(self, package_list, tgz_path):
+        cache_folder = self.conan_api.cache_folder
+        app = ConanApp(cache_folder)
+        out = ConanOutput()
+        name = os.path.basename(tgz_path)
+        with open(tgz_path, "wb") as tgz_handle:
+            tgz = gzopen_without_timestamps(name, mode="w", fileobj=tgz_handle)
+            for ref, ref_bundle in package_list.refs().items():
+                ref_layout = app.cache.recipe_layout(ref)
+                recipe_folder = os.path.relpath(ref_layout.base_folder, cache_folder)
+                ref_bundle["recipe_folder"] = recipe_folder
+                out.info(f"Saving {ref}: {recipe_folder}")
+                tgz.add(os.path.join(cache_folder, recipe_folder), recipe_folder, recursive=True)
+                for pref, pref_bundle in package_list.prefs(ref, ref_bundle).items():
+                    pref_layout = app.cache.pkg_layout(pref)
+                    pkg_folder = pref_layout.package()
+                    folder = os.path.relpath(pkg_folder, cache_folder)
+                    pref_bundle["package_folder"] = folder
+                    out.info(f"Saving {pref}: {folder}")
+                    tgz.add(os.path.join(cache_folder, folder), folder, recursive=True)
+                    if os.path.exists(pref_layout.metadata()):
+                        metadata_folder = os.path.relpath(pref_layout.metadata(), cache_folder)
+                        pref_bundle["metadata_folder"] = metadata_folder
+                        out.info(f"Saving {pref} metadata: {folder}")
+                        tgz.add(os.path.join(cache_folder, metadata_folder), metadata_folder,
+                                recursive=True)
+            serialized = json.dumps(package_list.serialize(), indent=2)
+            info = tarfile.TarInfo(name="pkglist.json")
+            data = serialized.encode('utf-8')
+            info.size = len(data)
+            tgz.addfile(tarinfo=info, fileobj=BytesIO(data))
+            tgz.close()
+
+    def restore(self, path):
+        with open(path, mode='rb') as file_handler:
+            the_tar = tarfile.open(fileobj=file_handler)
+            fileobj = the_tar.extractfile("pkglist.json")
+            pkglist = fileobj.read()
+            the_tar.extractall(path=self.conan_api.cache_folder)
+            the_tar.close()
+
+        out = ConanOutput()
+        package_list = PackagesList.deserialize(json.loads(pkglist))
+        cache = ClientCache(self.conan_api.cache_folder)
+        for ref, ref_bundle in package_list.refs().items():
+            ref.timestamp = revision_timestamp_now()
+            ref_bundle["timestamp"] = ref.timestamp
+            recipe_layout = cache.get_or_create_ref_layout(ref)
+            recipe_folder = ref_bundle["recipe_folder"]
+            rel_path = os.path.relpath(recipe_layout.base_folder, cache.cache_folder)
+            assert rel_path == recipe_folder, f"{rel_path}!={recipe_folder}"
+            out.info(f"Put: {ref} in {recipe_folder}")
+            for pref, pref_bundle in package_list.prefs(ref, ref_bundle).items():
+                pref.timestamp = revision_timestamp_now()
+                pref_bundle["timestamp"] = pref.timestamp
+                pkg_layout = cache.get_or_create_pkg_layout(pref)
+                pkg_folder = pref_bundle["package_folder"]
+                out.info(f"Restore: {pref} in {pkg_folder}")
+                # We need to put the package in the final location in the cache
+                shutil.move(os.path.join(cache.cache_folder, pkg_folder), pkg_layout.package())
+                metadata_folder = pref_bundle.get("metadata_folder")
+                if metadata_folder:
+                    out.info(f"Restore: {pref} metadata in {metadata_folder}")
+                    # We need to put the package in the final location in the cache
+                    shutil.move(os.path.join(cache.cache_folder, metadata_folder),
+                                pkg_layout.metadata())
+
+        return package_list
+
 
 def _resolve_latest_ref(app, ref):
     if ref.revision is None or ref.revision == "latest":

conan/cli/commands/cache.py

@@ -1,7 +1,9 @@
 from conan.api.conan_api import ConanAPI
 from conan.api.model import ListPattern
 from conan.api.output import cli_out_write
+from conan.cli import make_abs_path
 from conan.cli.command import conan_command, conan_subcommand, OnceArgument
+from conan.cli.commands.list import print_list_text, print_list_json
 from conan.errors import ConanException
 from conans.model.package_ref import PkgReference
 from conans.model.recipe_ref import RecipeReference
@@ -99,3 +101,35 @@ def cache_check_integrity(conan_api: ConanAPI, parser, subparser, *args):
     ref_pattern = ListPattern(args.pattern, rrev="*", package_id="*", prev="*")
     package_list = conan_api.list.select(ref_pattern, package_query=args.package_query)
     conan_api.cache.check_integrity(package_list)
+
+
+@conan_subcommand(formatters={"text": print_list_text,
+                              "json": print_list_json})
+def cache_save(conan_api: ConanAPI, parser, subparser, *args):
+    """
+    Get the artifacts from a package list and archive them
+    """
+    subparser.add_argument('file', help="Save to this file")
+    subparser.add_argument('pattern',
+                           help="A pattern in the form 'pkg/version#revision:package_id#revision', "
+                                "e.g: zlib/1.2.13:* means all binaries for zlib/1.2.13. "
+                                "If revision is not specified, it is assumed latest one.")
+    args = parser.parse_args(*args)
+    ref_pattern = ListPattern(args.pattern)
+    package_list = conan_api.list.select(ref_pattern)
+    tgz_path = make_abs_path(args.file)
+    conan_api.cache.save(package_list, tgz_path)
+    return {"results": {"Local Cache": package_list.serialize()}}
+
+
+@conan_subcommand(formatters={"text": print_list_text,
+                              "json": print_list_json})
+def cache_restore(conan_api: ConanAPI, parser, subparser, *args):
+    """
+    Put  the artifacts from a an archive into the cache
+    """
+    subparser.add_argument("file", help="Path to archive to restore")
+    args = parser.parse_args(*args)
+    path = make_abs_path(args.file)
+    package_list = conan_api.cache.restore(path)
+    return {"results": {"Local Cache": package_list.serialize()}}

conans/test/integration/command_v2/test_cache_save_restore.py

@@ -0,0 +1,123 @@
+import json
+import os
+import shutil
+
+from conans.test.assets.genconanfile import GenConanfile
+from conans.test.utils.tools import TestClient
+from conans.util.files import save, load
+
+
+def test_cache_save_restore():
+    c = TestClient()
+    c.save({"conanfile.py": GenConanfile().with_settings("os")})
+    c.run("create . --name=pkg --version=1.0 -s os=Linux")
+    c.run("create . --name=pkg --version=1.1 -s os=Linux")
+    c.run("create . --name=other --version=2.0 -s os=Linux")
+    c.run("cache save cache.tgz pkg/*:* ")
+    cache_path = os.path.join(c.current_folder, "cache.tgz")
+    assert os.path.exists(cache_path)
+    _validate_restore(cache_path)
+
+
+def test_cache_save_downloaded_restore():
+    """ what happens if we save packages downloaded from server, not
+    created
+    """
+    c = TestClient(default_server_user=True)
+    c.save({"conanfile.py": GenConanfile().with_settings("os")})
+    c.run("create . --name=pkg --version=1.0 -s os=Linux")
+    c.run("create . --name=pkg --version=1.1 -s os=Linux")
+    c.run("create . --name=other --version=2.0 -s os=Linux")
+    c.run("upload * -r=default -c")
+    c.run("remove * -c")
+    c.run("download *:* -r=default --metadata=*")
+    c.run("cache save cache.tgz pkg/*:* ")
+    cache_path = os.path.join(c.current_folder, "cache.tgz")
+    assert os.path.exists(cache_path)
+
+    _validate_restore(cache_path)
+
+
+def _validate_restore(cache_path):
+    c2 = TestClient()
+    # Create a package in the cache to check put doesn't interact badly
+    c2.save({"conanfile.py": GenConanfile().with_settings("os")})
+    c2.run("create . --name=pkg2 --version=3.0 -s os=Windows")
+    shutil.copy2(cache_path, c2.current_folder)
+    c2.run("cache restore cache.tgz")
+    c2.run("list *:*#*")
+    assert "pkg2/3.0" in c2.out
+    assert "pkg/1.0" in c2.out
+    assert "pkg/1.1" in c2.out
+    assert "other/2.0" not in c2.out
+
+    # Restore again, just in case
+    c2.run("cache restore cache.tgz")
+    c2.run("list *:*#*")
+    assert "pkg2/3.0" in c2.out
+    assert "pkg/1.0" in c2.out
+    assert "pkg/1.1" in c2.out
+    assert "other/2.0" not in c2.out
+
+
+def test_cache_save_restore_metadata():
+    c = TestClient()
+    c.save({"conanfile.py": GenConanfile().with_settings("os")})
+    c.run("create . --name=pkg --version=1.0 -s os=Linux")
+    pid = c.created_package_id("pkg/1.0")
+    # Add some metadata
+    c.run("cache path pkg/1.0 --folder=metadata")
+    metadata_path = str(c.stdout).strip()
+    myfile = os.path.join(metadata_path, "logs", "mylogs.txt")
+    save(myfile, "mylogs!!!!")
+    c.run(f"cache path pkg/1.0:{pid} --folder=metadata")
+    pkg_metadata_path = str(c.stdout).strip()
+    myfile = os.path.join(pkg_metadata_path, "logs", "mybuildlogs.txt")
+    save(myfile, "mybuildlogs!!!!")
+
+    c.run("cache save cache.tgz pkg/*:* ")
+    cache_path = os.path.join(c.current_folder, "cache.tgz")
+    assert os.path.exists(cache_path)
+
+    # restore and check
+    c2 = TestClient()
+    shutil.copy2(cache_path, c2.current_folder)
+    c2.run("cache restore cache.tgz")
+    c2.run("cache path pkg/1.0 --folder=metadata")
+    metadata_path = str(c2.stdout).strip()
+    myfile = os.path.join(metadata_path, "logs", "mylogs.txt")
+    assert load(myfile) == "mylogs!!!!"
+    c2.run(f"cache path pkg/1.0:{pid} --folder=metadata")
+    pkg_metadata_path = str(c2.stdout).strip()
+    myfile = os.path.join(pkg_metadata_path, "logs", "mybuildlogs.txt")
+    assert load(myfile) == "mybuildlogs!!!!"
+
+
+def test_cache_save_restore_multiple_revisions():
+    c = TestClient()
+    c.save({"conanfile.py": GenConanfile("pkg", "0.1")})
+    c.run("create .")
+    rrev1 = c.exported_recipe_revision()
+    c.save({"conanfile.py": GenConanfile("pkg", "0.1").with_class_attribute("var=42")})
+    c.run("create .")
+    rrev2 = c.exported_recipe_revision()
+    c.save({"conanfile.py": GenConanfile("pkg", "0.1").with_class_attribute("var=123")})
+    c.run("create .")
+    rrev3 = c.exported_recipe_revision()
+
+    def check_ordered_revisions(client):
+        client.run("list *#* --format=json")
+        revisions = json.loads(client.stdout)["Local Cache"]["pkg/0.1"]["revisions"]
+        assert revisions[rrev1]["timestamp"] < revisions[rrev2]["timestamp"]
+        assert revisions[rrev2]["timestamp"] < revisions[rrev3]["timestamp"]
+
+    check_ordered_revisions(c)
+
+    c.run("cache save cache.tgz pkg/*#*:* ")
+    cache_path = os.path.join(c.current_folder, "cache.tgz")
+
+    # restore and check
+    c2 = TestClient()
+    shutil.copy2(cache_path, c2.current_folder)
+    c2.run("cache restore cache.tgz")
+    check_ordered_revisions(c2)