storeapi: move from details (v1) to info (v2) (#2550)

Instead of accessing each element from the returned calls as dictionary
elements, this implementation offers classes with attributes and some
methods to interact with the API results.

Signed-off-by: Sergio Schvezov <sergio.schvezov@canonical.com>

snapcraft/_store.py

@@ -26,7 +26,7 @@
 import tempfile
 from datetime import datetime
 from subprocess import Popen
-from typing import Dict, Iterable, TextIO
+from typing import Dict, Iterable, Optional, TextIO
 
 # Ideally we would move stuff into more logical components
 from snapcraft.cli import echo
@@ -761,10 +761,19 @@ def close(snap_name, channel_names):
     logger.info(msg)
 
 
-def download(snap_name, channel, download_path, arch, except_hash=""):
+def download(
+    snap_name,
+    *,
+    arch: str,
+    download_path: str,
+    risk: str,
+    track: Optional[str] = None,
+    except_hash=""
+):
     """Download snap from the store to download_path.
     :param str snap_name: The snap name to download.
-    :param str channel: the channel to get the snap from.
+    :param str risk: the channel risk get the snap from.
+    :param str track: the specific channel track get the snap from.
     :param str download_path: the path to write the downloaded snap to.
     :param str arch: the architecture of the download as a deb arch.
     :param str except_hash: do not download if set to a sha3_384 hash that
@@ -775,7 +784,14 @@ def download(snap_name, channel, download_path, arch, except_hash=""):
     :returns: A sha3_384 of the file that was or would have been downloaded.
     """
     store = storeapi.StoreClient()
-    return store.download(snap_name, channel, download_path, arch, except_hash)
+    return store.download(
+        snap_name,
+        risk=risk,
+        track=track,
+        download_path=download_path,
+        arch=arch,
+        except_hash=except_hash,
+    )
 
 
 def status(snap_name, series, arch):
@@ -888,13 +904,13 @@ def validate(snap_name, validations, revoke=False, key=None):
     for validation in validations:
         gated_name, rev = validation.split("=", 1)
         echo.info("Getting details for {}".format(gated_name))
-        approved_data = store.cpi.get_package(gated_name, "stable")
+        approved_data = store.cpi.get_info(gated_name)
         assertion = {
             "type": "validation",
             "authority-id": authority_id,
             "series": release,
             "snap-id": snap_id,
-            "approved-snap-id": approved_data["snap_id"],
+            "approved-snap-id": approved_data.snap_id,
             "approved-snap-revision": rev,
             "timestamp": datetime.utcnow().isoformat() + "Z",
             "revoked": "false",

snapcraft/file_utils.py

@@ -1,6 +1,6 @@
 # -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
 #
-# Copyright (C) 2016 Canonical Ltd
+# Copyright (C) 2016-2019 Canonical Ltd
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
@@ -311,6 +311,14 @@ def requires_path_exists(path: str, error_fmt: str = None) -> Generator:
     yield
 
 
+def _file_reader_iter(path: str, block_size=2 ** 20):
+    with open(path, "rb") as f:
+        block = f.read(block_size)
+        while len(block) > 0:
+            yield block
+            block = f.read(block_size)
+
+
 def calculate_sha3_384(path: str) -> str:
     """Calculate sha3 384 hash, reading the file in 1MB chunks."""
     return calculate_hash(path, algorithm="sha3_384")
@@ -321,13 +329,8 @@ def calculate_hash(path: str, *, algorithm: str) -> str:
     # This will raise an AttributeError if algorithm is unsupported
     hasher = getattr(hashlib, algorithm)()
 
-    blocksize = 2 ** 20
-    with open(path, "rb") as f:
-        while True:
-            buf = f.read(blocksize)
-            if not buf:
-                break
-            hasher.update(buf)
+    for block in _file_reader_iter(path):
+        hasher.update(block)
     return hasher.hexdigest()
 
 

snapcraft/internal/build_providers/_snap.py

@@ -42,7 +42,37 @@ class _SnapOp(enum.Enum):
     REFRESH = 3
 
 
-def _get_snap_channel(snap_name: str) -> str:
+_VALID_RISKS = ["stable", "candidate", "beta", "edge"]
+
+
+class _Channel:
+    def __str__(self) -> str:
+        return self._channel
+
+    def __init__(self, channel: str) -> None:
+        channel_parts = channel.split("/")
+        if len(channel_parts) == 1:
+            self.track = None
+            self.risk = channel_parts[0]
+            self.branch = None
+        elif len(channel_parts) == 3:
+            self.track = channel_parts[0]
+            self.risk = channel_parts[1]
+            self.branch = channel_parts[2]
+        elif len(channel_parts) == 2 and channel_parts[0] in _VALID_RISKS:
+            self.track = None
+            self.risk = channel_parts[0]
+            self.branch = channel_parts[1]
+        elif len(channel_parts) == 2 and channel_parts[1] in _VALID_RISKS:
+            self.track = channel_parts[0]
+            self.risk = channel_parts[1]
+            self.branch = None
+        else:
+            raise RuntimeError("Channel logic failed for: {!r]".format(channel))
+        self._channel = channel
+
+
+def _get_snap_channel(snap_name: str) -> _Channel:
     """Returns the channel to use for snap_name."""
     env_channel = os.getenv("SNAPCRAFT_BUILD_ENVIRONMENT_CHANNEL_SNAPCRAFT", None)
     if env_channel is not None and snap_name == "snapcraft":
@@ -54,7 +84,7 @@ def _get_snap_channel(snap_name: str) -> str:
     else:
         channel = "latest/stable"
 
-    return channel
+    return _Channel(channel)
 
 
 class _SnapManager:
@@ -144,7 +174,7 @@ def get_assertions(self) -> List[List[str]]:
         if host_snap_info["revision"].startswith("x"):
             return []
 
-        assertions = []  # type: List[List[str]]
+        assertions = list()  # type: List[List[str]]
         assertions.append(
             ["snap-declaration", "snap-name={}".format(host_snap_repo.name)]
         )
@@ -160,10 +190,12 @@ def get_assertions(self) -> List[List[str]]:
     def _set_data(self) -> None:
         op = self.get_op()
         host_snap_repo = self._get_snap_repo()
-        install_cmd = ["snap"]
+
+        install_cmd = []  # type: List[str]
+        snap_revision = None
 
         if op == _SnapOp.INJECT:
-            install_cmd.append("install")
+            install_cmd = ["snap", "install"]
             host_snap_info = host_snap_repo.get_local_snap_info()
             snap_revision = host_snap_info["revision"]
 
@@ -176,20 +208,17 @@ def _set_data(self) -> None:
             snap_file_name = "{}_{}.snap".format(host_snap_repo.name, snap_revision)
             install_cmd.append(os.path.join(self._snap_dir, snap_file_name))
         elif op == _SnapOp.INSTALL or op == _SnapOp.REFRESH:
-            install_cmd.append(op.name.lower())
+            install_cmd = ["snap", op.name.lower()]
             snap_channel = _get_snap_channel(self.snap_name)
-            store_snap_info = storeapi.StoreClient().cpi.get_package(
-                self.snap_name, snap_channel, self._snap_arch
+            store_snap_info = storeapi.StoreClient().cpi.get_info(self.snap_name)
+            snap_channel_map = store_snap_info.get_channel_mapping(
+                risk=snap_channel.risk, track=snap_channel.track, arch=self._snap_arch
             )
-            snap_revision = store_snap_info["revision"]
-            confinement = store_snap_info["confinement"]
-            if confinement == "classic":
+            snap_revision = snap_channel_map.revision
+            if snap_channel_map.confinement == "classic":
                 install_cmd.append("--classic")
-            install_cmd.extend(["--channel", snap_channel])
+            install_cmd.extend(["--channel", snap_channel_map.channel_details.name])
             install_cmd.append(host_snap_repo.name)
-        elif op == _SnapOp.NOP:
-            install_cmd = []
-            snap_revision = None
 
         self.__install_cmd = install_cmd
         self.__revision = snap_revision

snapcraft/plugins/kernel.py

@@ -507,7 +507,12 @@ def _do_check_initrd(self, builtin, modules):
 
     def pull(self):
         super().pull()
-        snapcraft.download("core", "stable", self.os_snap, self.project.deb_arch)
+        snapcraft.download(
+            "core",
+            risk="stable",
+            download_path=self.os_snap,
+            arch=self.project.deb_arch,
+        )
 
     def do_configure(self):
         super().do_configure()

snapcraft/storeapi/_client.py

@@ -1,3 +1,19 @@
+# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
+#
+# Copyright 2016-2019 Canonical Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 import logging
 import requests
 from requests.adapters import HTTPAdapter
@@ -12,6 +28,7 @@
 # Set urllib3's logger to only emit errors, not warnings. Otherwise even
 # retries are printed, and they're nasty.
 logging.getLogger(requests.packages.urllib3.__package__).setLevel(logging.ERROR)
+logger = logging.getLogger(__name__)
 
 
 class Client:
@@ -60,6 +77,11 @@ def request(self, method, url, params=None, headers=None, **kwargs):
             headers = self._snapcraft_headers
 
         final_url = urllib.parse.urljoin(self.root_url, url)
+        logger.debug(
+            "Calling {} with params {} and headers {}".format(
+                final_url, params, headers
+            )
+        )
         try:
             response = self.session.request(
                 method, final_url, headers=headers, params=params, **kwargs

snapcraft/storeapi/_snap_index_client.py

@@ -1,8 +1,25 @@
+# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
+#
+# Copyright 2016-2019 Canonical Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 import contextlib
 import os
 from typing import Dict
 
 from ._client import Client
+from .info import SnapInfo
 
 from . import errors
 from . import logger, _macaroon_auth
@@ -28,7 +45,7 @@ def __init__(self, conf):
             ),
         )
 
-    def get_default_headers(self):
+    def get_default_headers(self, api="v2"):
         """Return default headers for CPI requests.
         Tries to build an 'Authorization' header with local credentials
         if they are available.
@@ -42,44 +59,44 @@ def get_default_headers(self):
 
         branded_store = os.getenv("SNAPCRAFT_UBUNTU_STORE")
         if branded_store:
-            headers["X-Ubuntu-Store"] = branded_store
+            if api == "v2":
+                headers["Snap-Device-Store"] = branded_store
+            elif api == "v1":
+                headers["X-Ubuntu-Store"] = branded_store
+            else:
+                logger.warning("Incorrect API version passed: {!r}.".format(api))
 
         return headers
 
-    def get_package(self, snap_name, channel=None, arch=None):
-        """Get the details for the specified snap.
+    def get_info(self, snap_name: str, *, arch: str = None) -> SnapInfo:
+        """Get the information for the specified snap.
 
         :param str snap_name: Name of the snap.
-        :param str channel: Channel of the snap.
         :param str arch: Architecture of the snap (none by default).
 
-        :return Details for the snap.
-        :rtype: dict
+        :return information for the snap.
+        :rtype: SnapInfo
         """
         headers = self.get_default_headers()
         headers.update(
             {
-                "Accept": "application/hal+json",
-                "X-Ubuntu-Series": constants.DEFAULT_SERIES,
+                "Accept": "application/json",
+                "Snap-Device-Series": constants.DEFAULT_SERIES,
             }
         )
-        if arch:
-            headers["X-Ubuntu-Architecture"] = arch
-
-        params = {
-            # FIXME LP: #1662665
-            "fields": "status,confinement,anon_download_url,download_url,"
-            "download_sha3_384,download_sha512,snap_id,"
-            "revision,release"
-        }
-        if channel is not None:
-            params["channel"] = channel
-        logger.debug("Getting details for {}".format(snap_name))
-        url = "api/v1/snaps/details/{}".format(snap_name)
+
+        params = dict()
+        params["fields"] = "channel-map,snap-id,name,publisher,confinement,revision"
+        if arch is not None:
+            params["architecture"] = arch
+        logger.debug("Getting information for {}".format(snap_name))
+        url = "v2/snaps/info/{}".format(snap_name)
         resp = self.get(url, headers=headers, params=params)
-        if resp.status_code != 200:
-            raise errors.SnapNotFoundError(snap_name, channel, arch)
-        return resp.json()
+        if resp.status_code == 404:
+            raise errors.SnapNotFoundError(snap_name, arch)
+        resp.raise_for_status()
+
+        return SnapInfo(**resp.json())
 
     def get_assertion(
         self, assertion_type: str, snap_id: str
@@ -91,7 +108,7 @@ def get_assertion(
 
         :return Assertion for the snap.
         """
-        headers = self.get_default_headers()
+        headers = self.get_default_headers(api="v1")
         logger.debug("Getting snap-declaration for {}".format(snap_id))
         url = "/api/v1/snaps/assertions/{}/{}/{}".format(
             assertion_type, constants.DEFAULT_SERIES, snap_id