Download (#2844)

* Refactored Subdir Metadata

* Plugged downloader into subdirdata

* Simplified Donwloader APIs

* Added DownloadMonitor and plugged it into channel_loader

* Simplified subdir API

* Fixed python bindings

* Changes according to review (on going)

* Added json parsers to subdirmetadata

libmamba/CMakeLists.txt

@@ -147,6 +147,7 @@ set(LIBMAMBA_SOURCES
     ${LIBMAMBA_SOURCE_DIR}/core/channel.cpp
     ${LIBMAMBA_SOURCE_DIR}/core/context.cpp
     ${LIBMAMBA_SOURCE_DIR}/core/download.cpp
+    ${LIBMAMBA_SOURCE_DIR}/core/download_progress_bar.cpp
     ${LIBMAMBA_SOURCE_DIR}/core/environment.cpp
     ${LIBMAMBA_SOURCE_DIR}/core/environments_manager.cpp
     ${LIBMAMBA_SOURCE_DIR}/core/error_handling.cpp
@@ -209,6 +210,7 @@ endforeach()
 set(LIBMAMBA_PUBLIC_HEADERS
     ${LIBMAMBA_INCLUDE_DIR}/mamba/version.hpp
     # Utility library
+    ${LIBMAMBA_INCLUDE_DIR}/mamba/util/json.hpp
     ${LIBMAMBA_INCLUDE_DIR}/mamba/util/deprecation.hpp
     ${LIBMAMBA_INCLUDE_DIR}/mamba/util/build.hpp
     ${LIBMAMBA_INCLUDE_DIR}/mamba/util/cast.hpp
@@ -238,6 +240,7 @@ set(LIBMAMBA_PUBLIC_HEADERS
     ${LIBMAMBA_INCLUDE_DIR}/mamba/core/palette.hpp
     ${LIBMAMBA_INCLUDE_DIR}/mamba/core/context.hpp
     ${LIBMAMBA_INCLUDE_DIR}/mamba/core/download.hpp
+    ${LIBMAMBA_INCLUDE_DIR}/mamba/core/download_progress_bar.hpp
     ${LIBMAMBA_INCLUDE_DIR}/mamba/core/environment.hpp
     ${LIBMAMBA_INCLUDE_DIR}/mamba/core/environments_manager.hpp
     ${LIBMAMBA_INCLUDE_DIR}/mamba/core/error_handling.hpp

libmamba/include/mamba/core/download.hpp

@@ -15,6 +15,7 @@
 #include <tl/expected.hpp>
 
 #include "mamba/core/context.hpp"
+#include "mamba/core/error_handling.hpp"
 
 namespace mamba
 {
@@ -23,7 +24,7 @@ namespace mamba
         int http_status = 0;
         std::string effective_url = "";
         std::size_t downloaded_size = 0;
-        std::size_t average_speed = 0;
+        std::size_t average_speed_Bps = 0;
     };
 
     struct DownloadSuccess
@@ -48,6 +49,7 @@ namespace mamba
     {
         std::size_t downloaded_size = 0;
         std::size_t total_to_download = 0;
+        std::size_t speed_Bps = 0;
     };
 
     using DownloadEvent = std::variant<DownloadProgress, DownloadError, DownloadSuccess>;
@@ -57,7 +59,7 @@ namespace mamba
         using progress_callback_t = std::function<void(const DownloadEvent&)>;
 
         // TODO: remove these functions when we plug a library with continuation
-        using on_success_callback_t = std::function<bool(const DownloadSuccess&)>;
+        using on_success_callback_t = std::function<expected_t<void>(const DownloadSuccess&)>;
         using on_failure_callback_t = std::function<void(const DownloadError&)>;
 
         std::string name;
@@ -66,8 +68,8 @@ namespace mamba
         bool head_only;
         bool ignore_failure;
         std::optional<std::size_t> expected_size = std::nullopt;
-        std::optional<std::string> if_none_match = std::nullopt;
-        std::optional<std::string> if_modified_since = std::nullopt;
+        std::optional<std::string> etag = std::nullopt;
+        std::optional<std::string> last_modified = std::nullopt;
 
         std::optional<progress_callback_t> progress = std::nullopt;
         std::optional<on_success_callback_t> on_success = std::nullopt;
@@ -82,29 +84,53 @@ namespace mamba
         );
     };
 
-    using DownloadRequestList = std::vector<DownloadRequest>;
-
-    struct MultiDownloadRequest
-    {
-        DownloadRequestList requests;
-    };
+    using MultiDownloadRequest = std::vector<DownloadRequest>;
 
     using DownloadResult = tl::expected<DownloadSuccess, DownloadError>;
-    using DownloadResultList = std::vector<DownloadResult>;
-
-    struct MultiDownloadResult
-    {
-        DownloadResultList results;
-    };
+    using MultiDownloadResult = std::vector<DownloadResult>;
 
     struct DownloadOptions
     {
+        using termination_function = std::optional<std::function<void()>>;
+
         bool fail_fast = false;
         bool sort = true;
+        termination_function on_unexpected_termination = std::nullopt;
+    };
+
+    class DownloadMonitor
+    {
+    public:
+
+        virtual ~DownloadMonitor() = default;
+
+        DownloadMonitor(const DownloadMonitor&) = delete;
+        DownloadMonitor& operator=(const DownloadMonitor&) = delete;
+        DownloadMonitor(DownloadMonitor&&) = delete;
+        DownloadMonitor& operator=(DownloadMonitor&&) = delete;
+
+        void observe(MultiDownloadRequest& requests, DownloadOptions& options);
+        void on_done();
+        void on_unexpected_termination();
+
+    protected:
+
+        DownloadMonitor() = default;
+
+    private:
+
+        virtual void observe_impl(MultiDownloadRequest& requests, DownloadOptions& options) = 0;
+        virtual void on_done_impl() = 0;
+        virtual void on_unexpected_termination_impl() = 0;
     };
 
-    MultiDownloadResult
-    download(MultiDownloadRequest requests, const Context& context, DownloadOptions options = {});
+    MultiDownloadResult download(
+        MultiDownloadRequest requests,
+        const Context& context,
+        DownloadOptions options = {},
+        DownloadMonitor* monitor = nullptr
+    );
+
 }
 
 #endif

libmamba/include/mamba/core/download_progress_bar.hpp

@@ -0,0 +1,65 @@
+// Copyright (c) 2019, QuantStack and Mamba Contributors
+//
+// Distributed under the terms of the BSD 3-Clause License.
+//
+// The full license is in the file LICENSE, distributed with this software.
+
+#ifndef MAMBA_CORE_DOWNLOAD_PROGRESS_BAR_HPP
+#define MAMBA_CORE_DOWNLOAD_PROGRESS_BAR_HPP
+
+#include <chrono>
+#include <functional>
+#include <vector>
+
+#include "mamba/core/context.hpp"
+#include "mamba/core/download.hpp"
+
+namespace mamba
+{
+    struct MonitorOptions
+    {
+        bool checking_download = false;
+        bool no_clear_progress_bar = false;
+    };
+
+    class DownloadProgressBar : public DownloadMonitor
+    {
+    public:
+
+        static bool can_monitor(const Context& context);
+
+        explicit DownloadProgressBar(MonitorOptions options = {});
+        virtual ~DownloadProgressBar() = default;
+
+        DownloadProgressBar(const DownloadProgressBar&) = delete;
+        DownloadProgressBar& operator=(const DownloadProgressBar&) = delete;
+
+        DownloadProgressBar(DownloadProgressBar&&) = delete;
+        DownloadProgressBar& operator=(DownloadProgressBar&&) = delete;
+
+        void reset_options(MonitorOptions options);
+
+    private:
+
+        void observe_impl(MultiDownloadRequest& requests, DownloadOptions& options) override;
+        void on_done_impl() override;
+        void on_unexpected_termination_impl() override;
+
+        void update_progress_bar(std::size_t index, const DownloadEvent& event);
+        void update_progress_bar(std::size_t index, const DownloadProgress& progress);
+        void update_progress_bar(std::size_t index, const DownloadError& error);
+        void update_progress_bar(std::size_t index, const DownloadSuccess& success);
+
+        void complete_checking_progress_bar(std::size_t index);
+
+        std::function<void(ProgressBarRepr&)> download_repr(std::size_t index);
+
+        using time_point = std::chrono::steady_clock::time_point;
+        std::vector<time_point> m_throttle_time;
+        std::vector<ProgressProxy> m_progress_bar;
+        MonitorOptions m_options;
+    };
+
+}
+
+#endif