Merge pull request #1577 from Klaim/klaim/env-lockfile

environment lockfile parsing & usage in `create` and `install` sub-commands

libmamba/CMakeLists.txt

@@ -133,6 +133,7 @@ set(LIBMAMBA_SOURCES
     ${LIBMAMBA_SOURCE_DIR}/core/util_os.cpp
     ${LIBMAMBA_SOURCE_DIR}/core/validate.cpp
     ${LIBMAMBA_SOURCE_DIR}/core/virtual_packages.cpp
+    ${LIBMAMBA_SOURCE_DIR}/core/env_lockfile.cpp
     # API (high-level)
     ${LIBMAMBA_SOURCE_DIR}/api/c_api.cpp
     ${LIBMAMBA_SOURCE_DIR}/api/channel_loader.cpp
@@ -192,8 +193,10 @@ set(LIBMAMBA_HEADERS
     ${LIBMAMBA_INCLUDE_DIR}/mamba/core/util.hpp
     ${LIBMAMBA_INCLUDE_DIR}/mamba/core/util_os.hpp
     ${LIBMAMBA_INCLUDE_DIR}/mamba/core/util_random.hpp
+    ${LIBMAMBA_INCLUDE_DIR}/mamba/core/util_scope.hpp
     ${LIBMAMBA_INCLUDE_DIR}/mamba/core/validate.hpp
     ${LIBMAMBA_INCLUDE_DIR}/mamba/core/virtual_packages.hpp
+    ${LIBMAMBA_INCLUDE_DIR}/mamba/core/env_lockfile.hpp
     # API (high-level)
     ${LIBMAMBA_INCLUDE_DIR}/mamba/api/c_api.h
     ${LIBMAMBA_INCLUDE_DIR}/mamba/api/channel_loader.hpp

libmamba/include/mamba/api/install.hpp

@@ -33,6 +33,7 @@ namespace mamba
                        int is_retry = 0);
 
     void install_explicit_specs(const std::vector<std::string>& specs, bool create_env = false);
+    void install_lockfile_specs(const fs::path& lockfile_specs, bool create_env = false);
 
     namespace detail
     {

libmamba/include/mamba/core/context.hpp

@@ -13,6 +13,7 @@
 #include <map>
 #include <string>
 #include <vector>
+#include <optional>
 
 #define ROOT_ENV_NAME "base"
 
@@ -111,6 +112,7 @@ namespace mamba
         // TODO check writable and add other potential dirs
         std::vector<fs::path> envs_dirs;
         std::vector<fs::path> pkgs_dirs;
+        std::optional<fs::path> env_lockfile;
 
         bool use_index_cache = false;
         std::size_t local_repodata_ttl = 1;  // take from header

libmamba/include/mamba/core/env_lockfile.hpp

@@ -0,0 +1,123 @@
+// Copyright (c) 2022, QuantStack and Mamba Contributors
+//
+// Distributed under the terms of the BSD 3-Clause License.
+//
+// The full license is in the file LICENSE, distributed with this software.
+
+
+#ifndef MAMBA_CORE_ENVIRONMENT_LOCKFILE_HPP
+#define MAMBA_CORE_ENVIRONMENT_LOCKFILE_HPP
+
+#include <string>
+#include <typeindex>
+#include <optional>
+#include <unordered_map>
+
+#include <tl/expected.hpp>
+
+#include "fsutil.hpp"
+#include "package_info.hpp"
+#include "error_handling.hpp"
+
+namespace mamba
+{
+
+    enum class file_parsing_error_code
+    {
+        unknown_failure,     /// Something failed while parsing but we can't identify what.
+        unsuported_version,  /// The version of the file does not matched supported ver.
+        parsing_failure,     /// The content of the file doesnt match the expected format/language
+                             /// structure or constraints.
+        invalid_data,        /// The structure of the data in the file is fine but some fields have
+                             /// invalid values for our purpose.
+    };
+
+    struct EnvLockFileError
+    {
+        file_parsing_error_code parsing_error_code = file_parsing_error_code::unknown_failure;
+        std::optional<std::type_index> yaml_error_type{};
+
+        static const EnvLockFileError& get_details(const mamba_error& error)
+        {
+            return std::any_cast<const EnvLockFileError&>(error.data());
+        }
+
+        template <typename StringT>
+        static mamba_error make_error(file_parsing_error_code error_code,
+                                      StringT&& msg,
+                                      std::optional<std::type_index> yaml_error_type = std::nullopt)
+        {
+            return mamba_error{ std::forward<StringT>(msg),
+                                mamba_error_code::env_lockfile_parsing_failed,
+                                EnvLockFileError{ error_code, yaml_error_type } };
+        }
+    };
+
+    class EnvironmentLockFile
+    {
+    public:
+        struct Channel
+        {
+            std::string url;
+            std::vector<std::string> used_env_vars;
+        };
+
+        struct Meta
+        {
+            std::unordered_map<std::string, std::string> content_hash;
+            std::vector<Channel> channels;
+            std::vector<std::string> platforms;
+            std::vector<std::string> sources;
+        };
+
+        struct Package
+        {
+            mamba::PackageInfo info;
+            bool is_optional = false;
+            std::string category;
+            std::string manager;
+            std::string platform;
+        };
+
+        EnvironmentLockFile(Meta metadata, std::vector<Package> packages)
+            : metadata(std::move(metadata))
+            , packages(std::move(packages))
+        {
+        }
+
+        std::vector<PackageInfo> get_packages_for(std::string_view category,
+                                                  std::string_view platform,
+                                                  std::string_view manager) const;
+
+        const std::vector<Package>& get_all_packages() const
+        {
+            return packages;
+        }
+        const Meta& get_metadata() const
+        {
+            return metadata;
+        }
+
+    private:
+        Meta metadata;
+        std::vector<Package> packages;
+    };
+
+    /// Read an environment lock YAML file and returns it's structured content or an error if
+    /// failed.
+    tl::expected<EnvironmentLockFile, mamba_error> read_environment_lockfile(
+        const fs::path& lockfile_location);
+
+
+    /// Returns `true` if the filename matches names of files which should be interpreted as conda
+    /// environment lockfile. NOTE: this does not check if the file exists.
+    inline bool is_env_lockfile_name(const std::string_view filename)
+    {
+        return ends_with(filename, "-lock.yml") || ends_with(filename, "-lock.yaml");
+    }
+
+
+}
+
+
+#endif

libmamba/include/mamba/core/error_handling.hpp

@@ -18,12 +18,13 @@ namespace mamba
     enum class mamba_error_code
     {
         unknown,
+        aggregated,
         prefix_data_not_loaded,
         subdirdata_not_loaded,
         cache_not_loaded,
         repodata_not_loaded,
         configurable_bad_cast,
-        aggregated
+        env_lockfile_parsing_failed,
     };
 
     class mamba_error : public std::runtime_error

libmamba/include/mamba/core/transaction.hpp

@@ -26,6 +26,7 @@
 #include "repo.hpp"
 #include "thread_utils.hpp"
 #include "transaction_context.hpp"
+#include "env_lockfile.hpp"
 
 extern "C"
 {
@@ -123,6 +124,11 @@ namespace mamba
                      MultiPackageCache& caches);
         MTransaction(MSolver& solver, MultiPackageCache& caches);
 
+        // Only use if the packages have been solved previously already.
+        MTransaction(MPool& pool,
+                     const std::vector<PackageInfo>& packages,
+                     MultiPackageCache& caches);
+
         ~MTransaction();
 
         MTransaction(const MTransaction&) = delete;
@@ -167,6 +173,10 @@ namespace mamba
     MTransaction create_explicit_transaction_from_urls(MPool& pool,
                                                        const std::vector<std::string>& urls,
                                                        MultiPackageCache& package_caches);
+
+    MTransaction create_explicit_transaction_from_lockfile(MPool& pool,
+                                                           const fs::path& env_lockfile_path,
+                                                           MultiPackageCache& package_caches);
 }  // namespace mamba
 
 #endif  // MAMBA_TRANSACTION_HPP

libmamba/include/mamba/core/util.hpp

@@ -326,6 +326,12 @@ namespace mamba
     std::tuple<std::vector<std::string>, std::unique_ptr<TemporaryFile>> prepare_wrapped_call(
         const fs::path& prefix, const std::vector<std::string>& cmd);
 
+    /// Returns `true` if the filename matches names of files which should be interpreted as YAML.
+    /// NOTE: this does not check if the file exists.
+    inline bool is_yaml_file_name(const std::string_view filename)
+    {
+        return ends_with(filename, ".yml") || ends_with(filename, ".yaml");
+    }
 
 }  // namespace mamba
 

libmamba/include/mamba/core/util_scope.hpp

@@ -0,0 +1,46 @@
+
+#ifndef MAMBA_CORE_UTIL_SCOPE_HPP
+#define MAMBA_CORE_UTIL_SCOPE_HPP
+
+#include <stdexcept>
+#include "spdlog/fmt/fmt.h"
+#include "mamba/core/output.hpp"
+
+namespace mamba
+{
+
+    template <typename F>
+    struct on_scope_exit
+    {
+        F func;
+
+        explicit on_scope_exit(F&& f)
+            : func(std::forward<F>(f))
+        {
+        }
+
+        ~on_scope_exit()
+        {
+            try
+            {
+                func();
+            }
+            catch (const std::exception& ex)
+            {
+                LOG_ERROR << fmt::format("Scope exit error (catched and ignored): {}", ex.what());
+            }
+            catch (...)
+            {
+                LOG_ERROR << "Scope exit unknown error (catched and ignored)";
+            }
+        }
+
+        // Deactivate copy & move until we implement moves
+        on_scope_exit(const on_scope_exit&) = delete;
+        on_scope_exit& operator=(const on_scope_exit&) = delete;
+        on_scope_exit(on_scope_exit&&) = delete;
+        on_scope_exit& operator=(on_scope_exit&&) = delete;
+    };
+}
+
+#endif

libmamba/src/api/create.cpp

@@ -59,7 +59,13 @@ namespace mamba
                 detail::store_platform_config(ctx.target_prefix, ctx.platform);
         }
 
-        if (!create_specs.empty())
+        if (Context::instance().env_lockfile)
+        {
+            const auto lockfile_path = Context::instance().env_lockfile.value();
+            LOG_DEBUG << "Lockfile: " << lockfile_path.string();
+            install_lockfile_specs(lockfile_path, true);
+        }
+        else if (!create_specs.empty())
         {
             if (use_explicit)
                 install_explicit_specs(create_specs, true);