Add tests for micromamba, transmutation and miniforge (continued) (#605)

* Add tests for micromamba, transmuation and miniforge

* unpack

* do it on the ci matrix instead

* simplify

* try naming?

* ternary

* single quotes

* remove general entry in matrix?

* find micromamba on windows too?

* manual path for windows

* add debug mode

* fix transmutation in conda

* pre-commit

* add tests for miniforge

* use double quotes just in case

* allow unbound vars

* shellcheck

* document micromamba & menu_packages compatibility

* trim spaces

* sync docs

* Skip miniforge + micromamba + windows

* trim

* simplify skip logic

* error out if micromamba used on windows, remove ci there

* add news

Co-authored-by: Chris Burr <christopher.burr@cern.ch>
Co-authored-by: Daniel Bast <2790401+dbast@users.noreply.github.com>

.github/workflows/main.yml

@@ -23,12 +23,24 @@ defaults:
     shell: bash
 jobs:
   package:
-    runs-on: ${{ matrix.os }}
+    name: ${{ matrix.os }}, Python ${{ matrix.pyver }}, ${{ matrix.micromamba && 'micromamba' || 'conda-standalone' }}
+    runs-on: ${{ matrix.os }}-latest
     strategy:
       fail-fast: false
       matrix:
-        os: [macos-latest, ubuntu-latest, windows-latest]
+        os: [macos, ubuntu, windows]
         pyver: ["3.7", "3.8", "3.9", "3.10"]
+        include:
+          - os: ubuntu
+            pyver: "3.9"
+            micromamba: true
+          - os: macos
+            pyver: "3.10"
+            micromamba: true
+          # Re-enable once micromamba supports menu creation
+          # - os: windows
+          #   pyver: "3.8"
+          #   micromamba: true
     env:
       PYTHONUNBUFFERED: True
     steps:
@@ -112,12 +124,28 @@ jobs:
           :: Careful with the trailing spaces before the >> redirect!
           echo CONSTRUCTOR_PFX_CERTIFICATE_PASSWORD=1234>> %GITHUB_ENV%
           echo CONSTRUCTOR_SIGNTOOL_PATH=C:\Program Files (x86)\Windows Kits\10\bin\10.0.17763.0\x86\signtool.exe>> %GITHUB_ENV%
+      - name: Set up conda executable
+        run: |
+          source $CONDA/etc/profile.d/conda.sh
+          if [[ "${{ matrix.micromamba }}" != "" ]]; then
+            conda create -yqp ./micromamba -c conda-forge micromamba
+            if [[ ${{ matrix.os }} == "windows" ]]; then
+              echo "CONSTRUCTOR_CONDA_EXE=./micromamba/Library/bin/micromamba.exe" >> $GITHUB_ENV
+            else
+              echo "CONSTRUCTOR_CONDA_EXE=./micromamba/bin/micromamba" >> $GITHUB_ENV
+            fi
+          else
+            conda activate constructor
+            echo "CONSTRUCTOR_CONDA_EXE=$CONDA_PREFIX/standalone_conda/conda.exe" >> $GITHUB_ENV
+          fi
       - name: Run examples and prepare artifacts
         run: |
           source $CONDA/etc/profile.d/conda.sh
           conda activate constructor
           mkdir -p examples_artifacts/
-          python scripts/run_examples.py --keep-artifacts=examples_artifacts/
+          python scripts/run_examples.py \
+            --keep-artifacts=examples_artifacts/ \
+            --conda-exe="${CONSTRUCTOR_CONDA_EXE}"
       - name: Test with conda-libmamba-solver
         run: |
           source $CONDA/etc/profile.d/conda.sh

CONSTRUCT.md

@@ -100,8 +100,8 @@ is contained as a result of resolving the specs for `python 2.7`.
 
 _required:_ no<br/>
 _type:_ list<br/>
-A list of packages with menu items to be instsalled. The packages must have
-necessary metadata in "Menu/<package name>.json").  Menu items are currently
+A list of packages with menu items to be installed. The packages must have
+necessary metadata in `Menu/<package name>.json`). Menu items are currently
 only supported on Windows. By default, all menu items will be installed;
 supplying this list allows a subset to be selected instead.
 

constructor/construct.py

@@ -79,9 +79,9 @@
 is contained as a result of resolving the specs for `python 2.7`.
 '''),
 
-    ('menu_packages',           False, list, '''
-A list of packages with menu items to be instsalled. The packages must have
-necessary metadata in "Menu/<package name>.json").  Menu items are currently
+    ('menu_packages',          False, list, '''
+A list of packages with menu items to be installed. The packages must have
+necessary metadata in `Menu/<package name>.json`). Menu items are currently
 only supported on Windows. By default, all menu items will be installed;
 supplying this list allows a subset to be selected instead.
 '''),

constructor/main.py

@@ -86,6 +86,9 @@ def main_build(dir_path, output_dir='.', platform=cc_platform,
 
     if platform != cc_platform and 'pkg' in itypes and not cc_platform.startswith('osx-'):
         sys.exit("Error: cannot construct a macOS 'pkg' installer on '%s'" % cc_platform)
+    if osname == "win" and "micromamba" in os.path.basename(info['_conda_exe']):
+        # TODO: Remove when shortcut creation is implemented on micromamba
+        sys.exit("Error: micromamba is not supported on Windows installers.")
 
     if verbose:
         print('conda packages download: %s' % info['_download_dir'])
@@ -293,7 +296,7 @@ def main():
                    version='%(prog)s {version}'.format(version=__version__))
 
     p.add_argument('--conda-exe',
-                   help="path to conda executable",
+                   help="path to conda executable (conda-standalone, micromamba)",
                    action="store",
                    metavar="CONDA_EXE")
 

constructor/preconda.py

@@ -20,7 +20,7 @@
 from .conda_interface import (CONDA_INTERFACE_VERSION, Dist, MatchSpec, default_prefix,
                               PrefixData, write_repodata, get_repodata, all_channel_urls)
 from .conda_interface import distro as conda_distro
-from .utils import get_final_channels
+from .utils import get_final_channels, ensure_transmuted_ext
 
 try:
     import json
@@ -127,6 +127,7 @@ def write_files(info, dst_dir):
 
     with open(join(dst_dir, 'urls'), 'w') as fo:
         for url, md5 in all_final_urls_md5s:
+            url = ensure_transmuted_ext(info, url)
             fo.write('%s#%s\n' % (url, md5))
 
     with open(join(dst_dir, 'urls.txt'), 'w') as fo:

constructor/utils.py

@@ -8,7 +8,7 @@
 import sys
 import hashlib
 import math
-from os.path import normpath, islink, isfile, isdir
+from os.path import normpath, islink, isfile, isdir, basename
 from os import sep, unlink
 from shutil import rmtree
 
@@ -123,6 +123,24 @@ def add_condarc(info):
         yield 'EOF'
 
 
+def ensure_transmuted_ext(info, url):
+    """
+    If transmuting, micromamba won't find the dist in the preconda tarball
+    unless it has the (correct and transmuted) extension. Otherwise, the command
+    `micromamba constructor --extract-tarballs` fails.
+    Unfortunately this means the `urls` file might end up containing
+    fake URLs, since those .conda archives might not really exist online,
+    and they were only created locally.
+    """
+    if (
+        info.get("transmute_file_type") == ".conda"
+        and "micromamba" in basename(info.get("_conda_exe", ""))
+    ):
+        if url.lower().endswith(".tar.bz2"):
+            url = url[:-8] + ".conda"
+    return url
+
+
 def get_final_url(info, url):
     mapping = info.get('channels_remap', [])
     for entry in mapping:

examples/miniforge/construct.yaml

@@ -0,0 +1,21 @@
+name: Miniforge3
+version: 4.10.1-0
+company: conda-forge
+
+channels:
+  - conda-forge
+
+write_condarc: True
+keep_pkgs: True
+transmute_file_type: .conda
+
+specs:
+  - python 3.9.*
+  - conda 4.10.1
+  - pip
+  - miniforge_console_shortcut 1.*  # [win]
+
+# Added for extra testing
+installer_type: all
+post_install: test_install.sh  # [unix]
+post_install: test_install.bat  # [win]

examples/miniforge/test_install.bat

@@ -0,0 +1,7 @@
+@ECHO ON
+call "%PREFIX%\Scripts\activate.bat
+conda install -yq jq || exit 1
+conda config --show-sources || exit 1
+conda config --json --show | jq -r ".channels[0]" > temp.txt
+set /p OUTPUT=<temp.txt
+if not "%OUTPUT%" == "conda-forge" exit 1

examples/miniforge/test_install.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -exo pipefail
+
+# shellcheck disable=SC1091
+source "$PREFIX/etc/profile.d/conda.sh"
+conda activate "$PREFIX"
+conda install -yq jq
+conda config --show-sources
+test "$(conda config --json --show | jq -r '.channels[0]')" = "conda-forge"

news/605-micromamba-tests

@@ -0,0 +1,20 @@
+### Enhancements
+
+* <news item>
+
+### Bug fixes
+
+* Add tests for `--conda-exe=<micromamba>` and fix found issues on Linux and macOS.
+  Not supported on Windows yet. (#503, #605)
+
+### Deprecations
+
+* <news item>
+
+### Docs
+
+* <news item>
+
+### Other
+
+* <news item>

scripts/run_examples.py

@@ -1,8 +1,6 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 """Run examples bundled with this repo."""
-
-# Standard library imports
+import argparse
 import os
 import subprocess
 import sys
@@ -11,7 +9,6 @@
 import shutil
 import time
 from datetime import timedelta
-
 from pathlib import Path
 
 from constructor.utils import rm_rf
@@ -32,10 +29,15 @@
 WITH_SPACES = {"extra_files", "noconda", "signing", "scripts"}
 
 
-def _execute(cmd):
+def _execute(cmd, **env_vars):
     print(' '.join(cmd))
     t0 = time.time()
-    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+    if env_vars:
+        env = os.environ.copy()
+        env.update(env_vars)
+    else:
+        env = None
+    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=env)
     try:
         stdout, stderr = p.communicate(timeout=420)
         errored = p.returncode != 0
@@ -45,7 +47,7 @@ def _execute(cmd):
         print('--- TEST TIMEOUT ---')
         errored = True
     t1 = time.time()
-    if errored:
+    if errored or "CONDA_VERBOSITY" in env_vars:
         print(f'--- RETURNCODE: {p.returncode} ---')
         if stdout:
             print('--- STDOUT ---')
@@ -57,7 +59,7 @@ def _execute(cmd):
     return errored
 
 
-def run_examples(keep_artifacts=None):
+def run_examples(keep_artifacts=None, conda_exe=None, debug=False):
     """Run examples bundled with the repository.
 
     Parameters
@@ -93,21 +95,36 @@ def run_examples(keep_artifacts=None):
             if os.path.exists(os.path.join(fpath, 'construct.yaml')):
                 example_paths.append(fpath)
 
-    # NSIS won't error out when running scripts unless we set this custom environment variable
+    # NSIS won't error out when running scripts unless
+    # we set this custom environment variable
     os.environ["NSIS_SCRIPTS_RAISE_ERRORS"] = "1"
 
     parent_output = tempfile.mkdtemp()
     tested_files = set()
     which_errored = {}
     for example_path in sorted(example_paths):
-        print(example_path)
-        print('-' * len(example_path))
-        output_dir = tempfile.mkdtemp(dir=parent_output)
+        example_name = Path(example_path).name
+        test_with_spaces = example_name in WITH_SPACES
+        print(example_name)
+        print('-' * len(example_name))
+        if (
+            sys.platform.startswith("win")
+            and conda_exe
+            and "micromamba" in os.path.basename(conda_exe).lower()
+        ):
+            print(
+                f"! Skipping {example_name}... Shortcut creation on Windows is "
+                "not supported with micromamba."
+            )
+            continue
+        output_dir = tempfile.mkdtemp(prefix=f"{example_name}-", dir=parent_output)
         # resolve path to avoid some issues with TEMPDIR on Windows
         output_dir = str(Path(output_dir).resolve())
-        example_name = Path(example_path).parent.name
-        test_with_spaces = example_name in WITH_SPACES
         cmd = COV_CMD + ['constructor', '-v', example_path, '--output-dir', output_dir]
+        if conda_exe:
+            cmd += ['--conda-exe', conda_exe]
+        if debug:
+            cmd.append("--debug")
         creation_errored = _execute(cmd)
         errored += creation_errored
         for fpath in os.listdir(output_dir):
@@ -118,8 +135,8 @@ def run_examples(keep_artifacts=None):
             test_suffix = "s p a c e s" if test_with_spaces else None
             env_dir = tempfile.mkdtemp(suffix=test_suffix, dir=output_dir)
             rm_rf(env_dir)
-            print('--- Testing %s' % fpath)
             fpath = os.path.join(output_dir, fpath)
+            print('--- Testing', os.path.basename(fpath))
             if ext == 'sh':
                 cmd = ['/bin/sh', fpath, '-b', '-p', env_dir]
             elif ext == 'pkg':
@@ -144,7 +161,8 @@ def run_examples(keep_artifacts=None):
                 # would be enough too :)
                 # This is why we have this weird .split() thingy down here:
                 cmd = ['cmd.exe', '/c', 'start', '/wait', fpath, '/S', *f'/D={env_dir}'.split()]
-            test_errored = _execute(cmd)
+            env = {"CONDA_VERBOSITY": "3"} if debug else {}
+            test_errored = _execute(cmd, **env)
             # Windows EXEs never throw a non-0 exit code, so we need to check the logs,
             # which are only written if a special NSIS build is used
             win_error_lines = []
@@ -220,6 +238,9 @@ def run_examples(keep_artifacts=None):
                     which_errored.setdefault(example_path, []).append("Could not find uninstaller!")
 
             if keep_artifacts:
+                dest = os.path.join(keep_artifacts, os.path.basename(fpath))
+                if os.path.isfile(dest):
+                    os.unlink(dest)
                 shutil.move(fpath, keep_artifacts)
         if creation_errored:
             which_errored.setdefault(example_path, []).append("Could not create installer!")
@@ -231,18 +252,29 @@ def run_examples(keep_artifacts=None):
         for installer, reasons in which_errored.items():
             print(f"+ {os.path.basename(installer)}")
             for reason in reasons:
-                print(f"---> {os.path.basename(reason)}")
-        print('Assets saved in: %s' % parent_output)
+                print(f"---> {reason}")
+        print('Assets saved in:', keep_artifacts or parent_output)
     else:
         print('All examples ran successfully!')
         shutil.rmtree(parent_output)
     return errored
 
 
+def cli():
+    p = argparse.ArgumentParser()
+    p.add_argument("--keep-artifacts")
+    p.add_argument("--conda-exe")
+    p.add_argument("--debug", action="store_true", default=False)
+    return p.parse_args()
+
+
 if __name__ == '__main__':
-    if len(sys.argv) >= 2 and sys.argv[1].startswith('--keep-artifacts='):
-        keep_artifacts = sys.argv[1].split("=")[1]
-    else:
-        keep_artifacts = None
-    n_errors = run_examples(keep_artifacts)
+    args = cli()
+    if args.conda_exe:
+        assert os.path.isfile(args.conda_exe)
+    n_errors = run_examples(
+        keep_artifacts=args.keep_artifacts,
+        conda_exe=args.conda_exe,
+        debug=args.debug
+    )
     sys.exit(n_errors)