From 80604d06d0c45f5126522f2c67cd074955663f17 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Sun, 13 Mar 2022 12:40:19 +0100 Subject: [PATCH 01/25] Add tests for micromamba, transmuation and miniforge --- examples/miniforge/construct.yaml | 16 ++++++++++ scripts/run_examples.py | 51 +++++++++++++++++++++++-------- 2 files changed, 54 insertions(+), 13 deletions(-) create mode 100644 examples/miniforge/construct.yaml diff --git a/examples/miniforge/construct.yaml b/examples/miniforge/construct.yaml new file mode 100644 index 00000000..7c1a8cf5 --- /dev/null +++ b/examples/miniforge/construct.yaml @@ -0,0 +1,16 @@ +name: Miniforge3 +version: 4.10.1-0 +company: conda-forge + +channels: + - conda-forge + +write_condarc: True +keep_pkgs: True +transmute_file_type: .conda + +specs: + - python 3.9.* + - conda 4.10.1 + - pip + - miniforge_console_shortcut 1.* # [win] diff --git a/scripts/run_examples.py b/scripts/run_examples.py index d7e67630..2f239111 100644 --- a/scripts/run_examples.py +++ b/scripts/run_examples.py @@ -1,16 +1,15 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- """Run examples bundled with this repo.""" - -# Standard library imports import os -import subprocess import sys import tempfile import platform import shutil +from itertools import product +from subprocess import PIPE, Popen from constructor.utils import rm_rf +from ruamel_yaml import round_trip_load, round_trip_dump try: import coverage # noqa @@ -29,15 +28,14 @@ def _execute(cmd): print(' '.join(cmd)) - p = subprocess.Popen(cmd, stderr=subprocess.PIPE) + p = Popen(cmd, stdout=PIPE, stderr=PIPE, universal_newlines=True) print('--- STDOUT ---') - _, stderr = p.communicate() + stdout, stderr = p.communicate() + print(stdout.strip()) if stderr: print('--- STDERR ---') - if PY3: - stderr = stderr.decode() print(stderr.strip()) - return p.returncode != 0 + return p.returncode != 0, stdout.strip() def run_examples(keep_artifacts=None): @@ -54,6 +52,19 @@ def run_examples(keep_artifacts=None): int Number of failed examples """ + conda_exes = [None] + if platform.system() != 'Windows': + env_name = "constructor-testing-micromamba" + which_cmd = ["conda", "run", "--name", env_name, "which", "micromamba"] + errored, micromamba_path = _execute(which_cmd) + if errored: + _execute(["conda", "create", "--name", env_name, "--yes", "-c", "conda-forge", "micromamba"]) + errored, micromamba_path = _execute(which_cmd) + if errored: + raise NotImplementedError("Failed to find micromamba") + print("Found micromamba at:", micromamba_path) + conda_exes += [micromamba_path] + example_paths = [] errored = 0 @@ -71,12 +82,26 @@ def run_examples(keep_artifacts=None): parent_output = tempfile.mkdtemp() tested_files = set() - for example_path in sorted(example_paths): + for example_path, conda_exe in product(sorted(example_paths), conda_exes): print(example_path) print('-' * len(example_path)) output_dir = tempfile.mkdtemp(dir=parent_output) - cmd = COV_CMD + ['constructor', example_path, '--output-dir', output_dir] - errored += _execute(cmd) + input_dir = os.path.join(output_dir, "input-files") + cmd = COV_CMD + ['constructor', input_dir, '--output-dir', output_dir] + + # Copy the input to a temporary directory so we can modify it if needed + shutil.copytree(example_path, input_dir) + if conda_exe: + with open(os.path.join(input_dir, "construct.yaml")) as fp: + data = round_trip_load(fp) + data["name"] = f"{data['name']}-{os.path.basename(conda_exe)}" + with open(os.path.join(input_dir, "construct.yaml"), "wt") as fp: + round_trip_dump(data, fp) + print("Setting conda.exe to:", conda_exe) + cmd += ['--conda-exe', conda_exe] + + # Create the installer + errored += _execute(cmd)[0] for fpath in os.listdir(output_dir): ext = fpath.rsplit('.', 1)[-1] if fpath in tested_files or ext not in ('sh', 'exe', 'pkg'): @@ -95,7 +120,7 @@ def run_examples(keep_artifacts=None): cmd = ['pkgutil', '--expand', fpath, env_dir] elif ext == 'exe': cmd = ['cmd.exe', '/c', 'start', '/wait', fpath, '/S', '/D=%s' % env_dir] - errored += _execute(cmd) + errored += _execute(cmd)[0] if keep_artifacts: shutil.move(fpath, keep_artifacts) print('') From ecc5abb32827c5ec4d1b4069a2da5fb4739cfcf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Wed, 11 Jan 2023 19:23:12 +0100 Subject: [PATCH 02/25] unpack --- scripts/run_examples.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/run_examples.py b/scripts/run_examples.py index f6a6dd8d..35551ab5 100644 --- a/scripts/run_examples.py +++ b/scripts/run_examples.py @@ -131,7 +131,7 @@ def run_examples(keep_artifacts=None): output_dir = tempfile.mkdtemp(prefix=f"{test_key}-", dir=parent_output) # resolve path to avoid some issues with TEMPDIR on Windows output_dir = str(Path(output_dir).resolve()) - creation_errored = _execute(cmd) + creation_errored, *_ = _execute(cmd) errored += creation_errored for fpath in os.listdir(output_dir): ext = fpath.rsplit('.', 1)[-1] @@ -168,7 +168,7 @@ def run_examples(keep_artifacts=None): # the point is to just have spaces in the installation path -- one would be enough too :) # This is why we have this weird .split() thingy down here: cmd = ['cmd.exe', '/c', 'start', '/wait', fpath, '/S', *f'/D={env_dir}'.split()] - test_errored = _execute(cmd) + test_errored, *_ = _execute(cmd) # Windows EXEs never throw a non-0 exit code, so we need to check the logs, # which are only written if a special NSIS build is used win_error_lines = [] @@ -220,7 +220,7 @@ def run_examples(keep_artifacts=None): # the tempdir cleanup later f"/S _?={env_dir}" ] - test_errored = _execute(cmd) + test_errored, *_ = _execute(cmd) errored += test_errored if test_errored: which_errored.setdefault(example_path, []).append( From 69f495b57e89ccf984753a2999d5260f53f79b61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Wed, 11 Jan 2023 19:51:23 +0100 Subject: [PATCH 03/25] do it on the ci matrix instead --- .github/workflows/main.yml | 24 +++++++++++++- scripts/run_examples.py | 68 ++++++++++++++------------------------ 2 files changed, 47 insertions(+), 45 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 15d0ecfd..25f8eb6c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -29,6 +29,16 @@ jobs: matrix: os: [macos-latest, ubuntu-latest, windows-latest] pyver: ["3.7", "3.8", "3.9", "3.10"] + include: + - os: ubuntu-latest + pyver: "3.9" + micromamba: true + - os: macos-latest + pyver: "3.10" + micromamba: true + - os: windows-latest + pyver: "3.8" + micromamba: true env: PYTHONUNBUFFERED: True steps: @@ -112,12 +122,24 @@ jobs: :: Careful with the trailing spaces before the >> redirect! echo CONSTRUCTOR_PFX_CERTIFICATE_PASSWORD=1234>> %GITHUB_ENV% echo CONSTRUCTOR_SIGNTOOL_PATH=C:\Program Files (x86)\Windows Kits\10\bin\10.0.17763.0\x86\signtool.exe>> %GITHUB_ENV% + - name: Set up conda executable + run: | + source $CONDA/etc/profile.d/conda.sh + if [[ "${{ matrix.micromamba }}" != "" ]]; then + conda create -yqp ./micromamba -c conda-forge micromamba + echo "CONSTRUCTOR_CONDA_EXE=$(pwd)/micromamba/bin/micromamba" >> $GITHUB_ENV + else + conda activate constructor + echo "CONSTRUCTOR_CONDA_EXE=$CONDA_PREFIX/standalone_conda/conda.exe" >> $GITHUB_ENV + fi - name: Run examples and prepare artifacts run: | source $CONDA/etc/profile.d/conda.sh conda activate constructor mkdir -p examples_artifacts/ - python scripts/run_examples.py --keep-artifacts=examples_artifacts/ + python scripts/run_examples.py \ + --keep-artifacts=examples_artifacts/ \ + --conda-exe="${CONSTRUCTOR_CONDA_EXE}" - name: Test with conda-libmamba-solver run: | source $CONDA/etc/profile.d/conda.sh diff --git a/scripts/run_examples.py b/scripts/run_examples.py index 35551ab5..1c5d73e0 100644 --- a/scripts/run_examples.py +++ b/scripts/run_examples.py @@ -1,5 +1,6 @@ #!/usr/bin/env python """Run examples bundled with this repo.""" +import argparse import os import sys import tempfile @@ -8,12 +9,9 @@ import time import subprocess from datetime import timedelta -from itertools import product - from pathlib import Path from constructor.utils import rm_rf -from ruamel_yaml import round_trip_load, round_trip_dump try: import coverage # noqa @@ -55,7 +53,7 @@ def _execute(cmd): return errored, stdout.strip(), stderr.strip() -def run_examples(keep_artifacts=None): +def run_examples(keep_artifacts=None, conda_exe=None): """Run examples bundled with the repository. Parameters @@ -69,19 +67,6 @@ def run_examples(keep_artifacts=None): int Number of failed examples """ - conda_exes = [None] - if platform.system() != 'Windows': - env_name = "constructor-testing-micromamba" - which_cmd = ["conda", "run", "--name", env_name, "which", "micromamba"] - errored, micromamba_path, _ = _execute(which_cmd) - if errored: - _execute(["conda", "create", "--name", env_name, "--yes", "-c", "conda-forge", "micromamba"]) - errored, micromamba_path, _ = _execute(which_cmd) - if errored: - micromamba_path = "micromamba-could-not-be-found" - print("Found micromamba at:", micromamba_path) - conda_exes.append(micromamba_path) - if sys.platform.startswith("win") and "NSIS_USING_LOG_BUILD" not in os.environ: print( "! Warning !" @@ -110,27 +95,18 @@ def run_examples(keep_artifacts=None): parent_output = tempfile.mkdtemp() tested_files = set() which_errored = {} - for example_path, conda_exe in product(sorted(example_paths), conda_exes): - example_name = Path(example_path).parent.name + for example_path in sorted(example_paths): + example_name = Path(example_path).name test_with_spaces = example_name in WITH_SPACES - test_key = example_name - cmd = COV_CMD + ['constructor', '-v', example_path] - if conda_exe: - if not os.path.isfile(conda_exe): - errored +=1 - which_errored.setdefault( - (example_path, conda_exe), [] - ).append("Could not find micromamba!") - continue - cmd += ["--conda-exe", conda_exe] - test_key = f"{example_name}+{os.path.basename(conda_exe)}" - - print(test_key) - print('-' * len(test_key)) + print(example_name) + print('-' * len(example_name)) - output_dir = tempfile.mkdtemp(prefix=f"{test_key}-", dir=parent_output) + output_dir = tempfile.mkdtemp(prefix=f"{example_name}-", dir=parent_output) # resolve path to avoid some issues with TEMPDIR on Windows output_dir = str(Path(output_dir).resolve()) + cmd = COV_CMD + ['constructor', '-v', example_path, '--output-dir', output_dir] + if conda_exe: + cmd += ['--conda-exe', conda_exe] creation_errored, *_ = _execute(cmd) errored += creation_errored for fpath in os.listdir(output_dir): @@ -141,10 +117,8 @@ def run_examples(keep_artifacts=None): test_suffix = "s p a c e s" if test_with_spaces else None env_dir = tempfile.mkdtemp(suffix=test_suffix, dir=output_dir) rm_rf(env_dir) - fpath = Path(output_dir) / fpath - if conda_exe: - fpath = fpath.rename(fpath.parent / f"{fpath.stem}.micromamba{fpath.suffix}") - print('--- Testing %s' % fpath.name) + fpath = os.path.join(output_dir, fpath) + print('--- Testing', os.path.basename(fpath)) if ext == 'sh': cmd = ['/bin/sh', fpath, '-b', '-p', env_dir] elif ext == 'pkg': @@ -246,7 +220,7 @@ def run_examples(keep_artifacts=None): if creation_errored: which_errored.setdefault(example_path, []).append("Could not create installer!") print() - + break print("-------------------------------") if errored: print('Some examples failed:') @@ -261,10 +235,16 @@ def run_examples(keep_artifacts=None): return errored +def cli(): + p = argparse.ArgumentParser() + p.add_argument("--keep-artifacts") + p.add_argument("--conda-exe") + return p.parse_args() + + if __name__ == '__main__': - if len(sys.argv) >=2 and sys.argv[1].startswith('--keep-artifacts='): - keep_artifacts = sys.argv[1].split("=")[1] - else: - keep_artifacts = None - n_errors = run_examples(keep_artifacts) + args = cli() + if args.conda_exe: + assert os.path.isfile(args.conda_exe) + n_errors = run_examples(keep_artifacts=args.keep_artifacts, conda_exe=args.conda_exe) sys.exit(n_errors) From a510b40e0ac8f96c9c5c7082b34a870bff12e177 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Wed, 11 Jan 2023 19:56:45 +0100 Subject: [PATCH 04/25] simplify --- scripts/run_examples.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/run_examples.py b/scripts/run_examples.py index 1c5d73e0..cbefd279 100644 --- a/scripts/run_examples.py +++ b/scripts/run_examples.py @@ -2,12 +2,12 @@ """Run examples bundled with this repo.""" import argparse import os +import subprocess import sys import tempfile import platform import shutil import time -import subprocess from datetime import timedelta from pathlib import Path @@ -50,7 +50,7 @@ def _execute(cmd): print('--- STDERR ---') print(stderr) print('--- Done in', timedelta(seconds=t1 - t0)) - return errored, stdout.strip(), stderr.strip() + return errored def run_examples(keep_artifacts=None, conda_exe=None): @@ -107,7 +107,7 @@ def run_examples(keep_artifacts=None, conda_exe=None): cmd = COV_CMD + ['constructor', '-v', example_path, '--output-dir', output_dir] if conda_exe: cmd += ['--conda-exe', conda_exe] - creation_errored, *_ = _execute(cmd) + creation_errored = _execute(cmd) errored += creation_errored for fpath in os.listdir(output_dir): ext = fpath.rsplit('.', 1)[-1] @@ -142,7 +142,7 @@ def run_examples(keep_artifacts=None, conda_exe=None): # the point is to just have spaces in the installation path -- one would be enough too :) # This is why we have this weird .split() thingy down here: cmd = ['cmd.exe', '/c', 'start', '/wait', fpath, '/S', *f'/D={env_dir}'.split()] - test_errored, *_ = _execute(cmd) + test_errored = _execute(cmd) # Windows EXEs never throw a non-0 exit code, so we need to check the logs, # which are only written if a special NSIS build is used win_error_lines = [] @@ -194,7 +194,7 @@ def run_examples(keep_artifacts=None, conda_exe=None): # the tempdir cleanup later f"/S _?={env_dir}" ] - test_errored, *_ = _execute(cmd) + test_errored = _execute(cmd) errored += test_errored if test_errored: which_errored.setdefault(example_path, []).append( @@ -220,12 +220,12 @@ def run_examples(keep_artifacts=None, conda_exe=None): if creation_errored: which_errored.setdefault(example_path, []).append("Could not create installer!") print() - break + print("-------------------------------") if errored: print('Some examples failed:') - for key, reasons in which_errored.items(): - print(f"+ {key}") + for installer, reasons in which_errored.items(): + print(f"+ {os.path.basename(installer)}") for reason in reasons: print(f"---> {reason}") print('Assets saved in: ', parent_output) From 277d7f77c8a1a04a1402038d367b6d3bdadefa1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Wed, 11 Jan 2023 19:59:31 +0100 Subject: [PATCH 05/25] try naming? --- .github/workflows/main.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 25f8eb6c..c25713b5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,20 +23,22 @@ defaults: shell: bash jobs: package: - runs-on: ${{ matrix.os }} + name: ${{ matrix.os }} Python ${{ matrix.pyver }} ${{ matrix.micromamba || "" }} + runs-on: ${{ matrix.os }}-latest strategy: fail-fast: false matrix: - os: [macos-latest, ubuntu-latest, windows-latest] + os: [macos, ubuntu, windows] pyver: ["3.7", "3.8", "3.9", "3.10"] + micromamba: [false] include: - - os: ubuntu-latest + - os: ubuntu pyver: "3.9" micromamba: true - - os: macos-latest + - os: macos pyver: "3.10" micromamba: true - - os: windows-latest + - os: windows pyver: "3.8" micromamba: true env: From ac690f9d4ca28dd6265a65a4b6b789c5fbe781e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Wed, 11 Jan 2023 20:02:07 +0100 Subject: [PATCH 06/25] ternary --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c25713b5..450572df 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,7 +23,7 @@ defaults: shell: bash jobs: package: - name: ${{ matrix.os }} Python ${{ matrix.pyver }} ${{ matrix.micromamba || "" }} + name: ${{ matrix.os }}, Python ${{ matrix.pyver }}, ${{ matrix.micromamba && "micromamba" || "conda-standalone" }} runs-on: ${{ matrix.os }}-latest strategy: fail-fast: false From 5b8516b6c29cf3d69bea2be5840d23e85ab7c358 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Wed, 11 Jan 2023 20:02:58 +0100 Subject: [PATCH 07/25] single quotes --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 450572df..a2077f2c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,7 +23,7 @@ defaults: shell: bash jobs: package: - name: ${{ matrix.os }}, Python ${{ matrix.pyver }}, ${{ matrix.micromamba && "micromamba" || "conda-standalone" }} + name: ${{ matrix.os }}, Python ${{ matrix.pyver }}, ${{ matrix.micromamba && 'micromamba' || 'conda-standalone' }} runs-on: ${{ matrix.os }}-latest strategy: fail-fast: false From e6d9bf0322d71d0c8dc3a33071d0dcd7ae6e97ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Wed, 11 Jan 2023 20:19:31 +0100 Subject: [PATCH 08/25] remove general entry in matrix? --- .github/workflows/main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a2077f2c..d27152fb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -30,7 +30,6 @@ jobs: matrix: os: [macos, ubuntu, windows] pyver: ["3.7", "3.8", "3.9", "3.10"] - micromamba: [false] include: - os: ubuntu pyver: "3.9" From 3416e7a1be6254a241060dc4bcc23cae53212f24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Wed, 11 Jan 2023 20:39:34 +0100 Subject: [PATCH 09/25] find micromamba on windows too? --- .github/workflows/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d27152fb..2a977c63 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -128,7 +128,8 @@ jobs: source $CONDA/etc/profile.d/conda.sh if [[ "${{ matrix.micromamba }}" != "" ]]; then conda create -yqp ./micromamba -c conda-forge micromamba - echo "CONSTRUCTOR_CONDA_EXE=$(pwd)/micromamba/bin/micromamba" >> $GITHUB_ENV + conda activate ./micromamba + echo "CONSTRUCTOR_CONDA_EXE=$(which micromamba)" >> $GITHUB_ENV else conda activate constructor echo "CONSTRUCTOR_CONDA_EXE=$CONDA_PREFIX/standalone_conda/conda.exe" >> $GITHUB_ENV From 2e38b5202bc3b61e431fc74c6545038f34004970 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Wed, 11 Jan 2023 20:59:13 +0100 Subject: [PATCH 10/25] manual path for windows --- .github/workflows/main.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2a977c63..5860ba44 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -128,8 +128,11 @@ jobs: source $CONDA/etc/profile.d/conda.sh if [[ "${{ matrix.micromamba }}" != "" ]]; then conda create -yqp ./micromamba -c conda-forge micromamba - conda activate ./micromamba - echo "CONSTRUCTOR_CONDA_EXE=$(which micromamba)" >> $GITHUB_ENV + if [[ ${{ matrix.os }} == "windows" ]]; then + echo "CONSTRUCTOR_CONDA_EXE=./micromamba/Library/bin/micromamba.exe" >> $GITHUB_ENV + else + echo "CONSTRUCTOR_CONDA_EXE=./micromamba/bin/micromamba" >> $GITHUB_ENV + fi else conda activate constructor echo "CONSTRUCTOR_CONDA_EXE=$CONDA_PREFIX/standalone_conda/conda.exe" >> $GITHUB_ENV From 3a6ec71d28d4a1950457e7c3621b492525f15b68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Thu, 12 Jan 2023 21:21:20 +0100 Subject: [PATCH 11/25] add debug mode --- scripts/run_examples.py | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/scripts/run_examples.py b/scripts/run_examples.py index cbefd279..035f8c4a 100644 --- a/scripts/run_examples.py +++ b/scripts/run_examples.py @@ -29,10 +29,15 @@ WITH_SPACES = {"extra_files", "noconda", "signing", "scripts"} -def _execute(cmd): +def _execute(cmd, **env_vars): print(' '.join(cmd)) t0 = time.time() - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + if env_vars: + env = os.environ.copy() + env.update(env_vars) + else: + env = None + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=env) try: stdout, stderr = p.communicate(timeout=420) errored = p.returncode != 0 @@ -42,7 +47,7 @@ def _execute(cmd): print('--- TEST TIMEOUT ---') errored = True t1 = time.time() - if errored: + if errored or "CONDA_VERBOSITY" in env_vars: if stdout: print('--- STDOUT ---') print(stdout) @@ -53,7 +58,7 @@ def _execute(cmd): return errored -def run_examples(keep_artifacts=None, conda_exe=None): +def run_examples(keep_artifacts=None, conda_exe=None, debug=False): """Run examples bundled with the repository. Parameters @@ -107,6 +112,8 @@ def run_examples(keep_artifacts=None, conda_exe=None): cmd = COV_CMD + ['constructor', '-v', example_path, '--output-dir', output_dir] if conda_exe: cmd += ['--conda-exe', conda_exe] + if debug: + cmd.append("--debug") creation_errored = _execute(cmd) errored += creation_errored for fpath in os.listdir(output_dir): @@ -142,7 +149,8 @@ def run_examples(keep_artifacts=None, conda_exe=None): # the point is to just have spaces in the installation path -- one would be enough too :) # This is why we have this weird .split() thingy down here: cmd = ['cmd.exe', '/c', 'start', '/wait', fpath, '/S', *f'/D={env_dir}'.split()] - test_errored = _execute(cmd) + env = {"CONDA_VERBOSITY": "3"} if debug else {} + test_errored = _execute(cmd, **env) # Windows EXEs never throw a non-0 exit code, so we need to check the logs, # which are only written if a special NSIS build is used win_error_lines = [] @@ -216,6 +224,9 @@ def run_examples(keep_artifacts=None, conda_exe=None): which_errored.setdefault(example_path, []).append("Could not find uninstaller!") if keep_artifacts: + dest = os.path.join(keep_artifacts, os.path.basename(fpath)) + if os.path.isfile(dest): + os.unlink(dest) shutil.move(fpath, keep_artifacts) if creation_errored: which_errored.setdefault(example_path, []).append("Could not create installer!") @@ -228,7 +239,7 @@ def run_examples(keep_artifacts=None, conda_exe=None): print(f"+ {os.path.basename(installer)}") for reason in reasons: print(f"---> {reason}") - print('Assets saved in: ', parent_output) + print('Assets saved in:', keep_artifacts or parent_output) else: print('All examples ran successfully!') shutil.rmtree(parent_output) @@ -239,6 +250,7 @@ def cli(): p = argparse.ArgumentParser() p.add_argument("--keep-artifacts") p.add_argument("--conda-exe") + p.add_argument("--debug", action="store_true", default=False) return p.parse_args() @@ -246,5 +258,9 @@ def cli(): args = cli() if args.conda_exe: assert os.path.isfile(args.conda_exe) - n_errors = run_examples(keep_artifacts=args.keep_artifacts, conda_exe=args.conda_exe) + n_errors = run_examples( + keep_artifacts=args.keep_artifacts, + conda_exe=args.conda_exe, + debug=args.debug + ) sys.exit(n_errors) From 9a4f949b39a503db2243bfafd793192e764e7327 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Thu, 12 Jan 2023 21:39:58 +0100 Subject: [PATCH 12/25] fix transmutation in conda --- constructor/preconda.py | 3 ++- constructor/utils.py | 20 +++++++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/constructor/preconda.py b/constructor/preconda.py index bc4ecda4..3a3306c4 100644 --- a/constructor/preconda.py +++ b/constructor/preconda.py @@ -19,7 +19,7 @@ from .conda_interface import (CONDA_INTERFACE_VERSION, Dist, MatchSpec, default_prefix, PrefixData, write_repodata, get_repodata, all_channel_urls) from .conda_interface import distro as conda_distro -from .utils import get_final_channels +from .utils import get_final_channels, ensure_transmuted_ext try: import json @@ -126,6 +126,7 @@ def write_files(info, dst_dir): with open(join(dst_dir, 'urls'), 'w') as fo: for url, md5 in all_final_urls_md5s: + url = ensure_transmuted_ext(info, url) fo.write('%s#%s\n' % (url, md5)) with open(join(dst_dir, 'urls.txt'), 'w') as fo: diff --git a/constructor/utils.py b/constructor/utils.py index 14790cad..5c65840c 100644 --- a/constructor/utils.py +++ b/constructor/utils.py @@ -8,7 +8,7 @@ import sys import hashlib import math -from os.path import normpath, islink, isfile, isdir +from os.path import normpath, islink, isfile, isdir, basename from os import sep, unlink from shutil import rmtree @@ -123,6 +123,24 @@ def add_condarc(info): yield 'EOF' +def ensure_transmuted_ext(info, url): + """ + If transmuting, micromamba won't find the dist in the preconda tarball + unless it has the (correct and transmuted) extension. Otherwise, the command + `micromamba constructor --extract-tarballs` fails. + Unfortunately this means the `urls` file might end up containing + fake URLs, since those .conda archives might not really exist online, + and they were only created locally. + """ + if ( + info.get("transmute_file_type") == ".conda" + and "micromamba" in basename(info.get("_conda_exe", "")) + ): + if url.lower().endswith(".tar.bz2"): + url = url[:-8] + ".conda" + return url + + def get_final_url(info, url): mapping = info.get('channels_remap', []) for entry in mapping: From 4774bc9005dee918835be54596d221d114ec5e5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Thu, 12 Jan 2023 21:43:23 +0100 Subject: [PATCH 13/25] pre-commit --- constructor/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/constructor/utils.py b/constructor/utils.py index 357f7e9e..4d48c3c9 100644 --- a/constructor/utils.py +++ b/constructor/utils.py @@ -133,9 +133,9 @@ def ensure_transmuted_ext(info, url): and they were only created locally. """ if ( - info.get("transmute_file_type") == ".conda" + info.get("transmute_file_type") == ".conda" and "micromamba" in basename(info.get("_conda_exe", "")) - ): + ): if url.lower().endswith(".tar.bz2"): url = url[:-8] + ".conda" return url From 618d37e000d0a6456205147806a044b20a0fcdef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Thu, 12 Jan 2023 22:13:31 +0100 Subject: [PATCH 14/25] add tests for miniforge --- examples/miniforge/construct.yaml | 5 +++++ examples/miniforge/test_install.bat | 7 +++++++ examples/miniforge/test_install.sh | 9 +++++++++ 3 files changed, 21 insertions(+) create mode 100644 examples/miniforge/test_install.bat create mode 100644 examples/miniforge/test_install.sh diff --git a/examples/miniforge/construct.yaml b/examples/miniforge/construct.yaml index 7c1a8cf5..b9354081 100644 --- a/examples/miniforge/construct.yaml +++ b/examples/miniforge/construct.yaml @@ -14,3 +14,8 @@ specs: - conda 4.10.1 - pip - miniforge_console_shortcut 1.* # [win] + +# Added for extra testing +installer_type: all +post_install: test_install.sh # [unix] +post_install: test_install.bat # [win] diff --git a/examples/miniforge/test_install.bat b/examples/miniforge/test_install.bat new file mode 100644 index 00000000..fb3dcb49 --- /dev/null +++ b/examples/miniforge/test_install.bat @@ -0,0 +1,7 @@ +@ECHO ON +call "%PREFIX%\Scripts\activate.bat +conda install -yq jq || exit 1 +conda config --show-sources || exit 1 +conda config --json --show | jq -r '.channels[0]' > temp.txt +set /p OUTPUT= Date: Thu, 12 Jan 2023 22:14:13 +0100 Subject: [PATCH 15/25] use double quotes just in case --- examples/miniforge/test_install.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/miniforge/test_install.bat b/examples/miniforge/test_install.bat index fb3dcb49..dff126d6 100644 --- a/examples/miniforge/test_install.bat +++ b/examples/miniforge/test_install.bat @@ -2,6 +2,6 @@ call "%PREFIX%\Scripts\activate.bat conda install -yq jq || exit 1 conda config --show-sources || exit 1 -conda config --json --show | jq -r '.channels[0]' > temp.txt +conda config --json --show | jq -r ".channels[0]" > temp.txt set /p OUTPUT= Date: Thu, 12 Jan 2023 23:15:14 +0100 Subject: [PATCH 16/25] allow unbound vars --- examples/miniforge/test_install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/miniforge/test_install.sh b/examples/miniforge/test_install.sh index 9be27c58..a2283f0e 100644 --- a/examples/miniforge/test_install.sh +++ b/examples/miniforge/test_install.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -euxo pipefail +set -exo pipefail source "$PREFIX/etc/profile.d/conda.sh" conda activate "$PREFIX" From b996a31438c9c06919d85df6323e2cb3df26c71b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Thu, 12 Jan 2023 23:22:52 +0100 Subject: [PATCH 17/25] shellcheck --- examples/miniforge/test_install.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/miniforge/test_install.sh b/examples/miniforge/test_install.sh index a2283f0e..93214c79 100644 --- a/examples/miniforge/test_install.sh +++ b/examples/miniforge/test_install.sh @@ -2,6 +2,7 @@ set -exo pipefail +# shellcheck disable=SC1091 source "$PREFIX/etc/profile.d/conda.sh" conda activate "$PREFIX" conda install -yq jq From b628a5c8bf80b73d762cbf1fcbe8d9f54b3f3d13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Sun, 15 Jan 2023 20:37:10 +0100 Subject: [PATCH 18/25] document micromamba & menu_packages compatibility --- CONSTRUCT.md | 8 ++++++-- constructor/construct.py | 10 +++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/CONSTRUCT.md b/CONSTRUCT.md index f23e7cd2..a87f712f 100644 --- a/CONSTRUCT.md +++ b/CONSTRUCT.md @@ -100,11 +100,15 @@ is contained as a result of resolving the specs for `python 2.7`. _required:_ no
_type:_ list
-A list of packages with menu items to be instsalled. The packages must have -necessary metadata in "Menu/.json"). Menu items are currently +A list of packages with menu items to be installed. The packages must have +necessary metadata in `Menu/.json`). Menu items are currently only supported on Windows. By default, all menu items will be installed; supplying this list allows a subset to be selected instead. +Note: `micromamba` does not support shortcut creation. Installers +that use `menu_packages` will fail the menu item creation steps +if you use `--conda-exe=`. + ## `ignore_duplicate_files` _required:_ no
diff --git a/constructor/construct.py b/constructor/construct.py index 3514f0b4..e7af4026 100644 --- a/constructor/construct.py +++ b/constructor/construct.py @@ -79,11 +79,15 @@ is contained as a result of resolving the specs for `python 2.7`. '''), - ('menu_packages', False, list, ''' -A list of packages with menu items to be instsalled. The packages must have -necessary metadata in "Menu/.json"). Menu items are currently + ('menu_packages', False, list, ''' +A list of packages with menu items to be installed. The packages must have +necessary metadata in `Menu/.json`). Menu items are currently only supported on Windows. By default, all menu items will be installed; supplying this list allows a subset to be selected instead. + +Note: `micromamba` does not support shortcut creation. Installers +that use `menu_packages` will fail the menu item creation steps +if you use `--conda-exe=`. '''), ('ignore_duplicate_files', False, bool, ''' From ab328e60493fdbd3e2f9ac385275d8161b94884b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Sun, 15 Jan 2023 20:38:46 +0100 Subject: [PATCH 19/25] trim spaces --- constructor/construct.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/constructor/construct.py b/constructor/construct.py index e7af4026..c829d202 100644 --- a/constructor/construct.py +++ b/constructor/construct.py @@ -86,7 +86,7 @@ supplying this list allows a subset to be selected instead. Note: `micromamba` does not support shortcut creation. Installers -that use `menu_packages` will fail the menu item creation steps +that use `menu_packages` will fail the menu item creation steps if you use `--conda-exe=`. '''), From a851783315aaf47dd4c3a74c7364e13770e01407 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Sun, 15 Jan 2023 20:50:24 +0100 Subject: [PATCH 20/25] sync docs --- CONSTRUCT.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONSTRUCT.md b/CONSTRUCT.md index a87f712f..04bc05c5 100644 --- a/CONSTRUCT.md +++ b/CONSTRUCT.md @@ -106,7 +106,7 @@ only supported on Windows. By default, all menu items will be installed; supplying this list allows a subset to be selected instead. Note: `micromamba` does not support shortcut creation. Installers -that use `menu_packages` will fail the menu item creation steps +that use `menu_packages` will fail the menu item creation steps if you use `--conda-exe=`. ## `ignore_duplicate_files` From 3f593df9db518c3181994ebe93b73e32caf4f8a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Sun, 15 Jan 2023 21:09:30 +0100 Subject: [PATCH 21/25] Skip miniforge + micromamba + windows --- examples/miniforge/construct.yaml | 2 ++ scripts/run_examples.py | 15 +++++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/examples/miniforge/construct.yaml b/examples/miniforge/construct.yaml index b9354081..71a24eb3 100644 --- a/examples/miniforge/construct.yaml +++ b/examples/miniforge/construct.yaml @@ -19,3 +19,5 @@ specs: installer_type: all post_install: test_install.sh # [unix] post_install: test_install.bat # [win] +menu_packages: # [win] + - miniforge_console_shortcut # [win] diff --git a/scripts/run_examples.py b/scripts/run_examples.py index d40f27bc..4041184b 100644 --- a/scripts/run_examples.py +++ b/scripts/run_examples.py @@ -95,7 +95,8 @@ def run_examples(keep_artifacts=None, conda_exe=None, debug=False): if os.path.exists(os.path.join(fpath, 'construct.yaml')): example_paths.append(fpath) - # NSIS won't error out when running scripts unless we set this custom environment variable + # NSIS won't error out when running scripts unless + # we set this custom environment variable os.environ["NSIS_SCRIPTS_RAISE_ERRORS"] = "1" parent_output = tempfile.mkdtemp() @@ -106,7 +107,17 @@ def run_examples(keep_artifacts=None, conda_exe=None, debug=False): test_with_spaces = example_name in WITH_SPACES print(example_name) print('-' * len(example_name)) - + if ( + sys.platform.startswith("win") + and conda_exe + and "micromamba" in os.path.basename(conda_exe).lower() + and "menu_packages" in (Path(example_path) / "construct.yaml").read_text() + ): + print( + f"Skipping {example_name}... 'menu_shortcuts' " + "not supported with micromamba." + ) + continue output_dir = tempfile.mkdtemp(prefix=f"{example_name}-", dir=parent_output) # resolve path to avoid some issues with TEMPDIR on Windows output_dir = str(Path(output_dir).resolve()) From 36a6cf9ea5f659b732910683e6534408a33264d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Sun, 15 Jan 2023 21:10:12 +0100 Subject: [PATCH 22/25] trim --- scripts/run_examples.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/run_examples.py b/scripts/run_examples.py index 4041184b..c40d09df 100644 --- a/scripts/run_examples.py +++ b/scripts/run_examples.py @@ -95,7 +95,7 @@ def run_examples(keep_artifacts=None, conda_exe=None, debug=False): if os.path.exists(os.path.join(fpath, 'construct.yaml')): example_paths.append(fpath) - # NSIS won't error out when running scripts unless + # NSIS won't error out when running scripts unless # we set this custom environment variable os.environ["NSIS_SCRIPTS_RAISE_ERRORS"] = "1" @@ -108,8 +108,8 @@ def run_examples(keep_artifacts=None, conda_exe=None, debug=False): print(example_name) print('-' * len(example_name)) if ( - sys.platform.startswith("win") - and conda_exe + sys.platform.startswith("win") + and conda_exe and "micromamba" in os.path.basename(conda_exe).lower() and "menu_packages" in (Path(example_path) / "construct.yaml").read_text() ): From f00f084de7ded0130df71dc737c6950aba2f723d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Mon, 16 Jan 2023 10:04:32 +0100 Subject: [PATCH 23/25] simplify skip logic --- examples/miniforge/construct.yaml | 2 -- scripts/run_examples.py | 3 +-- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/examples/miniforge/construct.yaml b/examples/miniforge/construct.yaml index 71a24eb3..b9354081 100644 --- a/examples/miniforge/construct.yaml +++ b/examples/miniforge/construct.yaml @@ -19,5 +19,3 @@ specs: installer_type: all post_install: test_install.sh # [unix] post_install: test_install.bat # [win] -menu_packages: # [win] - - miniforge_console_shortcut # [win] diff --git a/scripts/run_examples.py b/scripts/run_examples.py index c40d09df..7a00cc42 100644 --- a/scripts/run_examples.py +++ b/scripts/run_examples.py @@ -111,10 +111,9 @@ def run_examples(keep_artifacts=None, conda_exe=None, debug=False): sys.platform.startswith("win") and conda_exe and "micromamba" in os.path.basename(conda_exe).lower() - and "menu_packages" in (Path(example_path) / "construct.yaml").read_text() ): print( - f"Skipping {example_name}... 'menu_shortcuts' " + f"! Skipping {example_name}... Shortcut creation on Windows is " "not supported with micromamba." ) continue From 8f5ea3c61d2a6105111faf21d64da6458f2fb2e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Mon, 16 Jan 2023 10:39:20 +0100 Subject: [PATCH 24/25] error out if micromamba used on windows, remove ci there --- .github/workflows/main.yml | 7 ++++--- CONSTRUCT.md | 4 ---- constructor/construct.py | 4 ---- constructor/main.py | 5 ++++- 4 files changed, 8 insertions(+), 12 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 950ae46a..8ca24603 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -37,9 +37,10 @@ jobs: - os: macos pyver: "3.10" micromamba: true - - os: windows - pyver: "3.8" - micromamba: true + # Re-enable once micromamba supports menu creation + # - os: windows + # pyver: "3.8" + # micromamba: true env: PYTHONUNBUFFERED: True steps: diff --git a/CONSTRUCT.md b/CONSTRUCT.md index 04bc05c5..fb09991a 100644 --- a/CONSTRUCT.md +++ b/CONSTRUCT.md @@ -105,10 +105,6 @@ necessary metadata in `Menu/.json`). Menu items are currently only supported on Windows. By default, all menu items will be installed; supplying this list allows a subset to be selected instead. -Note: `micromamba` does not support shortcut creation. Installers -that use `menu_packages` will fail the menu item creation steps -if you use `--conda-exe=`. - ## `ignore_duplicate_files` _required:_ no
diff --git a/constructor/construct.py b/constructor/construct.py index c829d202..4e2502b8 100644 --- a/constructor/construct.py +++ b/constructor/construct.py @@ -84,10 +84,6 @@ necessary metadata in `Menu/.json`). Menu items are currently only supported on Windows. By default, all menu items will be installed; supplying this list allows a subset to be selected instead. - -Note: `micromamba` does not support shortcut creation. Installers -that use `menu_packages` will fail the menu item creation steps -if you use `--conda-exe=`. '''), ('ignore_duplicate_files', False, bool, ''' diff --git a/constructor/main.py b/constructor/main.py index 44e680da..3eff871b 100644 --- a/constructor/main.py +++ b/constructor/main.py @@ -86,6 +86,9 @@ def main_build(dir_path, output_dir='.', platform=cc_platform, if platform != cc_platform and 'pkg' in itypes and not cc_platform.startswith('osx-'): sys.exit("Error: cannot construct a macOS 'pkg' installer on '%s'" % cc_platform) + if osname == "win" and "micromamba" in os.path.basename(info['_conda_exe']): + # TODO: Remove when shortcut creation is implemented on micromamba + sys.exit("Error: micromamba is not supported on Windows installers.") if verbose: print('conda packages download: %s' % info['_download_dir']) @@ -293,7 +296,7 @@ def main(): version='%(prog)s {version}'.format(version=__version__)) p.add_argument('--conda-exe', - help="path to conda executable", + help="path to conda executable (conda-standalone, micromamba)", action="store", metavar="CONDA_EXE") From 6c535e1a340912af362d093c5bce8aa1a4985efc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Rodr=C3=ADguez-Guerra?= Date: Mon, 16 Jan 2023 10:58:32 +0100 Subject: [PATCH 25/25] add news --- news/605-micromamba-tests | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 news/605-micromamba-tests diff --git a/news/605-micromamba-tests b/news/605-micromamba-tests new file mode 100644 index 00000000..811cb377 --- /dev/null +++ b/news/605-micromamba-tests @@ -0,0 +1,20 @@ +### Enhancements + +* + +### Bug fixes + +* Add tests for `--conda-exe=` and fix found issues on Linux and macOS. + Not supported on Windows yet. (#503, #605) + +### Deprecations + +* + +### Docs + +* + +### Other + +*