diff --git a/devenv.lock b/devenv.lock index bcc0c5416..5f7d4f8f5 100644 --- a/devenv.lock +++ b/devenv.lock @@ -107,11 +107,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1689261696, - "narHash": "sha256-LzfUtFs9MQRvIoQ3MfgSuipBVMXslMPH/vZ+nM40LkA=", + "lastModified": 1690952720, + "narHash": "sha256-fPsiQHARfhVxXpWgcuSKvkYwSco8K13K7XevBpdIfPg=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "df1eee2aa65052a18121ed4971081576b25d6b5c", + "rev": "96112a3ed5d12bb1758b42c63b924c004b6c0bc9", "type": "github" }, "original": { @@ -164,11 +164,11 @@ "nixpkgs-stable": "nixpkgs-stable" }, "locked": { - "lastModified": 1688596063, - "narHash": "sha256-9t7RxBiKWHygsqXtiNATTJt4lim/oSYZV3RG8OjDDng=", + "lastModified": 1690743255, + "narHash": "sha256-dsJzQsyJGWCym1+LMyj2rbYmvjYmzeOrk7ypPrSFOPo=", "owner": "cachix", "repo": "pre-commit-hooks.nix", - "rev": "c8d18ba345730019c3faf412c96a045ade171895", + "rev": "fcbf4705d98398d084e6cb1c826a0b90a91d22d7", "type": "github" }, "original": { diff --git a/devenv.nix b/devenv.nix index ca33fc4bd..ed9453112 100644 --- a/devenv.nix +++ b/devenv.nix @@ -7,6 +7,7 @@ pkgs.cairo pkgs.xorg.libxcb pkgs.yaml2json + pkgs.tesh ]; languages.nix.enable = true; @@ -42,6 +43,7 @@ tmp="$(mktemp -d)" devenv init "$tmp" pushd "$tmp" + devenv version devenv ci popd rm -rf "$tmp" @@ -124,5 +126,6 @@ MD034 = false; }; - tests = config.lib.mkTests [ "devenv" ] ./examples; + tests = config.lib.mkTests ./examples + // config.lib.mkTests ./tests; } diff --git a/devenv.yaml b/devenv.yaml index d759fba2c..9cbe87bf5 100644 --- a/devenv.yaml +++ b/devenv.yaml @@ -5,11 +5,11 @@ inputs: url: github:domenkozar/nix/relaxed-flakes inputs: nixpkgs: - follows: "nixpkgs" + follows: nixpkgs devenv: url: path:./src/modules pre-commit-hooks: url: github:cachix/pre-commit-hooks.nix inputs: nixpkgs: - follows: "nixpkgs" \ No newline at end of file + follows: nixpkgs diff --git a/docs/binary-caching.md b/docs/binary-caching.md new file mode 100644 index 000000000..2189e01db --- /dev/null +++ b/docs/binary-caching.md @@ -0,0 +1,34 @@ +Typically [packages](./packages.md) come prebuilt with binaries provided by [the official binary cache](https://cache.nixos.org). + +If you're modifying a package or using a package that's not built upstream, +Nix will build it from source instead of downloading a binary. + +To prevent packages from being built more than once, there's seamless integration with +binary caches using [Cachix](https://cachix.org). + +## Setup + +If you'd like to push binaries to your own cache, you'll need [to create one](https://app.cachix.org/cache). + +After that you'll need to set `cachix authtoken XXX` with either [a personal auth token](https://app.cachix.org/personal-auth-tokens) or a cache token (that you can create in cache settings). + +## devenv.nix + +To specify `pre-commit-hooks` as a cache to pull from and `mycache` to pull from and push to: + +```nix title="devenv.nix" +{ + cachix.pull = [ "pre-commit-hooks" ]; + cachix.push = "mycache"; +} +``` + +# Pushing only in specific cases + +You'll likely not want every user to push to the cache. + +It's usually convenient to push [explicitly](./files-and-variables/#devenvlocalnix), for example as part of CI run: + +```shell-session +$ echo '{ cachix.push = "mycache"; }' > devenv.local.nix +``` \ No newline at end of file diff --git a/docs/community/contributing.md b/docs/community/contributing.md index 971fc75eb..ffe18bf4a 100644 --- a/docs/community/contributing.md +++ b/docs/community/contributing.md @@ -6,28 +6,23 @@ We have a rule that new features need to come with documentation and tests (`dev ## Preparing the `devenv` development environment -1. Follow the [installation instructions for Nix and Cachix](../../getting-started/#installation). +1. Follow the [installation instructions for Nix and Cachix](../../getting-started/#installation) and [install direnv](../../automatic-shell-activation/). 2. `git clone https://github.com/cachix/devenv.git` 3. `cd devenv` -4. To build the project, run `nix-build`. - -5. `./result/bin/devenv shell` - -6. Once you have made changes, run `./result/bin/devenv shell` again. - -To automate this workflow, [install and use direnv](../../automatic-shell-activation/). +4. To build the project, run `direnv allow .`. ## Repository structure -- The `devenv` CLI is in `src/devenv.nix`. -- The `flake.nix` auto-generation logic lies in `src/flake.nix`. +- The `devenv` CLI is in `src/devenv/cli.py`. +- The `flake.nix` auto-generation logic lies in `src/modules/flake.tmpl.nix`. - All modules related to `devenv.nix` are in `src/modules/`. -- Examples are automatically tested on CI and are the best way to work on developing new modules, see `examples/`. +- Examples are automatically tested on CI and are the best way to work on developing new modules, see `examples/` and `tests/` - Documentation is in `docs/`. - To run a development server, run `devenv up`. +- To run a test, run `devnenv test `. ## Contributing language improvements diff --git a/docs/tests.md b/docs/tests.md new file mode 100644 index 000000000..7fbec6286 --- /dev/null +++ b/docs/tests.md @@ -0,0 +1,55 @@ +To ease testing of your environments, +we provide a way to define the tests and to run them. + +## Writing devenv tests + +A simple test would look like: + +```nix title="devenv.nix" +{ pkgs, ... }: { + tests.basic = { + nix = '' + { pkgs, ... }: { + packages = [ pkgs.ncdu ]; + } + ''; + test = '' + ncdu --version | grep "ncdu 2.2" + ''; + }; +} +``` + +```shell-session +$ devenv test +✔ Gathering tests in 0.3s. +• Found 1 test(s), running 1: +• Testing basic ... +• Running $ devenv ci +• Running .test.sh. +✔ Running basic in 16.7s. +``` + +## Defining tests in a folder + +A simple test with a tesh script: + +```shell-session +$ ls tests/mytest/ +.test.sh devenv.nix devenv.yaml +``` + +And define it: + +```nix title="devenv.nix" +{ config, ... }: { + tests = config.lib.mkTests ./tests; +} +``` + +Run tests: + +```shell-session +$ devenv test +... +``` \ No newline at end of file diff --git a/examples/overlays/devenv.nix b/examples/overlays/devenv.nix index c27092e92..d7eae18cc 100644 --- a/examples/overlays/devenv.nix +++ b/examples/overlays/devenv.nix @@ -1,7 +1,11 @@ { pkgs, ... }: { - packages = [ pkgs.rust-bin.stable.latest.default ]; + packages = [ + # from the rust-overlay + pkgs.rust-bin.stable.latest.default - services.blackfire.enable = true; + # from subflake + pkgs.hello2 + ]; } diff --git a/examples/overlays/devenv.yaml b/examples/overlays/devenv.yaml index 205510e49..044462b43 100644 --- a/examples/overlays/devenv.yaml +++ b/examples/overlays/devenv.yaml @@ -2,6 +2,10 @@ allowUnfree: true inputs: nixpkgs: url: github:NixOS/nixpkgs/nixpkgs-unstable + subflake: + url: path:./subflake + overlays: + - default rust-overlay: url: github:oxalica/rust-overlay overlays: diff --git a/examples/overlays/subflake/flake.nix b/examples/overlays/subflake/flake.nix new file mode 100644 index 000000000..d3ebe7a83 --- /dev/null +++ b/examples/overlays/subflake/flake.nix @@ -0,0 +1,7 @@ +{ + outputs = { ... }: { + overlays.default = self: super: { + hello2 = self.hello; + }; + }; +} diff --git a/mkdocs.yml b/mkdocs.yml index 3f62f0f09..17aa93111 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -38,7 +38,9 @@ nav: - Processes: processes.md - Services: services.md - Containers: containers.md + - Binary Caching: binary-caching.md - Pre-Commit Hooks: pre-commit-hooks.md + - Tests: tests.md - Common Patterns: common-patterns.md - Writing devenv.yaml: - Inputs: inputs.md diff --git a/poetry.lock b/poetry.lock index a9cdf341d..bee670838 100644 --- a/poetry.lock +++ b/poetry.lock @@ -464,13 +464,13 @@ files = [ [[package]] name = "mkdocs" -version = "1.5.0" +version = "1.5.1" description = "Project documentation with Markdown." optional = false python-versions = ">=3.7" files = [ - {file = "mkdocs-1.5.0-py3-none-any.whl", hash = "sha256:91a75e3a5a75e006b2149814d5c56af170039ceda0732f51e7af1a463599c00d"}, - {file = "mkdocs-1.5.0.tar.gz", hash = "sha256:ff54eac0b74bf39a2e91f179e2ac16ef36f0294b9ab161c22f564382b30a31ae"}, + {file = "mkdocs-1.5.1-py3-none-any.whl", hash = "sha256:67e889f8d8ba1fe5decdfc59f5f8f21d6a8925a129339e93dede303bdea03a98"}, + {file = "mkdocs-1.5.1.tar.gz", hash = "sha256:f2f323c62fffdf1b71b84849e39aef56d6852b3f0a5571552bca32cefc650209"}, ] [package.dependencies] @@ -520,20 +520,20 @@ pyyaml = "*" [[package]] name = "mkdocs-material" -version = "9.1.20" +version = "9.1.21" description = "Documentation that simply works" optional = false python-versions = ">=3.7" files = [ - {file = "mkdocs_material-9.1.20-py3-none-any.whl", hash = "sha256:152db66f667825d5aa3398386fe4d227640ec393c31e7cf109b114a569fc40fc"}, - {file = "mkdocs_material-9.1.20.tar.gz", hash = "sha256:91621b6a6002138c72d50a0beef20ed12cf367d2af27d1f53382562b3a9625c7"}, + {file = "mkdocs_material-9.1.21-py3-none-any.whl", hash = "sha256:58bb2f11ef240632e176d6f0f7d1cff06be1d11c696a5a1b553b808b4280ed47"}, + {file = "mkdocs_material-9.1.21.tar.gz", hash = "sha256:71940cdfca84ab296b6362889c25395b1621273fb16c93deda257adb7ff44ec8"}, ] [package.dependencies] colorama = ">=0.4" jinja2 = ">=3.0" markdown = ">=3.2" -mkdocs = ">=1.4.2" +mkdocs = ">=1.5.0" mkdocs-material-extensions = ">=1.1" pygments = ">=2.14" pymdown-extensions = ">=9.9.1" @@ -584,13 +584,13 @@ files = [ [[package]] name = "pathspec" -version = "0.11.1" +version = "0.11.2" description = "Utility library for gitignore style pattern matching of file paths." optional = false python-versions = ">=3.7" files = [ - {file = "pathspec-0.11.1-py3-none-any.whl", hash = "sha256:d8af70af76652554bd134c22b3e8a1cc46ed7d91edcdd721ef1a0c51a84a5293"}, - {file = "pathspec-0.11.1.tar.gz", hash = "sha256:2798de800fa92780e33acca925945e9a19a133b715067cf165b8866c15a31687"}, + {file = "pathspec-0.11.2-py3-none-any.whl", hash = "sha256:1d6ed233af05e679efb96b1851550ea95bbb64b7c490b0f5aa52996c11e92a20"}, + {file = "pathspec-0.11.2.tar.gz", hash = "sha256:e0d8d0ac2f12da61956eb2306b69f9469b42f4deb0f3cb6ed47b9cce9996ced3"}, ] [[package]] @@ -674,18 +674,18 @@ tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "pa [[package]] name = "platformdirs" -version = "3.9.1" +version = "3.10.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." optional = false python-versions = ">=3.7" files = [ - {file = "platformdirs-3.9.1-py3-none-any.whl", hash = "sha256:ad8291ae0ae5072f66c16945166cb11c63394c7a3ad1b1bc9828ca3162da8c2f"}, - {file = "platformdirs-3.9.1.tar.gz", hash = "sha256:1b42b450ad933e981d56e59f1b97495428c9bd60698baab9f3eb3d00d5822421"}, + {file = "platformdirs-3.10.0-py3-none-any.whl", hash = "sha256:d7c24979f292f916dc9cbf8648319032f551ea8c49a4c9bf2fb556a02070ec1d"}, + {file = "platformdirs-3.10.0.tar.gz", hash = "sha256:b45696dab2d7cc691a3226759c0d3b00c47c8b6e293d96f6436f733303f77f6d"}, ] [package.extras] -docs = ["furo (>=2023.5.20)", "proselint (>=0.13)", "sphinx (>=7.0.1)", "sphinx-autodoc-typehints (>=1.23,!=1.23.4)"] -test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.3.1)", "pytest-cov (>=4.1)", "pytest-mock (>=3.10)"] +docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.1)", "sphinx-autodoc-typehints (>=1.24)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)"] [[package]] name = "pycparser" @@ -1083,4 +1083,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.12" -content-hash = "0f22aaee71aeb64eeb85f1d625333b93ec21eeab62918b70c8f5777370a64841" +content-hash = "ce967b04c0d8c2818599d333bdcf495f9932b7771fcd151f41c6579c22c1e812" diff --git a/pyproject.toml b/pyproject.toml index 00af8f891..2b07d122d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ click = "^8.1.4" strictyaml = "^1.7.3" terminaltables = "^3.1.10" filelock = "^3.12.0" +requests = "^2.31.0" [tool.poetry.group.docs] optional = true diff --git a/src/devenv/cli.py b/src/devenv/cli.py index 1a8e4ad2b..af3f4b40c 100644 --- a/src/devenv/cli.py +++ b/src/devenv/cli.py @@ -1,3 +1,4 @@ +import functools import os import shlex import shutil @@ -7,20 +8,19 @@ import time import re import sys +import pkgutil +import json from filelock import FileLock from contextlib import suppress from pathlib import Path -import pkgutil -import json - import click import terminaltables import strictyaml - +import requests from .yaml import validate_and_parse_yaml, read_yaml, write_yaml, schema -from .log import log, log_task +from .log import log, log_task, log_error, log_warning, log_info, log_debug NIX_FLAGS = [ @@ -43,19 +43,29 @@ FLAKE_FILE = Path(".devenv.flake.nix") FLAKE_LOCK = "devenv.lock" +# home vars +if 'XDG_DATA_HOME' not in os.environ: + DEVENV_HOME = Path(os.environ['HOME']) / '.devenv' +else: + DEVENV_HOME = Path(os.environ['XDG_DATA_HOME']) / '.devenv' +DEVENV_HOME_GC = DEVENV_HOME / 'gc' +DEVENV_HOME_GC.mkdir(parents=True, exist_ok=True) +CACHIX_KNOWN_PUBKEYS = DEVENV_HOME / "cachix_pubkeys.json" + # define system like x86_64-linux SYSTEM = os.uname().machine.lower().replace("arm", "aarch") + "-" + os.uname().sysname.lower() -def run_nix(command: str, skip_exc_wrapping=False, replace_shell=False) -> str: +def run_nix(command: str, replace_shell=False, use_cachix=False) -> str: ctx = click.get_current_context() nix_flags = ctx.obj['nix_flags'] flags = " ".join(NIX_FLAGS) + " " + " ".join(nix_flags) command_flags = " ".join(ctx.obj['command_flags']) return run_command(f"nix {flags} {command} {command_flags}", - skip_exc_wrapping=skip_exc_wrapping, - replace_shell=replace_shell) + replace_shell=replace_shell, + use_cachix=use_cachix) -def run_command(command: str, disable_stderr=False, skip_exc_wrapping=False, replace_shell=False) -> str: +def run_command(command: str, disable_stderr=False, replace_shell=False, use_cachix=False) -> str: + nix = "" if command.startswith("nix"): if os.environ.get("DEVENV_NIX"): nix = os.path.join(os.environ["DEVENV_NIX"], "bin") @@ -64,6 +74,18 @@ def run_command(command: str, disable_stderr=False, skip_exc_wrapping=False, rep log("$DEVENV_NIX is not set, but required as devenv doesn't work without a few Nix patches.", level="error") log("Please follow https://devenv.sh/getting-started/ to install devenv.", level="error") exit(1) + if use_cachix: + caches, known_keys = get_cachix_caches() + pull_caches = ' '.join(map(lambda cache: f'https://{cache}.cachix.org', caches.get('pull'))) + command = f"{command} --option extra-trusted-public-keys '{' '.join(known_keys.values())}'" + command = f"{command} --option extra-substituters '{pull_caches}'" + push_cache = caches.get("push") + if push_cache: + if shutil.which("cachix") is None: + log_warning("cachix is not installed, not pushing. Please follow https://devenv.sh/getting-started/#2-install-cachix to install cachix.", level="error") + else: + command = f"cachix watch-exec {push_cache} {command}" + try: if click.get_current_context().obj['verbose']: log(f"Running command: {command}", level="debug") @@ -81,12 +103,9 @@ def run_command(command: str, disable_stderr=False, skip_exc_wrapping=False, rep stderr=None if not disable_stderr else subprocess.DEVNULL, universal_newlines=True).stdout.strip() except subprocess.CalledProcessError as e: - if skip_exc_wrapping: - exit(e.returncode) - else: - click.echo("\n", err=True) - log(f"Following command exited with code {e.returncode}:\n\n {e.cmd}", level="error") - exit(e.returncode) + click.echo("\n", err=True) + log(f"Following command exited with code {e.returncode}:\n\n {e.cmd}", level="error") + exit(e.returncode) CONTEXT_SETTINGS = dict(max_content_width=120) @@ -135,13 +154,8 @@ def cli(ctx, disable_eval_cache, offline, system, debugger, nix_flags, verbose): if disable_eval_cache: ctx.obj['nix_flags'] += ['--option', 'eval-cache', 'false'] - if 'XDG_DATA_HOME' not in os.environ: - ctx.obj['gc_root'] = os.path.join(os.environ['HOME'], '.devenv', 'gc') - else: - ctx.obj['gc_root'] = os.path.join(os.environ['XDG_DATA_HOME'], 'devenv', 'gc') - - Path(ctx.obj['gc_root']).mkdir(parents=True, exist_ok=True) - ctx.obj['gc_project'] = os.path.join(ctx.obj['gc_root'], str(int(time.time() * 1000))) + ctx.obj['gc_root'] = DEVENV_HOME_GC + ctx.obj['gc_project'] = DEVENV_HOME_GC / str(int(time.time() * 1000)) @cli.group() def processes(): @@ -161,7 +175,7 @@ def add_gc(name, store_path): """Register a GC root""" ctx = click.get_current_context() run_command(f'nix-store --add-root "{os.environ["DEVENV_GC"]}/{name}" -r {store_path} >/dev/null') - os.symlink(store_path, f'{ctx.obj["gc_project"]}-{name}', True) + symlink_force(store_path, f'{ctx.obj["gc_project"]}-{name}') @cli.command(hidden=True) @@ -213,7 +227,7 @@ def gc(ctx): click.echo(f' Deleted {len(removed_symlinks)} dangling symlinks.') click.echo() - log(f'Running garbage collection (this process may take some time) ...', level="info") + log('Running garbage collection (this process may take some time) ...', level="info") # TODO: ideally nix would report some statistics about the GC as JSON run_nix(f'store delete --recursive {" ".join(to_gc)}') @@ -245,7 +259,7 @@ def get_dev_environment(ctx, logging=True): action = suppress() with action: gc_root = os.path.join(os.environ['DEVENV_GC'], 'shell') - env = run_nix(f"print-dev-env --profile '{gc_root}'") + env = run_nix(f"print-dev-env --profile '{gc_root}'", use_cachix=True) run_command(f"nix-env -p '{gc_root}' --delete-generations old", disable_stderr=True) symlink_force(Path(f'{ctx.obj["gc_project"]}-shell'), gc_root) return env, gc_root @@ -271,22 +285,24 @@ def shell(ctx, cmd, extra_args): run_nix(f"develop '{gc_root}'", replace_shell=True) def symlink_force(src, dst): + src = Path(src) + dst = Path(dst) # locking is needed until https://github.com/python/cpython/pull/14464 with FileLock(f"{dst}.lock", timeout=10): - src.unlink(missing_ok=True) - Path(src).symlink_to(dst) + dst.unlink(missing_ok=True) + dst.symlink_to(src) @cli.command( help="Starts processes in foreground. See http://devenv.sh/processes", short_help="Starts processes in foreground. See http://devenv.sh/processes", ) -@click.argument('command', required=False) +@click.argument('process', required=False) @click.option('--detach', '-d', is_flag=True, help="Starts processes in the background.") @click.pass_context -def up(ctx, command, detach): +def up(ctx, process, detach): with log_task('Building processes'): ctx.invoke(assemble) - procfilescript = run_nix(f"build --no-link --print-out-paths '.#procfileScript'") + procfilescript = run_nix("build --no-link --print-out-paths '.#procfileScript'", use_cachix=True) with open(procfilescript, 'r') as file: contents = file.read().strip() if contents == '': @@ -300,7 +316,7 @@ def up(ctx, command, detach): with open(processes_script, 'w') as f: f.write(f"""#!/usr/bin/env bash {env} -{procfilescript} {command or ""} +{procfilescript} {process or ""} """) os.chmod(processes_script, 0o755) @@ -315,7 +331,7 @@ def up(ctx, command, detach): file.write(str(process.pid)) log(f" PID is {process.pid}.", level="info") log(f" See logs: $ tail -f {PROCESSES_LOG}", level="info") - log(f" Stop: $ devenv processes stop", level="info") + log(" Stop: $ devenv processes stop", level="info") else: os.execv(processes_script, [processes_script]) @@ -348,7 +364,7 @@ def stop(): def search(ctx, name): """Search packages matching NAME in nixpkgs input.""" ctx.invoke(assemble) - options = run_nix(f"build --no-link --print-out-paths '.#optionsJSON' ") + options = run_nix("build --no-link --print-out-paths '.#optionsJSON' ", use_cachix=True) search = run_nix(f"search --json nixpkgs {name}") with open(Path(options) / 'share' / 'doc' / 'nixos' / 'options.json') as f: @@ -407,7 +423,7 @@ def container(ctx, registry, copy, copy_args, docker_run, container_name): with log_task(f'Building {container_name} container'): ctx.invoke(assemble) # NOTE: we need --impure here to read DEVENV_CONTAINER - spec = run_nix(f"build --impure --print-out-paths --no-link .#devenv.containers.\"{container_name}\".derivation") + spec = run_nix(f"build --impure --print-out-paths --no-link .#devenv.containers.\"{container_name}\".derivation", use_cachix=True) click.echo(spec) # copy container @@ -415,7 +431,7 @@ def container(ctx, registry, copy, copy_args, docker_run, container_name): with log_task(f'Copying {container_name} container'): # we need --impure here for DEVENV_CONTAINER copy_script = run_nix(f"build --print-out-paths --no-link \ - --impure .#devenv.containers.\"{container_name}\".copyScript") + --impure .#devenv.containers.\"{container_name}\".copyScript", use_cachix=True) if docker_run: registry = "docker-daemon:" @@ -429,7 +445,7 @@ def container(ctx, registry, copy, copy_args, docker_run, container_name): log(f'Starting {container_name} container', level="info") # we need --impure here for DEVENV_CONTAINER docker_script = run_nix(f"build --print-out-paths --no-link --impure \ - .#devenv.containers.\"{container_name}\".dockerRun") + .#devenv.containers.\"{container_name}\".dockerRun", use_cachix=True) subprocess.run(docker_script) @@ -515,14 +531,16 @@ def update(ctx, input_name): if input_name: run_nix(f"flake lock --update-input {input_name}") else: - run_nix(f"flake update") + run_nix("flake update") @cli.command() @click.pass_context def ci(ctx): """Builds your developer environment and checks if everything builds.""" ctx.invoke(assemble) - output_path = run_nix(f"build --no-link --print-out-paths --impure .#ci") + print("running ci") + print(run_command("cat ${FLAKE_FILE}")) + output_path = run_nix("build --no-link --print-out-paths .#ci", use_cachix=True) add_gc('ci', output_path) @cli.command(hidden=True) @@ -576,11 +594,11 @@ def add(ctx, name, url, follows): @click.pass_context def test(ctx, debug, names): ctx.invoke(assemble) - with log_task(f"Gathering tests", newline=False): - tests = json.loads(run_nix(f"eval .#devenv.tests --json")) + with log_task("Gathering tests", newline=False): + tests = json.loads(run_nix("eval .#devenv.tests --json")) if not names: - names = "local" + names = [ "local" ] # group tests by tags tags = {} @@ -598,12 +616,12 @@ def test(ctx, debug, names): if tag_tests: selected_tests.extend(tag_tests) - log(f"Found {len(selected_tests)} tests:", level="info") + log(f"Found {len(tests)} test(s), running {len(selected_tests)}:", level="info") pwd = os.getcwd() for name in selected_tests: - with log_task(f" Running {name}"): + with log_task(f" Testing {name}"): with tempfile.TemporaryDirectory(prefix=name + "_") as tmpdir: os.chdir(tmpdir) test = tests[name] @@ -646,16 +664,16 @@ def test(ctx, debug, names): devenv = sys.argv[0] has_processes = False try: - log(" Running $ devenv ci", level="info") + log(" Running $ devenv ci ...", level="info") run_command(f"{devenv} ci") has_processes = os.path.exists(".devenv/gc/ci") and "-devenv-up" in run_command("cat .devenv/gc/ci") if has_processes: - log(" Detected processes. Starting them.", level="info") + log(" Starting processes ...", level="info") run_command(f"{devenv} up -d") # stream logs - p = subprocess.Popen(f"tail -f .devenv/processes.log", + p = subprocess.Popen("tail -f .devenv/processes.log", shell=True, ) else: @@ -663,7 +681,7 @@ def test(ctx, debug, names): try: if os.path.exists(".test.sh"): - log(" Detected .test.sh. Running it.", level="info") + log(" Running .test.sh ...", level="info") run_command(f"{devenv} shell bash ./.test.sh") finally: if has_processes and not debug: @@ -671,18 +689,67 @@ def test(ctx, debug, names): if p: p.kill() except BaseException as e: - log(f"Test {name} failed.", level="error") + log_error(f"Test {name} failed.") if debug: - log(f"Entering shell because of the --debug flag:", level="warning") + log("Entering shell because of the --debug flag:", level="warning") log(f" - devenv: {devenv}", level="warning") log(f" - cwd: {tmpdir}", level="warning") if has_processes: - log(f" - up logs: .devenv/processes.log:", level="warning") + log(" - up logs: .devenv/processes.log:", level="warning") os.execv("/bin/sh", ["/bin/sh"]) else: + log_warning('Pass --debug flag to enter shell.') raise e def write_if_defined(file, content): if content: with open(file, 'w') as f: - f.write(content) \ No newline at end of file + f.write(content) + +@functools.cache +def get_cachix_caches(): + """Get the full list of cachix caches we need and their public keys. + + This is cached because it's expensive to run. + """ + + caches = json.loads(run_nix("eval .#devenv.cachix --json")) + if CACHIX_KNOWN_PUBKEYS.exists(): + known_keys = json.loads(CACHIX_KNOWN_PUBKEYS.read_text()) + else: + known_keys = {} + new_known_keys = {} + for name in caches.get("pull", []): + if name not in known_keys: + resp = requests.get(f"https://cachix.org/api/v1/cache/{name}") + if resp.status_code in [401, 404]: + log_error(f"Cache {name} does not exist or you don't have a CACHIX_AUTH_TOKEN configured.") + # TODO: instruct how to best configure netrc + #log_error("To configure a token, run `cachix authtoken `.") + log_error("To create a cache, go to https://app.cachix.org/.") + exit(1) + else: + resp.raise_for_status() + pubkey = resp.json()["publicSigningKeys"][0] + new_known_keys[name] = pubkey + + if caches.get("pull"): + log_info(f"Using Cachix: {', '.join(caches.get('pull', []))} ") + if new_known_keys: + for name, pubkey in new_known_keys.items(): + log_info(f" Trusting {name}.cachix.org on first use with the public key {pubkey}") + known_keys.update(new_known_keys) + CACHIX_KNOWN_PUBKEYS.write_text(json.dumps(known_keys)) + return caches, known_keys + +@cli.command() +@click.argument('attrs', nargs=-1, required=True) +@click.pass_context +def build(ctx, attrs): + """Build attributes in your devenv.nix.""" + ctx.invoke(assemble) + attrs = " ".join(map(lambda attr: f'.#devenv.{attr}', attrs)) + output = run_nix(f"build --print-out-paths --print-build-logs --no-link {attrs}", use_cachix=True) + log("Built:", level="info") + for path in output.splitlines(): + log(path, level="info") \ No newline at end of file diff --git a/src/devenv/log.py b/src/devenv/log.py index 4d25a5233..64ac34c07 100644 --- a/src/devenv/log.py +++ b/src/devenv/log.py @@ -35,3 +35,15 @@ def log(message, level: LogLevel): click.echo(click.style("✖ ", fg="red") + message, err=True) case "debug": click.echo(click.style("• ", fg="magenta") + message, err=True) + +def log_error(message): + log(message, "error") + +def log_warning(message): + log(message, "warning") + +def log_info(message): + log(message, "info") + +def log_debug(message): + log(message, "debug") \ No newline at end of file diff --git a/src/modules/cachix.nix b/src/modules/cachix.nix new file mode 100644 index 000000000..32caad229 --- /dev/null +++ b/src/modules/cachix.nix @@ -0,0 +1,28 @@ +{ lib, config, ... }: +let + cfg = config.cachix; +in +{ + options.cachix = { + pull = lib.mkOption { + type = lib.types.listOf lib.types.str; + description = "What caches to pull from."; + }; + + push = lib.mkOption { + type = lib.types.nullOr lib.types.str; + description = "What cache to push to. Automatically also adds it to the list of caches to pull from."; + default = null; + }; + }; + + config = { + cachix.pull = [ "devenv" ] + ++ (lib.optionals (cfg.push != null) [ config.cachix.push ]); + + warnings = lib.optionals (lib.versionOlder config.devenv.cliVersion "1.0") '' + For cachix.push and cachix.pull attributes to have an effect, + upgrade to devenv 1.0 or later. + ''; + }; +} diff --git a/src/modules/flake.tmpl.nix b/src/modules/flake.tmpl.nix index de839cb91..92f403721 100644 --- a/src/modules/flake.tmpl.nix +++ b/src/modules/flake.tmpl.nix @@ -88,9 +88,7 @@ inherit (config) info procfileScript procfileEnv procfile; ci = config.ciDerivation; }; - devenv = { - inherit (config) containers tests; - }; + devenv = config; devShell."${system}" = config.shell; }; } diff --git a/src/modules/languages/python.nix b/src/modules/languages/python.nix index 3946fa7bb..360e2cf15 100644 --- a/src/modules/languages/python.nix +++ b/src/modules/languages/python.nix @@ -181,6 +181,8 @@ in (lib.mkIf (cfg.version != null) (nixpkgs-python.packages.${pkgs.stdenv.system}.${cfg.version} or (throw "Unsupported Python version, see https://github.com/cachix/nixpkgs-python#supported-python-versions"))) ]; + cachix.pull = lib.mkIf (cfg.version != null) [ "nixpkgs-python" ]; + packages = [ cfg.package ] ++ (lib.optional cfg.poetry.enable cfg.poetry.package); diff --git a/src/modules/lib.nix b/src/modules/lib.nix index 63a7ddce3..7ffd7cc04 100644 --- a/src/modules/lib.nix +++ b/src/modules/lib.nix @@ -32,10 +32,10 @@ in inputs.${name} or (throw "To use '${attribute}', ${command}\n\n"); - mkTests = tags: folder: + mkTests = folder: let mk = dir: { - inherit tags; + tags = [ "local" ]; src = "${folder}/${dir}"; }; in diff --git a/src/modules/tests.nix b/src/modules/tests.nix index 994c8fc5b..5eba38745 100644 --- a/src/modules/tests.nix +++ b/src/modules/tests.nix @@ -35,6 +35,7 @@ let src = lib.mkOption { type = lib.types.nullOr lib.types.path; + default = null; description = "Source code with all the files."; }; }; diff --git a/src/modules/top-level.nix b/src/modules/top-level.nix index f91b63b3b..3e320a4eb 100644 --- a/src/modules/top-level.nix +++ b/src/modules/top-level.nix @@ -135,7 +135,7 @@ in default = [ ]; example = [ "you should fix this or that" ]; description = '' - This option allows modules to express warnings about theV + This option allows modules to express warnings about the configuration. For example, `lib.mkRenamedOptionModule` uses this to display a warning message when a renamed option is used. ''; @@ -175,6 +175,7 @@ in ./debug.nix ./lib.nix ./tests.nix + ./cachix.nix ] ++ (listEntries ./languages) ++ (listEntries ./services) @@ -233,8 +234,10 @@ in pkgs.mkShell ({ name = "devenv-shell"; packages = [ profile ]; - shellHook = config.enterShell; - debug = config.devenv.debug; + shellHook = '' + ${lib.optionalString config.devenv.debug "set -x"} + ${config.enterShell} + ''; } // config.env) ); diff --git a/tests/cli/.test.sh b/tests/cli/.test.sh new file mode 100755 index 000000000..d9cdf6e3c --- /dev/null +++ b/tests/cli/.test.sh @@ -0,0 +1,2 @@ +set -xe +devenv build languages.python.package \ No newline at end of file diff --git a/tests/cli/devenv.nix b/tests/cli/devenv.nix new file mode 100644 index 000000000..ecce39d5a --- /dev/null +++ b/tests/cli/devenv.nix @@ -0,0 +1,3 @@ +{ ... }: { + languages.python.enable = true; +}