From c5324bcad01fa2001ffd0f19a3893bc0a4e8d8e7 Mon Sep 17 00:00:00 2001 From: Shelvacu Date: Mon, 22 Jul 2024 20:57:09 -0700 Subject: [PATCH] Grab proot from bootstrap zip rather than including its nix path directly. This means that the cachix substituter (or already having the package in your nix store somehow) is no longer required to build. This required reworking the deploy script. As a bonus you can now omit the second argument and it will tell you what it would have copied instead of copying anything. This is fixes one source of impurity, but for now flake builds will still require the --impure flag --- .github/workflows/emulator.yml | 2 +- flake.nix | 2 +- modules/environment/login/default.nix | 23 ++- .../environment/login/proot-attrs/aarch64.nix | 5 + .../environment/login/proot-attrs/default.nix | 4 + .../environment/login/proot-attrs/x86_64.nix | 5 + pkgs/default.nix | 2 + scripts/deploy.nix | 63 +++--- scripts/deploy.py | 181 ++++++++++++++++++ scripts/deploy.sh | 89 --------- scripts/local-deploy.sh | 10 + scripts/setup.py | 13 ++ 12 files changed, 278 insertions(+), 121 deletions(-) create mode 100644 modules/environment/login/proot-attrs/aarch64.nix create mode 100644 modules/environment/login/proot-attrs/default.nix create mode 100644 modules/environment/login/proot-attrs/x86_64.nix create mode 100644 scripts/deploy.py delete mode 100755 scripts/deploy.sh create mode 100755 scripts/local-deploy.sh create mode 100644 scripts/setup.py diff --git a/.github/workflows/emulator.yml b/.github/workflows/emulator.yml index 9ecd277f..9a495229 100644 --- a/.github/workflows/emulator.yml +++ b/.github/workflows/emulator.yml @@ -47,7 +47,7 @@ jobs: rm -rf n-o-d mkdir -p n-o-d git -C . archive --format=tar.gz --prefix n-o-d/ HEAD > n-o-d/archive.tar.gz - ARCHES=x86_64 nix run '.#deploy' -- file:///data/local/tmp/n-o-d/archive.tar.gz n-o-d/ + ARCHES=x86_64 nix run '.#deploy' -- file:///data/local/tmp/n-o-d/archive.tar.gz --rsync-target n-o-d/ tar cf n-o-d.tar n-o-d - name: Store zipball and channel tarball to inject (n-o-d) diff --git a/flake.nix b/flake.nix index ad92e985..6f6c6c45 100644 --- a/flake.nix +++ b/flake.nix @@ -59,7 +59,7 @@ deploy = { type = "app"; - program = toString (import ./scripts/deploy.nix { inherit nixpkgs system; }); + program = import ./scripts/deploy.nix { inherit nixpkgs system; }; }; }); diff --git a/modules/environment/login/default.nix b/modules/environment/login/default.nix index b4f7a936..75d09d2a 100644 --- a/modules/environment/login/default.nix +++ b/modules/environment/login/default.nix @@ -37,7 +37,7 @@ in prootStatic = mkOption { type = types.package; - readOnly = true; + # not readOnly, this needs to be overridden when building bootstrap zip internal = true; description = "proot-static package."; }; @@ -84,14 +84,27 @@ in environment.files = { inherit login loginInner; + # Ideally this would build the static proot binary, but doing that on aarch64 is HARD so instead pull it from the bootstrap tarball prootStatic = let - crossCompiledPaths = { - aarch64-linux = "/nix/store/7qd99m1w65x2vgqg453nd70y60sm3kay-proot-termux-static-aarch64-unknown-linux-android-unstable-2024-05-04"; - x86_64-linux = "/nix/store/pakj3svvw84rhkzdc6211yhc2cgvc21f-proot-termux-static-x86_64-unknown-linux-android-unstable-2024-05-04"; + attrs = (import ./proot-attrs).${targetSystem}; + prootFile = pkgs.fetchurl { + name = "proot-static-file"; + inherit (attrs) url hash; + + downloadToTemp = true; + executable = true; + postFetch = '' + ${pkgs.unzip}/bin/unzip -u $downloadedFile bin/proot-static + echo $PWD >&2 + mv bin/proot-static $out + ''; }; in - "${crossCompiledPaths.${targetSystem}}"; + pkgs.runCommand "proot-static" { } '' + mkdir -p $out/bin + cp ${prootFile} $out/bin/proot-static + ''; }; }; diff --git a/modules/environment/login/proot-attrs/aarch64.nix b/modules/environment/login/proot-attrs/aarch64.nix new file mode 100644 index 00000000..b1e4e57c --- /dev/null +++ b/modules/environment/login/proot-attrs/aarch64.nix @@ -0,0 +1,5 @@ +# WARNING: This file is autogenerated by the deploy script. Any changes will be overridden +{ + url = "https://nix-on-droid.unboiled.info/bootstrap-testing/bootstrap-aarch64.zip"; + hash = "sha256-fZyqldmHWbv2e6543mwb5UPcKxcODRg+PDvZNcjVyUU="; +} diff --git a/modules/environment/login/proot-attrs/default.nix b/modules/environment/login/proot-attrs/default.nix new file mode 100644 index 00000000..891ce5e8 --- /dev/null +++ b/modules/environment/login/proot-attrs/default.nix @@ -0,0 +1,4 @@ +{ + x86_64-linux = import ./x86_64.nix; + aarch64-linux = import ./aarch64.nix; +} diff --git a/modules/environment/login/proot-attrs/x86_64.nix b/modules/environment/login/proot-attrs/x86_64.nix new file mode 100644 index 00000000..8fad9d7a --- /dev/null +++ b/modules/environment/login/proot-attrs/x86_64.nix @@ -0,0 +1,5 @@ +# WARNING: This file is autogenerated by the deploy script. Any changes will be overridden +{ + url = "https://nix-on-droid.unboiled.info/bootstrap-testing/bootstrap-x86_64.zip"; + hash = "sha256-1WZBmFNEmZucOwuzDAFO+Sl+b2XO7Lp1aQr/4egl4wU="; +} diff --git a/pkgs/default.nix b/pkgs/default.nix index 1840faa9..15c26d67 100644 --- a/pkgs/default.nix +++ b/pkgs/default.nix @@ -42,6 +42,8 @@ let # Fix invoking bash after initial build. user.shell = "${initialPackageInfo.bash}/bin/bash"; + environment.files.prootStatic = pkgs.lib.mkForce customPkgs.prootTermux; + build = { channel = { nixpkgs = urlOptionValue nixpkgsChannelURL "NIXPKGS_CHANNEL_URL"; diff --git a/scripts/deploy.nix b/scripts/deploy.nix index f959b214..e7952d4b 100644 --- a/scripts/deploy.nix +++ b/scripts/deploy.nix @@ -4,31 +4,44 @@ let pkgs = nixpkgs.legacyPackages.${system}; - - runtimePackages = with pkgs; [ - coreutils - git - gnugrep - gnused - gnutar - gzip - jq - nix - openssh - rsync + pypkgs = pkgs.python311Packages; + disablePyLints = [ + "line-too-long" + "missing-module-docstring" + "wrong-import-position" # import should be at top of file: we purposefully don't import click and such so that users that try to run the script directly get a friendly error + "missing-function-docstring" + # c'mon, it's a script + "too-many-locals" + "too-many-branches" + "too-many-statements" ]; -in + deriv = pypkgs.buildPythonApplication { + pname = "deploy"; + version = "0.0"; + src = ./.; + + inherit (pkgs) nix git rsync; -pkgs.runCommand - "deploy" -{ - preferLocalBuild = true; - allowSubstitutes = false; -} - '' - install -D -m755 ${./deploy.sh} $out + propagatedBuildInputs = [ pypkgs.click ]; - substituteInPlace $out \ - --subst-var-by bash "${pkgs.bash}" \ - --subst-var-by path "${pkgs.lib.makeBinPath runtimePackages}" - '' + doCheck = true; + nativeCheckInputs = with pypkgs; [ mypy pylint black ]; + checkPhase = '' + mypy --strict --no-color deploy.py + PYLINTHOME="$PWD/.pylint" pylint \ + --score=n \ + --clear-cache-post-run=y \ + --disable=${pkgs.lib.concatStringsSep "," disablePyLints} \ + deploy.py + black --check --diff deploy.py + ''; + + patchPhase = '' + substituteInPlace deploy.py \ + --subst-var nix \ + --subst-var git \ + --subst-var rsync + ''; + }; +in +"${deriv}/bin/deploy" diff --git a/scripts/deploy.py b/scripts/deploy.py new file mode 100644 index 00000000..a6507867 --- /dev/null +++ b/scripts/deploy.py @@ -0,0 +1,181 @@ +import os +import sys + +GIT = "@git@/bin/git" +NIX = "@nix@/bin/nix" +NIX_HASH = "@nix@/bin/nix-hash" +RSYNC = "@rsync@/bin/rsync" + +if GIT.startswith("@"): + sys.stderr.write( + "Do not run this script directly, instead try: nix run .#deploy -- --help" + ) + sys.exit(1) + +import subprocess +import re +import inspect +from typing import Never +from pathlib import Path + +import click + + +def err(text: str) -> Never: + sys.stderr.write(text) + sys.exit(1) + + +def run(*args: str) -> None: + subprocess.run(args, check=True) + + +def run_capture(*args: str, env: dict[str, str] | None = None) -> str: + proc = subprocess.run( + args, check=True, stdout=subprocess.PIPE, stderr=None, env=env + ) + return proc.stdout.decode("utf-8").strip() + + +def log(msg: str) -> None: + print(f"> {msg}") + + +@click.command() +@click.option( + "--rsync-target", + help="Where bootstrap zipballs and source tarball will be copied to. If given, this is passed directly to rsync so it can be a local folder, ssh path, etc. For production builds this should be a webroot directory that will be served at bootstrap-url", +) +@click.option( + "--bootstrap-url", + help="URL where bootstrap zip files are available. Defaults to folder part of public-url if not given.", +) +@click.option( + "--arches", + default="aarch64,x86_64", + help="Which architectures to build for, comma-separated.", +) +@click.argument("public-url") +def go( + public_url: str, + rsync_target: str | None, + bootstrap_url: str | None, + arches: str, +) -> None: + """ + Builds bootstrap zip balls and source code tar ball (for usage as a channel or flake). If rsync_target is specified, uploads it to the directory specified in rsync_target. The contents of this directory should be reachable by the android device with public_url. + + Examples: + + \b + $ nix run .#deploy -- \\ + 'https://example.com/bootstrap/source.tar.gz' \\ + --rsync-target 'user@host:/path/to/bootstrap' + + \b + $ nix run .#deploy -- \\ + 'github:USER/nix-on-droid/BRANCH' \\ + --rsync-target 'user@host:/path/to/bootstrap' \\ + --bootstrap-url 'https://example.com/bootstrap/' + + \b + $ nix run .#deploy -- \\ + 'file:///data/local/tmp/n-o-d/archive.tar.gz' + + ^ useful for testing. Note this is a path on the android device running the APK, not on the build machine + """ + repo_dir = run_capture(GIT, "rev-parse", "--show-toplevel") + os.chdir(repo_dir) + source_file = "source.tar.gz" + if (m := re.search("^github:(.*)/(.*)/(.*)", public_url)) is not None: + channel_url = f"https://github.com/{m[1]}/{m[2]}/archive/{m[3]}.tar.gz" + if bootstrap_url is None: + err("--botstrap-url must be provided for github URLs") + elif re.search("^(https?|file)://", public_url): + channel_url = public_url + else: + err(f"unsupported url {public_url}") + + # for CI and local testing + if (m := re.search("^file:///(.*)/archive.tar.gz$", public_url)) is not None: + flake_url = f"/{m[1]}/unpacked" + else: + flake_url = public_url + base_url = re.sub("/[^/]*$", "", public_url) + if bootstrap_url is None: + bootstrap_url = base_url + + log(f"channel_url = {channel_url}") + log(f"flake_url = {flake_url}") + log(f"base_url = {base_url}") + log(f"bootstrap_url = {bootstrap_url}") + + uploads: list[str] = [] + + for arch in arches.split(","): + log(f"building {arch} proot...") + proot = run_capture( + NIX, "build", "--no-link", "--print-out-paths", f".#prootTermux-{arch}" + ) + proot_hash = run_capture( + NIX_HASH, "--type", "sha256", "--sri", f"{proot}/bin/proot-static" + ) + attrs_file = Path(f"modules/environment/login/proot-attrs/{arch}.nix") + attrs_text = inspect.cleandoc( + f""" + # WARNING: This file is autogenerated by the deploy script. Any changes will be overridden + {{ + url = "{bootstrap_url}/bootstrap-{arch}.zip"; + hash = "{proot_hash}"; + }} + """ + ) + # nixpkgs-fmt insists files must end with a newline + attrs_text = attrs_text + "\n" + write_attrs_file = True + if not attrs_file.exists(): + log(f"warn: {attrs_file} not present; creating") + elif (old_attrs_text := attrs_file.read_text(encoding="utf-8")) != attrs_text: + log(f"updating contents of {attrs_file}") + print("<<<<<<") + print(old_attrs_text) + print("======") + print(attrs_text) + print(">>>>>>") + else: + write_attrs_file = False + log(f"no changes needed to {attrs_file}") + + if write_attrs_file: + attrs_file.write_text(attrs_text, newline="\n", encoding="utf-8") + log(f"adding {attrs_file} to git index") + run(GIT, "add", str(attrs_file)) + + bootstrap_zip_store_path = run_capture( + NIX, + "build", + "--no-link", + "--print-out-paths", + "--impure", + f".#bootstrapZip-{arch}", + env={ + "NIX_ON_DROID_CHANNEL_URL": channel_url, + "NIX_ON_DROID_FLAKE_URL": flake_url, + }, + ) + uploads.append(bootstrap_zip_store_path + f"/bootstrap-{arch}.zip") + + log("creating tarball of current HEAD") + run(GIT, "archive", "--prefix", "nix-on-droid/", "--output", source_file, "HEAD") + uploads.append(source_file) + + if rsync_target is not None: + log("uploading artifacts...") + run(RSYNC, "--progress", *uploads, rsync_target) + else: + log(f"Would have uploaded {uploads}") + + +if __name__ == "__main__": + # pylint: disable = no-value-for-parameter + go() diff --git a/scripts/deploy.sh b/scripts/deploy.sh deleted file mode 100755 index b54479fa..00000000 --- a/scripts/deploy.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!@bash@/bin/bash -set -euo pipefail - -PATH=@path@ - -if [[ $# -ne 2 ]]; then - cat >&2 < - -Builds bootstrap zip ball and source code tar ball (for usage as a channel or -flake) and uploads it to the directory specified in . The -contents of this directory should be reachable by the android device with -. - -Examples: -$ nix run .#deploy -- 'https://example.com/bootstrap/source.tar.gz' 'user@host:/path/to/bootstrap' -$ nix run .#deploy -- 'github:USER/nix-on-droid/BRANCH' 'user@host:/path/to/bootstrap' - -EOF - exit 1 -fi - -PUBLIC_URL="$1" -RSYNC_TARGET="$2" -: ${ARCHES:=aarch64 x86_64} - -# this allows to run this script from every place in this git repo -REPO_DIR="$(git rev-parse --show-toplevel)" - -cd "$REPO_DIR" - -SOURCE_FILE="source.tar.gz" - -function log() { - echo "> $*" -} - - -if [[ "$PUBLIC_URL" =~ ^github:(.*)/(.*)/(.*) ]]; then - export NIX_ON_DROID_CHANNEL_URL="https://github.com/${BASH_REMATCH[1]}/${BASH_REMATCH[2]}/archive/${BASH_REMATCH[3]}.tar.gz" -else - [[ "$PUBLIC_URL" =~ ^https?:// ]] || \ - [[ "$PUBLIC_URL" =~ ^file:/// ]] || \ - { echo "unsupported url $PUBLIC_URL" >&2; exit 1; } - export NIX_ON_DROID_CHANNEL_URL="$PUBLIC_URL" -fi -# special case for local / CI testing -if [[ "$PUBLIC_URL" =~ ^file:///(.*)/archive.tar.gz ]]; then - export NIX_ON_DROID_FLAKE_URL="/${BASH_REMATCH[1]}/unpacked" -else - export NIX_ON_DROID_FLAKE_URL="$PUBLIC_URL" -fi -log "NIX_ON_DROID_CHANNEL_URL=$NIX_ON_DROID_CHANNEL_URL" -log "NIX_ON_DROID_FLAKE_URL=$NIX_ON_DROID_FLAKE_URL" - - -PROOT_HASH_FILE="modules/environment/login/default.nix" -UPLOADS=() -for arch in $ARCHES; do - log "building $arch proot..." - proot="$(nix build --no-link --print-out-paths ".#prootTermux-${arch}")" - - if grep -q "$arch-linux = \"$proot\";" "$PROOT_HASH_FILE"; then - log "keeping $arch proot path in $PROOT_HASH_FILE" - elif grep -q "$arch-linux = \"/nix/store/" "$PROOT_HASH_FILE"; then - log "patching $arch proot path in $PROOT_HASH_FILE..." - grep "$arch-linux = \"/nix/store/" "$PROOT_HASH_FILE" - sed -i "s|$arch-linux = \"/nix/store/.*\";|$arch-linux = \"$proot\";|" "$PROOT_HASH_FILE" - log " ->" - grep "$arch-linux = \"/nix/store/" "$PROOT_HASH_FILE" - else - log "no $arch proot hash found in $PROOT_HASH_FILE!" - exit 1 - fi - - log "building $arch bootstrapZip..." - BOOTSTRAP_ZIP="$(nix build --no-link --print-out-paths --impure ".#bootstrapZip-${arch}")" - UPLOADS+=($BOOTSTRAP_ZIP/bootstrap-$arch.zip) -done - - -log "creating tar ball of current HEAD..." -git archive --prefix nix-on-droid/ --output "$SOURCE_FILE" HEAD -UPLOADS+=($SOURCE_FILE) - - -log "uploading artifacts..." -rsync --progress "${UPLOADS[@]}" "$RSYNC_TARGET" diff --git a/scripts/local-deploy.sh b/scripts/local-deploy.sh new file mode 100755 index 00000000..1c9bc3cd --- /dev/null +++ b/scripts/local-deploy.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -xevo pipefail +tmp=`mktemp -d`/n-o-d +mkdir $tmp +cd "$(dirname "$0")"/.. +git -C . archive --format=tar.gz --prefix n-o-d/ HEAD > $tmp/archive.tar.gz +ARCHES=x86_64 nix run .#deploy -- file:///data/local/tmp/n-o-d/archive.tar.gz --rsync-target $tmp/ +adb shell 'rm -rf /data/local/tmp/n-o-d' +adb push $tmp /data/local/tmp/ +adb shell 'cd /data/local/tmp/n-o-d && tar xzof archive.tar.gz && mv n-o-d unpacked' diff --git a/scripts/setup.py b/scripts/setup.py new file mode 100644 index 00000000..11bf1097 --- /dev/null +++ b/scripts/setup.py @@ -0,0 +1,13 @@ +from setuptools import setup, find_packages + +setup( + name='nix-on-droid-deploy-script', + version='0.0', + packages=[], + py_modules=["deploy"], + entry_points={ + 'console_scripts': [ + 'deploy=deploy:go', + ], + }, +)