testscripts: better caching with Docker (#2245)

Mount a Docker volume to /nix when running `docker-testscripts` to speed
up tests. Also set `XDG_CACHE_HOME=/nix/cache` so that the tarball and
eval cache persist to the volume. Update `setupTestEnv` to share the pip
cache between tests (the same way we do with Nix's cache).

This cuts the duration of python linker tests in half (down from about 1
minute to about 30s).

devbox.json

@@ -57,8 +57,8 @@
         "trap 'rm -f testscripts-linux-amd64 testscripts-linux-arm64' EXIT",
         "GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go test -c -o testscripts-linux-amd64",
         "GOOS=linux GOARCH=arm64 CGO_ENABLED=0 go test -c -o testscripts-linux-arm64",
-        "image=$(docker build --quiet --tag devbox-ubuntu:noble --platform linux/amd64 .)",
-        "docker run --rm --platform linux/amd64 -e DEVBOX_RUN_FAILING_TESTS -e DEVBOX_RUN_PROJECT_TESTS -e DEVBOX_DEBUG $image \"$@\"",
+        "image=$(docker build --quiet --tag devbox-testscripts-ubuntu:noble --platform linux/amd64 .)",
+        "docker run --rm --mount type=volume,src=devbox-testscripts-amd64,dst=/nix --platform linux/amd64 -e DEVBOX_RUN_FAILING_TESTS -e DEVBOX_RUN_PROJECT_TESTS -e DEVBOX_DEBUG $image \"$@\"",
       ],
     },
   },

testscripts/Dockerfile

@@ -19,13 +19,19 @@ FROM $BASEIMAGE:$BASETAG
 ARG TARGETOS
 ARG TARGETARCH
 
+ENV XDG_CACHE_HOME=/nix/cache
+
 COPY --from=installer --link /nix-installer-$TARGETOS-$TARGETARCH nix-installer
 RUN <<EOF
 set -e
 
+mkdir -p "$XDG_CACHE_HOME"
+
 # Setting this env var is the same as passing --extra-conf to nix-installer.
 export NIX_INSTALLER_EXTRA_CONF="filter-syscalls = false
-sandbox = false"
+sandbox = false
+experimental-features = nix-command flakes fetch-closure ca-derivations
+use-xdg-base-directories = true"
 
 ./nix-installer install linux --no-confirm --logger full --init none
 rm nix-installer
@@ -35,6 +41,6 @@ ENV SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt
 COPY --link */*.test.txt /devbox/testscripts/
 COPY --link testscripts-$TARGETOS-$TARGETARCH /devbox/testscripts/test
 
-VOLUME /nix/store
+VOLUME "/nix"
 WORKDIR /devbox/testscripts
 ENTRYPOINT [ "/devbox/testscripts/test" ]

testscripts/languages/python_old_glibc.test.txt

@@ -31,7 +31,6 @@ except psycopg2.OperationalError:
   },
   "env": {
     "PIP_DISABLE_PIP_VERSION_CHECK": "1",
-    "PIP_NO_CACHE_DIR":              "1",
     "PIP_NO_INPUT":                  "1",
     "PIP_NO_PYTHON_VERSION_WARNING": "1",
     "PIP_PROGRESS_BAR":              "off",

testscripts/testrunner/examplesrunner.go

@@ -90,7 +90,7 @@ func runSingleDevboxTestscript(t *testing.T, dir, projectDir string) {
 		t.Error(err)
 	}
 
-	params := getTestscriptParams(t, testscriptDir)
+	params := getTestscriptParams(testscriptDir)
 
 	// save a reference to the original params.Setup so that we can wrap it below
 	setup := params.Setup

testscripts/testrunner/setupenv.go

@@ -13,68 +13,66 @@ import (
 	"go.jetpack.io/devbox/internal/xdg"
 )
 
-func setupTestEnv(t *testing.T, envs *testscript.Env) error {
-	setupPATH(envs)
-
-	setupHome(t, envs)
-
-	err := setupCacheHome(envs)
-	if err != nil {
-		return err
-	}
-
-	propagateEnvVars(envs,
+// setupTestEnv configures env for devbox tests.
+func setupTestEnv(env *testscript.Env) error {
+	setupPATH(env)
+	setupHome(env)
+	setupCacheHome(env)
+	propagateEnvVars(env,
 		debug.DevboxDebug, // to enable extra logging
 		"SSL_CERT_FILE",   // so HTTPS works with Nix-installed certs
 	)
 	return nil
 }
 
-func setupHome(t *testing.T, envs *testscript.Env) {
-	// We set a HOME env-var because:
-	// 1. testscripts overrides it to /no-home, presumably to improve isolation
-	// 2. but many language tools rely on a $HOME being set, and break due to 1.
-	//    examples include ~/.dotnet folder and GOCACHE=$HOME/Library/Caches/go-build
-	envs.Setenv(envir.Home, t.TempDir())
+// setupHome sets the test's HOME to a unique temp directory. The testscript
+// package sets it to /no-home by default (presumably to improve isolation), but
+// this breaks most programs.
+func setupHome(env *testscript.Env) {
+	env.Setenv(envir.Home, env.T().(testing.TB).TempDir())
 }
 
-func setupPATH(envs *testscript.Env) {
-	// Ensure path is empty so that we rely only on the PATH set by devbox
-	// itself.
-	// The one entry we need to keep is the /bin directory in the testing directory.
-	// That directory is setup by the testing framework itself, and it's what allows
-	// us to call our own custom "devbox" command.
-	oldPath := envs.Getenv(envir.Path)
-	newPath := strings.Split(oldPath, ":")[0]
-	envs.Setenv(envir.Path, newPath)
+// setupPATH removes all directories from the test's PATH to ensure that it only
+// uses the PATH set by devbox. The one exception is the testscript's bin
+// directory, which contains the commands given to testscript.RunMain
+// (such as devbox itself).
+func setupPATH(env *testscript.Env) {
+	s, _, _ := strings.Cut(env.Getenv(envir.Path), string(filepath.ListSeparator))
+	env.Setenv(envir.Path, s)
 }
 
-func setupCacheHome(envs *testscript.Env) error {
-	// Both devbox itself and nix occasionally create some files in
-	// XDG_CACHE_HOME (which defaults to ~/.cache). For purposes of this
-	// test set it to a location within the test's working directory:
-	cacheHome := filepath.Join(envs.WorkDir, ".cache")
-	envs.Setenv(envir.XDGCacheHome, cacheHome)
-	err := os.MkdirAll(cacheHome, 0o755) // Ensure dir exists.
-	if err != nil {
-		return err
-	}
+// setupCacheHome sets the test's XDG_CACHE_HOME to a unique temp directory so
+// that it doesn't share caches with other tests or the user's system. For
+// programs where this would make tests too slow, it symlinks specific cache
+// subdirectories to a shared location that persists between test runs. For
+// example, $WORK/.cache/nix would symlink to $XDG_CACHE_HOME/devbox-tests/nix
+// so that Nix doesn't re-download tarballs for every test.
+func setupCacheHome(env *testscript.Env) {
+	t := env.T().(testing.TB) //nolint:varnamelen
 
-	// There is one directory we do want to share across tests: nix's cache.
-	// Without it tests are very slow, and nix would end up re-downloading
-	// nixpkgs every time.
-	// Here we create a shared location for nix's cache, and symlink from
-	// the test's working directory.
-	err = os.MkdirAll(xdg.CacheSubpath("devbox-tests/nix"), 0o755) // Ensure dir exists.
+	cacheHome := filepath.Join(env.WorkDir, ".cache")
+	env.Setenv(envir.XDGCacheHome, cacheHome)
+	err := os.MkdirAll(cacheHome, 0o755)
 	if err != nil {
-		return err
-	}
-	err = os.Symlink(xdg.CacheSubpath("devbox-tests/nix"), filepath.Join(cacheHome, "nix"))
-	if err != nil {
-		return err
+		t.Fatal("create XDG_CACHE_HOME for test:", err)
 	}
 
-	return nil
+	// Symlink cache subdirectories that we want to share and persist
+	// between tests.
+	sharedCacheDir := xdg.CacheSubpath("devbox-tests")
+	for _, subdir := range []string{"nix", "pip"} {
+		sharedSubdir := filepath.Join(sharedCacheDir, subdir)
+		err := os.MkdirAll(sharedSubdir, 0o755)
+		if err != nil {
+			t.Fatal("create shared XDG_CACHE_HOME subdir:", err)
+		}
+
+		testSubdir := filepath.Join(cacheHome, subdir)
+		err = os.Symlink(sharedSubdir, testSubdir)
+		if err != nil {
+			t.Fatal("symlink test's XDG_CACHE_HOME subdir to shared XDG_CACHE_HOME subdir:", err)
+		}
+	}
 }
 
 // propagateEnvVars propagates the values of environment variables to the test

testscripts/testrunner/testrunner.go

@@ -43,7 +43,7 @@ func RunTestscripts(t *testing.T, testscriptsDir string) {
 			continue
 		}
 
-		testscript.Run(t, getTestscriptParams(t, dir))
+		testscript.Run(t, getTestscriptParams(dir))
 	}
 }
 
@@ -80,12 +80,12 @@ func copyFileCmd(script *testscript.TestScript, neg bool, args []string) {
 	script.Check(err)
 }
 
-func getTestscriptParams(t *testing.T, dir string) testscript.Params {
+func getTestscriptParams(dir string) testscript.Params {
 	return testscript.Params{
 		Dir:                 dir,
 		RequireExplicitExec: true,
 		TestWork:            false, // Set to true if you're trying to debug a test.
-		Setup:               func(env *testscript.Env) error { return setupTestEnv(t, env) },
+		Setup:               func(env *testscript.Env) error { return setupTestEnv(env) },
 		Cmds: map[string]func(ts *testscript.TestScript, neg bool, args []string){
 			"cp":                           copyFileCmd,
 			"devboxjson.packages.contains": assertDevboxJSONPackagesContains,