Run package spec tests for packaging targets

Before this the package tests were only run when building a container
and not when just building a package.

Signed-off-by: Brian Goff <cpuguy83@gmail.com>

frontend/azlinux/handle_container.go

@@ -30,59 +30,12 @@ func handleContainer(w worker) gwclient.BuildFunc {
 				return nil, nil, fmt.Errorf("error creating rpm: %w", err)
 			}
 
-			rpms, err := readRPMs(ctx, client, rpmDir)
-			if err != nil {
-				return nil, nil, err
-			}
-
-			st, err := specToContainerLLB(w, spec, targetKey, rpmDir, rpms, sOpt, pg)
-			if err != nil {
-				return nil, nil, err
-			}
-
-			def, err := st.Marshal(ctx, pg)
-			if err != nil {
-				return nil, nil, fmt.Errorf("error marshalling llb: %w", err)
-			}
-
-			res, err := client.Solve(ctx, gwclient.SolveRequest{
-				Definition: def.ToPB(),
-			})
-			if err != nil {
-				return nil, nil, err
-			}
-
 			img, err := resolveBaseConfig(ctx, w, client, platform, spec, targetKey)
 			if err != nil {
 				return nil, nil, errors.Wrap(err, "could not resolve base image config")
 			}
 
-			ref, err := res.SingleRef()
-			if err != nil {
-				return nil, nil, err
-			}
-
-			base, err := w.Base(sOpt, pg)
-			if err != nil {
-				return nil, nil, err
-			}
-
-			withTestDeps := func(in llb.State) llb.State {
-				deps := spec.GetTestDeps(targetKey)
-				if len(deps) == 0 {
-					return in
-				}
-				return base.Run(
-					w.Install(spec.GetTestDeps(targetKey), atRoot("/tmp/rootfs")),
-					pg,
-					dalec.ProgressGroup("Install test dependencies"),
-				).AddMount("/tmp/rootfs", in)
-			}
-
-			if err := frontend.RunTests(ctx, client, spec, ref, withTestDeps, targetKey); err != nil {
-				return nil, nil, err
-			}
-
+			ref, err := runTests(ctx, client, w, spec, sOpt, rpmDir, targetKey)
 			return ref, img, err
 		})
 	}

frontend/azlinux/handle_rpm.go

@@ -47,11 +47,75 @@ func handleRPM(w worker) gwclient.BuildFunc {
 			if err != nil {
 				return nil, nil, err
 			}
+
+			if imgRef, err := runTests(ctx, client, w, spec, sOpt, st, targetKey, pg); err != nil {
+				// return the container ref in case of error so it can be used to debug
+				// the installed package state.
+				cfg, _ := resolveBaseConfig(ctx, w, client, platform, spec, targetKey)
+				return imgRef, cfg, err
+			}
+
 			return ref, &dalec.DockerImageSpec{}, nil
 		})
 	}
 }
 
+// runTests runs the package tests
+// The returned reference is the solved container state
+func runTests(ctx context.Context, client gwclient.Client, w worker, spec *dalec.Spec, sOpt dalec.SourceOpts, rpmDir llb.State, targetKey string, opts ...llb.ConstraintsOpt) (gwclient.Reference, error) {
+	withDeps, err := withTestDeps(w, spec, sOpt, targetKey)
+	if err != nil {
+		return nil, err
+	}
+
+	rpms, err := readRPMs(ctx, client, rpmDir)
+	if err != nil {
+		return nil, err
+	}
+
+	imgSt, err := specToContainerLLB(w, spec, targetKey, rpmDir, rpms, sOpt, opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	def, err := imgSt.Marshal(ctx, opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	res, err := client.Solve(ctx, gwclient.SolveRequest{Definition: def.ToPB()})
+	if err != nil {
+		return nil, err
+	}
+
+	ref, err := res.SingleRef()
+	if err != nil {
+		return nil, err
+	}
+
+	err = frontend.RunTests(ctx, client, spec, ref, withDeps, targetKey)
+	return ref, err
+}
+
+func withTestDeps(w worker, spec *dalec.Spec, sOpt dalec.SourceOpts, targetKey string, opts ...llb.ConstraintsOpt) (llb.StateOption, error) {
+	base, err := w.Base(sOpt, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return func(in llb.State) llb.State {
+		deps := spec.GetTestDeps(targetKey)
+		if len(deps) == 0 {
+			return in
+		}
+		return base.Run(
+			w.Install(spec.GetTestDeps(targetKey), atRoot("/tmp/rootfs")),
+			dalec.WithConstraints(opts...),
+			dalec.ProgressGroup("Install test dependencies"),
+		).AddMount("/tmp/rootfs", in)
+
+	}, nil
+}
+
 // Creates and installs an rpm meta-package that requires the passed in deps as runtime-dependencies
 func installBuildDepsPackage(target string, packageName string, w worker, deps map[string]dalec.PackageConstraints, installOpts ...installOpt) installFunc {
 	// depsOnly is a simple dalec spec that only includes build dependencies and their constraints

frontend/jammy/handle_container.go

@@ -3,12 +3,10 @@ package jammy
 import (
 	"context"
 	"encoding/json"
-	"io/fs"
 	"strings"
 
 	"github.com/Azure/dalec"
 	"github.com/Azure/dalec/frontend"
-	"github.com/Azure/dalec/frontend/pkg/bkfs"
 	"github.com/moby/buildkit/client/llb"
 	"github.com/moby/buildkit/client/llb/sourceresolver"
 	gwclient "github.com/moby/buildkit/frontend/gateway/client"
@@ -37,57 +35,12 @@ func handleContainer(ctx context.Context, client gwclient.Client) (*gwclient.Res
 			return nil, nil, err
 		}
 
-		worker, err := workerBase(sOpt, opt)
-		if err != nil {
-			return nil, nil, err
-		}
-
-		var includeTestRepo bool
-
-		workerFS, err := bkfs.FromState(ctx, &worker, client)
-		if err != nil {
-			return nil, nil, err
-		}
-
-		// Check if there there is a test repo in the worker image.
-		// We'll mount that into the target container while installing packages.
-		_, repoErr := fs.Stat(workerFS, testRepoPath[1:])
-		_, listErr := fs.Stat(workerFS, testRepoSourceListPath[1:])
-		if listErr == nil && repoErr == nil {
-			// This is a test and we need to include the repo from the worker image
-			// into target container.
-			includeTestRepo = true
-			frontend.Warn(ctx, client, worker, "Including test repo from worker image")
-		}
-
-		st := buildImageRootfs(worker, spec, sOpt, deb, targetKey, includeTestRepo, opt)
-
-		def, err := st.Marshal(ctx)
-		if err != nil {
-			return nil, nil, err
-		}
-
 		img, err := buildImageConfig(ctx, client, spec, platform, targetKey)
 		if err != nil {
 			return nil, nil, err
 		}
 
-		res, err := client.Solve(ctx, gwclient.SolveRequest{
-			Definition: def.ToPB(),
-		})
-		if err != nil {
-			return nil, nil, err
-		}
-
-		ref, err := res.SingleRef()
-		if err != nil {
-			return nil, nil, err
-		}
-
-		if err := frontend.RunTests(ctx, client, spec, ref, installTestDeps(spec, targetKey, opt), targetKey); err != nil {
-			return nil, nil, err
-		}
-
+		ref, err := runTests(ctx, client, spec, sOpt, deb, targetKey, opt)
 		return ref, img, err
 	})
 }

frontend/jammy/handle_deb.go

@@ -3,11 +3,13 @@ package jammy
 import (
 	"context"
 	"fmt"
+	"io/fs"
 	"strings"
 
 	"github.com/Azure/dalec"
 	"github.com/Azure/dalec/frontend"
 	"github.com/Azure/dalec/frontend/deb"
+	"github.com/Azure/dalec/frontend/pkg/bkfs"
 	"github.com/containerd/platforms"
 	"github.com/moby/buildkit/client/llb"
 	gwclient "github.com/moby/buildkit/frontend/gateway/client"
@@ -24,7 +26,8 @@ func handleDeb(ctx context.Context, client gwclient.Client) (*gwclient.Result, e
 			return nil, nil, err
 		}
 
-		st, err := buildDeb(ctx, client, spec, sOpt, targetKey, dalec.ProgressGroup("Building Jammy deb package: "+spec.Name))
+		opt := dalec.ProgressGroup("Building Jammy deb package: " + spec.Name)
+		st, err := buildDeb(ctx, client, spec, sOpt, targetKey, opt)
 		if err != nil {
 			return nil, nil, err
 		}
@@ -45,6 +48,11 @@ func handleDeb(ctx context.Context, client gwclient.Client) (*gwclient.Result, e
 		if err != nil {
 			return nil, nil, err
 		}
+
+		if ref, err := runTests(ctx, client, spec, sOpt, st, targetKey, opt); err != nil {
+			cfg, _ := buildImageConfig(ctx, client, spec, platform, targetKey)
+		}
+
 		if platform == nil {
 			p := platforms.DefaultSpec()
 			platform = &p
@@ -53,6 +61,53 @@ func handleDeb(ctx context.Context, client gwclient.Client) (*gwclient.Result, e
 	})
 }
 
+func runTests(ctx context.Context, client gwclient.Client, spec *dalec.Spec, sOpt dalec.SourceOpts, deb llb.State, targetKey string, opts ...llb.ConstraintsOpt) (gwclient.Reference, error) {
+	worker, err := workerBase(sOpt, opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	var includeTestRepo bool
+
+	workerFS, err := bkfs.FromState(ctx, &worker, client)
+	if err != nil {
+		return nil, err
+	}
+
+	// Check if there there is a test repo in the worker image.
+	// We'll mount that into the target container while installing packages.
+	_, repoErr := fs.Stat(workerFS, testRepoPath[1:])
+	_, listErr := fs.Stat(workerFS, testRepoSourceListPath[1:])
+	if listErr == nil && repoErr == nil {
+		// This is a test and we need to include the repo from the worker image
+		// into target container.
+		includeTestRepo = true
+		frontend.Warn(ctx, client, worker, "Including test repo from worker image")
+	}
+
+	st := buildImageRootfs(worker, spec, sOpt, deb, targetKey, includeTestRepo, opts...)
+
+	def, err := st.Marshal(ctx, opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	res, err := client.Solve(ctx, gwclient.SolveRequest{
+		Definition: def.ToPB(),
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	ref, err := res.SingleRef()
+	if err != nil {
+		return nil, err
+	}
+
+	err = frontend.RunTests(ctx, client, spec, ref, installTestDeps(spec, targetKey, opts...), targetKey)
+	return ref, err
+}
+
 func installPackages(ls ...string) llb.RunOption {
 	return dalec.RunOptFunc(func(ei *llb.ExecInfo) {
 		// This only runs apt-get update if the pkgcache is older than 10 minutes.

test/azlinux_test.go

@@ -1253,6 +1253,13 @@ Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/boot
 		ctx := startTestSpan(baseCtx, t)
 		testLinuxSymlinkArtifacts(ctx, t, testConfig)
 	})
+
+	t.Run("test package tests cause build to fail", func(t *testing.T) {
+		t.Parallel()
+		ctx := startTestSpan(baseCtx, t)
+		testLinuxPackageTestsFail(ctx, t, testConfig)
+	})
+
 }
 
 func testCustomLinuxWorker(ctx context.Context, t *testing.T, targetCfg targetConfig, workerCfg workerConfig) {
@@ -1685,3 +1692,83 @@ func testLinuxSymlinkArtifacts(ctx context.Context, t *testing.T, cfg testLinuxC
 		assert.NilError(t, err)
 	})
 }
+
+func testLinuxPackageTestsFail(ctx context.Context, t *testing.T, cfg testLinuxConfig) {
+	t.Run("negative test", func(t *testing.T) {
+		t.Parallel()
+		ctx := startTestSpan(ctx, t)
+
+		spec := &dalec.Spec{
+			Name:        "test-package-tests",
+			Version:     "0.0.1",
+			Revision:    "42",
+			Description: "Testing package tests",
+			License:     "MIT",
+			Tests: []*dalec.TestSpec{
+				{
+					Name: "Test that tests fail the build",
+					Files: map[string]dalec.FileCheckOutput{
+						"/non-existing-file": {},
+					},
+				},
+			},
+		}
+
+		testEnv.RunTest(ctx, t, func(ctx context.Context, client gwclient.Client) {
+			sr := newSolveRequest(withSpec(ctx, t, spec), withBuildTarget(cfg.Target.Package))
+			_, err := client.Solve(ctx, sr)
+			assert.ErrorContains(t, err, "lstat /non-existing-file: no such file or directory")
+
+			sr = newSolveRequest(withSpec(ctx, t, spec), withBuildTarget(cfg.Target.Container))
+			_, err = client.Solve(ctx, sr)
+			assert.ErrorContains(t, err, "lstat /non-existing-file: no such file or directory")
+		})
+	})
+
+	t.Run("positive test", func(t *testing.T) {
+		t.Parallel()
+		ctx := startTestSpan(ctx, t)
+
+		spec := &dalec.Spec{
+			Name:        "test-package-tests",
+			Version:     "0.0.1",
+			Revision:    "42",
+			Description: "Testing package tests",
+			License:     "MIT",
+			Sources: map[string]dalec.Source{
+				"test-file": {
+					Inline: &dalec.SourceInline{
+						File: &dalec.SourceInlineFile{
+							Contents: "hello world",
+						},
+					},
+				},
+			},
+			Artifacts: dalec.Artifacts{
+				DataDirs: map[string]dalec.ArtifactConfig{
+					"test-file": {},
+				},
+			},
+			Tests: []*dalec.TestSpec{
+				{
+					Name: "Test that tests fail the build",
+					Files: map[string]dalec.FileCheckOutput{
+						"/usr/share/test-file": {},
+					},
+				},
+			},
+		}
+
+		testEnv.RunTest(ctx, t, func(ctx context.Context, client gwclient.Client) {
+			sr := newSolveRequest(withSpec(ctx, t, spec), withBuildTarget(cfg.Target.Package))
+			res := solveT(ctx, t, client, sr)
+			_, err := res.SingleRef()
+			assert.NilError(t, err)
+
+			sr = newSolveRequest(withSpec(ctx, t, spec), withBuildTarget(cfg.Target.Container))
+			res = solveT(ctx, t, client, sr)
+			_, err = res.SingleRef()
+			assert.NilError(t, err)
+		})
+	})
+}