Add support for allowing network access during build

docs/spec.schema.json

@@ -18,6 +18,14 @@
 					},
 					"type": "object",
 					"description": "Env is the list of environment variables to set for all commands in this step group."
+				},
+				"network_mode": {
+					"type": "string",
+					"enum": [
+						"none",
+						"sandbox"
+					],
+					"description": "NetworkMode sets the network mode to use during the build phase.\nAccepted values: none, sandbox\nDefault: none"
 				}
 			},
 			"additionalProperties": false,

frontend/azlinux/handle_rpm.go

@@ -240,8 +240,11 @@ func specToRpmLLB(ctx context.Context, w worker, client gwclient.Client, spec *d
 	if err != nil {
 		return llb.Scratch(), err
 	}
+
 	specPath := filepath.Join("SPECS", spec.Name, spec.Name+".spec")
-	st := rpm.Build(br, base, specPath, opts...)
+
+	builder := base.With(dalec.SetBuildNetworkMode(spec))
+	st := rpm.Build(br, builder, specPath, opts...)
 
 	return frontend.MaybeSign(ctx, client, st, spec, targetKey, sOpt)
 }

frontend/deb/pkg.go

@@ -7,7 +7,6 @@ import (
 
 	"github.com/Azure/dalec"
 	"github.com/moby/buildkit/client/llb"
-	"github.com/moby/buildkit/solver/pb"
 	"github.com/pkg/errors"
 )
 
@@ -103,7 +102,6 @@ func SourcePackage(sOpt dalec.SourceOpts, worker llb.State, spec *dalec.Spec, ta
 		llb.AddMount("/work/pkg/debian", dr, llb.SourcePath("debian")), // This cannot be readonly because the debian directory gets modified by dpkg-buildpackage
 		llb.AddMount("/work/pkg/debian/patches", patches, llb.Readonly),
 		llb.AddEnv("DH_VERBOSE", "1"),
-		llb.Network(pb.NetMode_NONE),
 		mountSources(sources, "/work/pkg", sanitizeSourceKey),
 		dalec.RunOptFunc(func(ei *llb.ExecInfo) {
 			// Mount all the tar+gz'd sources into the build which will get picked p by debbuild
@@ -118,11 +116,7 @@ func SourcePackage(sOpt dalec.SourceOpts, worker llb.State, spec *dalec.Spec, ta
 	return work.AddMount("/tmp/out", llb.Scratch()), nil
 }
 
-func BuildDeb(worker llb.State, spec *dalec.Spec, sOpt dalec.SourceOpts, targetKey, distroVersionID string, opts ...llb.ConstraintsOpt) (llb.State, error) {
-	srcPkg, err := SourcePackage(sOpt, worker, spec, targetKey, distroVersionID)
-	if err != nil {
-		return llb.Scratch(), errors.Wrap(err, "error creating debian source package")
-	}
+func BuildDeb(worker llb.State, spec *dalec.Spec, srcPkg llb.State, distroVersionID string, opts ...llb.ConstraintsOpt) (llb.State, error) {
 
 	dirName := filepath.Join("/work", spec.Name+"_"+spec.Version+"-"+spec.Revision)
 	st := worker.
@@ -131,7 +125,6 @@ func BuildDeb(worker llb.State, spec *dalec.Spec, sOpt dalec.SourceOpts, targetK
 			llb.Dir(dirName),
 			llb.AddEnv("DH_VERBOSE", "1"),
 			llb.AddMount(dirName, srcPkg),
-			llb.Network(pb.NetMode_NONE),
 			dalec.WithConstraints(opts...),
 		).AddMount("/tmp/out", llb.Scratch())
 

frontend/jammy/handle_deb.go

@@ -216,8 +216,14 @@ func buildDeb(ctx context.Context, client gwclient.Client, spec *dalec.Spec, sOp
 		return llb.Scratch(), errors.Wrap(err, "error creating deb for build dependencies")
 	}
 
-	worker = worker.With(installBuildDeps)
-	st, err := deb.BuildDeb(worker, spec, sOpt, targetKey, versionID, opts...)
+	builder := worker.With(installBuildDeps)
+	srcPkg, err := deb.SourcePackage(sOpt, builder, spec, targetKey, versionID, opts...)
+	if err != nil {
+		return llb.Scratch(), err
+	}
+
+	builder = builder.With(dalec.SetBuildNetworkMode(spec))
+	st, err := deb.BuildDeb(builder, spec, srcPkg, versionID, opts...)
 	if err != nil {
 		return llb.Scratch(), err
 	}
@@ -301,7 +307,13 @@ func buildDepends(worker llb.State, sOpt dalec.SourceOpts, spec *dalec.Spec, tar
 
 	pg := dalec.ProgressGroup("Install build dependencies")
 	opts = append(opts, pg)
-	pkg, err := deb.BuildDeb(worker, depsSpec, sOpt, targetKey, "", append(opts, dalec.ProgressGroup("Create intermediate deb for build dependencies"))...)
+
+	srcPkg, err := deb.SourcePackage(sOpt, worker, depsSpec, targetKey, "", opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	pkg, err := deb.BuildDeb(worker, depsSpec, srcPkg, "", append(opts, dalec.ProgressGroup("Create intermediate deb for build dependnencies"))...)
 	if err != nil {
 		return nil, errors.Wrap(err, "error creating intermediate package for installing build dependencies")
 	}

frontend/rpm/rpmbuild.go

@@ -33,7 +33,6 @@ func Build(topDir, workerImg llb.State, specPath string, opts ...llb.Constraints
 		llb.AddMount("/build/top", topDir),
 		llb.AddMount("/build/tmp", llb.Scratch(), llb.Tmpfs()),
 		llb.Dir("/build/top"),
-		llb.Network(llb.NetModeNone),
 		dalec.WithConstraints(opts...),
 	).
 		AddMount("/build/out", llb.Scratch())

frontend/windows/handle_zip.go

@@ -143,12 +143,12 @@ func buildBinaries(ctx context.Context, spec *dalec.Spec, worker llb.State, clie
 	binaries := maps.Keys(spec.Artifacts.Binaries)
 	script := generateInvocationScript(binaries)
 
-	st := worker.Run(
+	builder := worker.With(dalec.SetBuildNetworkMode(spec))
+	st := builder.Run(
 		dalec.ShArgs(script.String()),
 		llb.Dir("/build"),
 		withSourcesMounted("/build", patched, spec.Sources),
 		llb.AddMount("/tmp/scripts", buildScript),
-		llb.Network(llb.NetModeNone),
 	).AddMount(outputDir, llb.Scratch())
 
 	return frontend.MaybeSign(ctx, client, st, spec, targetKey, sOpt)

helpers.go

@@ -2,6 +2,7 @@ package dalec
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"fmt"
 	"path"
@@ -555,3 +556,24 @@ func GetRepoKeys(worker llb.State, configs []PackageRepositoryConfig, cfg *RepoP
 
 	return WithRunOptions(keys...), names, nil
 }
+
+const (
+	netModeNone    = "none"
+	netModeSandbox = "sandbox"
+)
+
+// SetBuildNetworkMode returns an [llb.StateOption] that determines which
+func SetBuildNetworkMode(spec *Spec) llb.StateOption {
+	switch spec.Build.NetworkMode {
+	case "", netModeNone:
+		return llb.Network(llb.NetModeNone)
+	case netModeSandbox:
+		return llb.Network(llb.NetModeSandbox)
+	default:
+		return func(in llb.State) llb.State {
+			return in.Async(func(context.Context, llb.State, *llb.Constraints) (llb.State, error) {
+				return in, fmt.Errorf("invalid build network mode %q", spec.Build.NetworkMode)
+			})
+		}
+	}
+}

load.go

@@ -316,6 +316,14 @@ func (s *Spec) SubstituteArgs(env map[string]string) error {
 		s.Build.Env[k] = updated
 	}
 
+	if s.Build.NetworkMode != "" {
+		updated, err := expandArgs(lex, s.Build.NetworkMode, args)
+		if err != nil {
+			appendErr(fmt.Errorf("error performing shell expansion on build network mode: %s: %w", s.Build.NetworkMode, err))
+		}
+		s.Build.NetworkMode = updated
+	}
+
 	for i, step := range s.Build.Steps {
 		bs := &step
 		if err := bs.processBuildArgs(lex, args, i); err != nil {
@@ -448,18 +456,20 @@ func (s *Spec) FillDefaults() {
 }
 
 func (s Spec) Validate() error {
+	var outErr error
+
 	for name, src := range s.Sources {
 		if strings.ContainsRune(name, os.PathSeparator) {
-			return &InvalidSourceError{Name: name, Err: sourceNamePathSeparatorError}
+			outErr = goerrors.Join(outErr, &InvalidSourceError{Name: name, Err: sourceNamePathSeparatorError})
 		}
 		if err := src.validate(); err != nil {
-			return &InvalidSourceError{Name: name, Err: fmt.Errorf("error validating source ref %q: %w", name, err)}
+			outErr = goerrors.Join(&InvalidSourceError{Name: name, Err: fmt.Errorf("error validating source ref %q: %w", name, err)})
 		}
 
 		if src.DockerImage != nil && src.DockerImage.Cmd != nil {
 			for p, cfg := range src.DockerImage.Cmd.CacheDirs {
 				if _, err := sharingMode(cfg.Mode); err != nil {
-					return &InvalidSourceError{Name: name, Err: errors.Wrapf(err, "invalid sharing mode for source %q with cache mount at path %q", name, p)}
+					outErr = goerrors.Join(&InvalidSourceError{Name: name, Err: errors.Wrapf(err, "invalid sharing mode for source %q with cache mount at path %q", name, p)})
 				}
 			}
 		}
@@ -468,30 +478,32 @@ func (s Spec) Validate() error {
 	for _, t := range s.Tests {
 		for p, cfg := range t.CacheDirs {
 			if _, err := sharingMode(cfg.Mode); err != nil {
-				return errors.Wrapf(err, "invalid sharing mode for test %q with cache mount at path %q", t.Name, p)
+				outErr = goerrors.Join(errors.Wrapf(err, "invalid sharing mode for test %q with cache mount at path %q", t.Name, p))
 			}
 		}
 	}
 
-	var patchErr error
 	for src, patches := range s.Patches {
 		for _, patch := range patches {
 			patchSrc, ok := s.Sources[patch.Source]
 			if !ok {
-				patchErr = goerrors.Join(patchErr, &InvalidPatchError{Source: src, PatchSpec: &patch, Err: errMissingSource})
+				outErr = goerrors.Join(outErr, &InvalidPatchError{Source: src, PatchSpec: &patch, Err: errMissingSource})
 				continue
 			}
 
 			if err := validatePatch(patch, patchSrc); err != nil {
-				patchErr = goerrors.Join(patchErr, &InvalidPatchError{Source: src, PatchSpec: &patch, Err: err})
+				outErr = goerrors.Join(outErr, &InvalidPatchError{Source: src, PatchSpec: &patch, Err: err})
 			}
 		}
 	}
-	if patchErr != nil {
-		return patchErr
+
+	switch s.Build.NetworkMode {
+	case "", netModeNone, netModeSandbox:
+	default:
+		outErr = goerrors.Join(outErr, fmt.Errorf("invalid network mode: %q: valid values %s", s.Build.NetworkMode, []string{netModeNone, netModeSandbox}))
 	}
 
-	return nil
+	return outErr
 }
 
 func validatePatch(patch PatchSpec, patchSrc Source) error {

spec.go

@@ -401,6 +401,11 @@ type ArtifactBuild struct {
 	Steps []BuildStep `yaml:"steps" json:"steps" jsonschema:"required"`
 	// Env is the list of environment variables to set for all commands in this step group.
 	Env map[string]string `yaml:"env,omitempty" json:"env,omitempty"`
+
+	// NetworkMode sets the network mode to use during the build phase.
+	// Accepted values: none, sandbox
+	// Default: none
+	NetworkMode string `yaml:"network_mode,omitempty" json:"network_mode,omitempty" jsonschema:"enum=none,enum=sandbox"`
 }
 
 // BuildStep is used to execute a command to build the artifact(s).

test/azlinux_test.go

@@ -228,41 +228,6 @@ func testLinuxDistro(ctx context.Context, t *testing.T, testConfig testLinuxConf
 		})
 	})
 
-	t.Run("should not have internet access during build", func(t *testing.T) {
-		t.Parallel()
-		spec := dalec.Spec{
-			Name:        "test-no-internet-access",
-			Version:     "0.0.1",
-			Revision:    "1",
-			License:     "MIT",
-			Website:     "https://github.com/azure/dalec",
-			Vendor:      "Dalec",
-			Packager:    "Dalec",
-			Description: "Should not have internet access during build",
-			Dependencies: &dalec.PackageDependencies{
-				Build: map[string]dalec.PackageConstraints{"curl": {}},
-			},
-			Build: dalec.ArtifactBuild{
-				Steps: []dalec.BuildStep{
-					{
-						Command: fmt.Sprintf("curl --head -ksSf %s > /dev/null", externalTestHost),
-					},
-				},
-			},
-		}
-
-		testEnv.RunTest(ctx, t, func(ctx context.Context, gwc gwclient.Client) {
-			sr := newSolveRequest(withSpec(ctx, t, &spec), withBuildTarget(testConfig.Target.Container))
-			sr.Evaluate = true
-
-			_, err := gwc.Solve(ctx, sr)
-			var xErr *moby_buildkit_v1_frontend.ExitError
-			if !errors.As(err, &xErr) {
-				t.Fatalf("expected exit error, got %T: %v", errors.Unwrap(err), err)
-			}
-		})
-	})
-
 	t.Run("container", func(t *testing.T) {
 		const src2Patch3File = "patch3"
 		src2Patch3Content := []byte(`
@@ -1409,6 +1374,12 @@ Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/boot
 		ctx := startTestSpan(baseCtx, t)
 		testLinuxPackageTestsFail(ctx, t, testConfig)
 	})
+
+	t.Run("build network mode", func(t *testing.T) {
+		t.Parallel()
+		ctx := startTestSpan(baseCtx, t)
+		testBuildNetworkMode(ctx, t, testConfig.Target)
+	})
 }
 
 func testCustomLinuxWorker(ctx context.Context, t *testing.T, targetCfg targetConfig, workerCfg workerConfig) {

test/network_test.go

@@ -0,0 +1,83 @@
+package test
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"testing"
+
+	"github.com/Azure/dalec"
+	gwclient "github.com/moby/buildkit/frontend/gateway/client"
+	moby_buildkit_v1_frontend "github.com/moby/buildkit/frontend/gateway/pb"
+	"gotest.tools/v3/assert"
+)
+
+func testBuildNetworkMode(ctx context.Context, t *testing.T, cfg targetConfig) {
+	type testCase struct {
+		mode            string
+		canHazInternetz bool // :)
+	}
+
+	cases := []testCase{
+		{mode: "", canHazInternetz: false},
+		{mode: "none", canHazInternetz: false},
+		{mode: "sandbox", canHazInternetz: true},
+	}
+
+	for _, tc := range cases {
+		name := "mode=" + tc.mode
+		if tc.mode == "" {
+			name += "<unset>"
+		}
+
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+			ctx := startTestSpan(ctx, t)
+
+			spec := dalec.Spec{
+				Name:        "test-build-network-mode",
+				Version:     "0.0.1",
+				Revision:    "1",
+				License:     "MIT",
+				Website:     "https://github.com/azure/dalec",
+				Vendor:      "Dalec",
+				Packager:    "Dalec",
+				Description: "Should not have internet access during build",
+				Dependencies: &dalec.PackageDependencies{
+					Build: map[string]dalec.PackageConstraints{"curl": {}},
+				},
+				Build: dalec.ArtifactBuild{
+					NetworkMode: tc.mode,
+					Steps: []dalec.BuildStep{
+						{
+							Command: fmt.Sprintf("curl --head -ksSf %s > /dev/null", externalTestHost),
+						},
+						{
+							Command: "touch foo",
+						},
+					},
+				},
+				Artifacts: dalec.Artifacts{
+					// This is here so the windows can use this test
+					// Windows needs to have a non-empty output to suceeed.
+					Binaries: map[string]dalec.ArtifactConfig{"foo": {}},
+				},
+			}
+
+			testEnv.RunTest(ctx, t, func(ctx context.Context, gwc gwclient.Client) {
+				sr := newSolveRequest(withSpec(ctx, t, &spec), withBuildTarget(cfg.Package))
+
+				_, err := gwc.Solve(ctx, sr)
+				if tc.canHazInternetz {
+					assert.NilError(t, err)
+					return
+				}
+
+				var xErr *moby_buildkit_v1_frontend.ExitError
+				if !errors.As(err, &xErr) {
+					t.Fatalf("expected exit error, got %T: %v", errors.Unwrap(err), err)
+				}
+			})
+		})
+	}
+}