add multi module go generation support

docs/spec.schema.json

@@ -395,7 +395,15 @@
 			"description": "Frontend encapsulates the configuration for a frontend to forward a build target to."
 		},
 		"GeneratorGomod": {
-			"properties": {},
+			"properties": {
+				"paths": {
+					"items": {
+						"type": "string"
+					},
+					"type": "array",
+					"description": "Paths is the list of paths to run the generator on. Used to generate multi-module in a single source."
+				}
+			},
 			"additionalProperties": false,
 			"type": "object",
 			"description": "GeneratorGomod is used to generate a go module cache from go module sources"

generator_gomod.go

@@ -1,6 +1,8 @@
 package dalec
 
 import (
+	"path/filepath"
+
 	"github.com/moby/buildkit/client/llb"
 	"github.com/pkg/errors"
 )
@@ -30,20 +32,27 @@ func (s *Spec) HasGomods() bool {
 
 func withGomod(g *SourceGenerator, srcSt, worker llb.State, opts ...llb.ConstraintsOpt) func(llb.State) llb.State {
 	return func(in llb.State) llb.State {
-		const workDir = "/work/src"
-		var srcMount llb.RunOption
-		if g.Subpath != "" {
-			srcMount = llb.AddMount(workDir, srcSt, llb.SourcePath(g.Subpath))
-		} else {
-			srcMount = llb.AddMount(workDir, srcSt)
+		workDir := "/work/src"
+		joinedWorkDir := filepath.Join(workDir, g.Subpath)
+		srcMount := llb.AddMount(workDir, srcSt)
+
+		paths := g.Gomod.Paths
+		if g.Gomod.Paths == nil {
+			paths = []string{"."}
+		}
+		states := make([]llb.State, 0, len(paths))
+
+		for _, path := range paths {
+			currentState := worker.Run(
+				ShArgs("go mod download"),
+				llb.AddEnv("GOMODCACHE", gomodCacheDir),
+				llb.Dir(filepath.Join(joinedWorkDir, path)),
+				srcMount,
+				WithConstraints(opts...),
+			).AddMount(gomodCacheDir, in)
+			states = append(states, currentState)
 		}
-		return worker.Run(
-			ShArgs("go mod download"),
-			llb.AddEnv("GOMODCACHE", gomodCacheDir),
-			llb.Dir(workDir),
-			srcMount,
-			WithConstraints(opts...),
-		).AddMount(gomodCacheDir, in)
+		return MergeAtPath(in, states, "/")
 	}
 }
 

spec.go

@@ -307,6 +307,8 @@ type Source struct {
 
 // GeneratorGomod is used to generate a go module cache from go module sources
 type GeneratorGomod struct {
+	// Paths is the list of paths to run the generator on. Used to generate multi-module in a single source.
+	Paths []string `yaml:"paths,omitempty" json:"paths,omitempty"`
 }
 
 // SourceGenerator holds the configuration for a source generator.

test/source_test.go

@@ -247,6 +247,48 @@ github.com/cpuguy83/tar2go v0.3.1 h1:DMWlaIyoh9FBWR4hyfZSOEDA7z8rmCiGF1IJIzlTlR8
 github.com/cpuguy83/tar2go v0.3.1/go.mod h1:2Ys2/Hu+iPHQRa4DjIVJ7UAaKnDhAhNACeK3A0Rr5rM=
 `
 
+const alternativeGomodFixtureMain = `package main
+
+import (
+	"fmt"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func main() {
+	msg := "This is a dummy test from module2"
+	fmt.Println(msg)
+	assert.True(nil, true, msg)
+}
+`
+
+const alternativeGomodFixtureMod = `module example.com/module2
+
+go 1.20
+
+require github.com/stretchr/testify v1.7.0
+
+require (
+	github.com/davecgh/go-spew v1.1.0 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
+)
+`
+
+const alternativeGomodFixtureSum = `
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+`
+
 func TestSourceWithGomod(t *testing.T) {
 	t.Parallel()
 
@@ -297,6 +339,7 @@ index ea874f5..ba38f84 100644
 	}
 
 	const srcName = "src1"
+
 	baseSpec := func() *dalec.Spec {
 		return &dalec.Spec{
 			Sources: map[string]dalec.Source{
@@ -320,12 +363,42 @@ index ea874f5..ba38f84 100644
 		}
 	}
 
-	t.Run("no patch", func(t *testing.T) {
-		t.Parallel()
-		testEnv.RunTest(baseCtx, t, func(ctx context.Context, gwc gwclient.Client) {
-			checkModule(ctx, gwc, "github.com/cpuguy83/tar2go@v0.3.1", baseSpec())
+	tests := map[string]struct {
+		module       string
+		sourceInline *dalec.SourceInline
+	}{
+		"no patch": {
+			module: "github.com/cpuguy83/tar2go@v0.3.1",
+		},
+		"alternative go fixture": {
+			module: "github.com/stretchr/testify@v1.7.0",
+			sourceInline: &dalec.SourceInline{
+				Dir: &dalec.SourceInlineDir{
+					Files: map[string]*dalec.SourceInlineFile{
+						"main.go": {Contents: alternativeGomodFixtureMain},
+						"go.mod":  {Contents: alternativeGomodFixtureMod},
+						"go.sum":  {Contents: alternativeGomodFixtureSum},
+					},
+				},
+			},
+		},
+	}
+
+	for name, tt := range tests {
+
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+			spec := baseSpec()
+			if tt.sourceInline != nil {
+				source := spec.Sources[srcName]
+				source.Inline = tt.sourceInline
+				spec.Sources[srcName] = source
+			}
+			testEnv.RunTest(baseCtx, t, func(ctx context.Context, gwc gwclient.Client) {
+				checkModule(ctx, gwc, tt.module, spec)
+			})
 		})
-	})
+	}
 
 	t.Run("with patch", func(t *testing.T) {
 		t.Run("file", func(t *testing.T) {
@@ -375,6 +448,77 @@ index ea874f5..ba38f84 100644
 	})
 }
 
+func TestMultiGoModuleGen(t *testing.T) {
+	t.Parallel()
+	/*
+		dir/
+			module1/
+				go.mod
+				go.sum
+				main.go
+			module2/
+				go.mod
+				go.sum
+				main.go
+	*/
+	contextSt := llb.Scratch().File(llb.Mkdir("/dir", 0644)).
+		File(llb.Mkdir("/dir/module1", 0644)).
+		File(llb.Mkfile("/dir/module1/go.mod", 0644, []byte(alternativeGomodFixtureMod))).
+		File(llb.Mkfile("/dir/module1/go.sum", 0644, []byte(alternativeGomodFixtureSum))).
+		File(llb.Mkfile("/dir/module1/main.go", 0644, []byte(alternativeGomodFixtureMain))).
+		File(llb.Mkdir("/dir/module2", 0644)).
+		File(llb.Mkfile("/dir/module2/go.mod", 0644, []byte(gomodFixtureMod))).
+		File(llb.Mkfile("/dir/module2/go.sum", 0644, []byte(gomodFixtureSum))).
+		File(llb.Mkfile("/dir/module2/main.go", 0644, []byte(gomodFixtureMain)))
+
+	const contextName = "multi-module"
+	spec := &dalec.Spec{
+		Name: "test-dalec-context-source",
+		Sources: map[string]dalec.Source{
+			"src": {
+				Context: &dalec.SourceContext{Name: contextName},
+				Generate: []*dalec.SourceGenerator{
+					{
+						Gomod: &dalec.GeneratorGomod{
+							Paths: []string{"./dir/module1", "./dir/module2"},
+						},
+					},
+				},
+			},
+		},
+		Dependencies: &dalec.PackageDependencies{
+			Build: map[string]dalec.PackageConstraints{
+				"golang": {
+					Version: []string{},
+				},
+			},
+		},
+	}
+
+	runTest(t, func(ctx context.Context, gwc gwclient.Client) {
+		req := newSolveRequest(withSpec(ctx, t, spec), withBuildContext(ctx, t, contextName, contextSt), withBuildTarget("debug/gomods"))
+		res := solveT(ctx, t, gwc, req)
+		ref, err := res.SingleRef()
+		if err != nil {
+			t.Fatal(err)
+		}
+		deps := []string{"github.com/cpuguy83/tar2go@v0.3.1", "github.com/stretchr/testify@v1.7.0"}
+		for _, dep := range deps {
+			stat, err := ref.StatFile(ctx, gwclient.StatRequest{
+				Path: dep,
+			})
+
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if !fs.FileMode(stat.Mode).IsDir() {
+				t.Fatal("expected directory")
+			}
+		}
+	})
+}
+
 func TestSourceContext(t *testing.T) {
 	t.Parallel()