llbsolver: fileop implementation

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>

cache/refs.go

@@ -311,7 +311,7 @@ func (sr *mutableRef) updateLastUsed() bool {
 
 func (sr *mutableRef) commit(ctx context.Context) (ImmutableRef, error) {
 	if !sr.mutable || len(sr.refs) == 0 {
-		return nil, errors.Wrapf(errInvalid, "invalid mutable ref")
+		return nil, errors.Wrapf(errInvalid, "invalid mutable ref %p", sr)
 	}
 
 	id := identity.NewID()

client/client_test.go

@@ -51,6 +51,8 @@ func (nopWriteCloser) Close() error { return nil }
 func TestClientIntegration(t *testing.T) {
 	integration.Run(t, []integration.Test{
 		testRelativeWorkDir,
+		testFileOpMkdirMkfile,
+		testFileOpCopyRm,
 		testCallDiskUsage,
 		testBuildMultiMount,
 		testBuildHTTPSource,
@@ -633,6 +635,103 @@ func testRelativeWorkDir(t *testing.T, sb integration.Sandbox) {
 	require.Equal(t, []byte("/test1/test2\n"), dt)
 }
 
+func testFileOpMkdirMkfile(t *testing.T, sb integration.Sandbox) {
+	requiresLinux(t)
+	c, err := New(context.TODO(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	st := llb.Scratch().
+		File(llb.Mkdir("/foo", 0700).Mkfile("bar", 0600, []byte("contents")))
+
+	def, err := st.Marshal()
+	require.NoError(t, err)
+
+	destDir, err := ioutil.TempDir("", "buildkit")
+	require.NoError(t, err)
+	defer os.RemoveAll(destDir)
+
+	_, err = c.Solve(context.TODO(), def, SolveOpt{
+		Exporter:          ExporterLocal,
+		ExporterOutputDir: destDir,
+	}, nil)
+	require.NoError(t, err)
+
+	fi, err := os.Stat(filepath.Join(destDir, "foo"))
+	require.NoError(t, err)
+	require.Equal(t, true, fi.IsDir())
+
+	dt, err := ioutil.ReadFile(filepath.Join(destDir, "bar"))
+	require.NoError(t, err)
+	require.Equal(t, []byte("contents"), dt)
+}
+
+func testFileOpCopyRm(t *testing.T, sb integration.Sandbox) {
+	requiresLinux(t)
+	c, err := New(context.TODO(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	dir, err := tmpdir(
+		fstest.CreateFile("myfile", []byte("data0"), 0600),
+		fstest.CreateDir("sub", 0700),
+		fstest.CreateFile("sub/foo", []byte("foo0"), 0600),
+		fstest.CreateFile("sub/bar", []byte("bar0"), 0600),
+	)
+	require.NoError(t, err)
+	defer os.RemoveAll(dir)
+
+	dir2, err := tmpdir(
+		fstest.CreateFile("file2", []byte("file2"), 0600),
+	)
+	require.NoError(t, err)
+	defer os.RemoveAll(dir)
+
+	st := llb.Scratch().
+		File(
+			llb.Copy(llb.Local("mylocal"), "myfile", "myfile2").
+				Copy(llb.Local("mylocal"), "sub", "out").
+				Rm("out/foo").
+				Copy(llb.Local("mylocal2"), "file2", "/"))
+
+	def, err := st.Marshal()
+	require.NoError(t, err)
+
+	destDir, err := ioutil.TempDir("", "buildkit")
+	require.NoError(t, err)
+	defer os.RemoveAll(destDir)
+
+	_, err = c.Solve(context.TODO(), def, SolveOpt{
+		Exporter:          ExporterLocal,
+		ExporterOutputDir: destDir,
+		LocalDirs: map[string]string{
+			"mylocal":  dir,
+			"mylocal2": dir2,
+		},
+	}, nil)
+	require.NoError(t, err)
+
+	dt, err := ioutil.ReadFile(filepath.Join(destDir, "myfile2"))
+	require.NoError(t, err)
+	require.Equal(t, []byte("data0"), dt)
+
+	fi, err := os.Stat(filepath.Join(destDir, "out"))
+	require.NoError(t, err)
+	require.Equal(t, true, fi.IsDir())
+
+	dt, err = ioutil.ReadFile(filepath.Join(destDir, "out/bar"))
+	require.NoError(t, err)
+	require.Equal(t, []byte("bar0"), dt)
+
+	_, err = os.Stat(filepath.Join(destDir, "out/foo"))
+	require.Equal(t, true, os.IsNotExist(err))
+
+	dt, err = ioutil.ReadFile(filepath.Join(destDir, "file2"))
+	require.NoError(t, err)
+	require.Equal(t, []byte("file2"), dt)
+
+}
+
 func testCallDiskUsage(t *testing.T, sb integration.Sandbox) {
 	c, err := New(context.TODO(), sb.Address())
 	require.NoError(t, err)

client/llb/fileop.go

@@ -634,8 +634,8 @@ func (f *FileOp) Marshal(c *Constraints) (digest.Digest, []byte, *pb.OpMetadata,
 		}
 
 		pfo.Actions = append(pfo.Actions, &pb.FileAction{
-			Input:          getIndex(st.input, len(state.actions), st.inputRelative),
-			SecondaryInput: getIndex(st.input2, len(state.actions), st.input2Relative),
+			Input:          getIndex(st.input, len(state.inputs), st.inputRelative),
+			SecondaryInput: getIndex(st.input2, len(state.inputs), st.input2Relative),
 			Output:         output,
 			Action:         st.action.toProtoAction(parent, st.base),
 		})

client/llb/fileop_test.go

@@ -76,7 +76,7 @@ func TestFileMkdirChain(t *testing.T) {
 	require.Nil(t, mkdir.Owner)
 
 	action = f.Actions[1]
-	require.Equal(t, 3, int(action.Input))
+	require.Equal(t, 1, int(action.Input))
 	require.Equal(t, -1, int(action.SecondaryInput))
 	require.Equal(t, -1, int(action.Output))
 	mkdir = action.Action.(*pb.FileAction_Mkdir).Mkdir
@@ -86,7 +86,7 @@ func TestFileMkdirChain(t *testing.T) {
 	require.Nil(t, mkdir.Owner)
 
 	action = f.Actions[2]
-	require.Equal(t, 4, int(action.Input))
+	require.Equal(t, 2, int(action.Input))
 	require.Equal(t, -1, int(action.SecondaryInput))
 	require.Equal(t, 0, int(action.Output))
 	mkdir = action.Action.(*pb.FileAction_Mkdir).Mkdir
@@ -96,6 +96,52 @@ func TestFileMkdirChain(t *testing.T) {
 	require.Nil(t, mkdir.Owner)
 }
 
+func TestFileMkdirMkfile(t *testing.T) {
+	t.Parallel()
+
+	st := Scratch().File(Mkdir("/foo", 0700).Mkfile("bar", 0700, []byte("data")))
+	def, err := st.Marshal()
+
+	require.NoError(t, err)
+
+	m, arr := parseDef(t, def.Def)
+	require.Equal(t, 2, len(arr))
+
+	dgst, idx := last(t, arr)
+	require.Equal(t, 0, idx)
+	require.Equal(t, m[dgst], arr[0])
+
+	f := arr[0].Op.(*pb.Op_File).File
+	require.Equal(t, len(arr[1].Inputs), 1)
+	require.Equal(t, m[arr[1].Inputs[0].Digest], arr[0])
+	require.Equal(t, 0, int(arr[1].Inputs[0].Index))
+
+	require.Equal(t, 2, len(f.Actions))
+
+	action := f.Actions[0]
+	require.Equal(t, -1, int(action.Input))
+	require.Equal(t, -1, int(action.SecondaryInput))
+	require.Equal(t, -1, int(action.Output))
+
+	mkdir := action.Action.(*pb.FileAction_Mkdir).Mkdir
+
+	require.Equal(t, "/foo", mkdir.Path)
+	require.Equal(t, 0700, int(mkdir.Mode))
+	require.Equal(t, int64(-1), mkdir.Timestamp)
+
+	action = f.Actions[1]
+	require.Equal(t, 0, int(action.Input))
+	require.Equal(t, -1, int(action.SecondaryInput))
+	require.Equal(t, 0, int(action.Output))
+
+	mkfile := action.Action.(*pb.FileAction_Mkfile).Mkfile
+
+	require.Equal(t, "/bar", mkfile.Path)
+	require.Equal(t, 0700, int(mkfile.Mode))
+	require.Equal(t, "data", string(mkfile.Data))
+	require.Equal(t, int64(-1), mkfile.Timestamp)
+}
+
 func TestFileMkfile(t *testing.T) {
 	t.Parallel()
 
@@ -202,7 +248,7 @@ func TestFileSimpleChains(t *testing.T) {
 	require.Equal(t, "/tmp/sub/foo", rm.Path)
 
 	action = f.Actions[1]
-	require.Equal(t, 2, int(action.Input))
+	require.Equal(t, 1, int(action.Input))
 	require.Equal(t, -1, int(action.SecondaryInput))
 	require.Equal(t, 0, int(action.Output))
 
@@ -224,15 +270,15 @@ func TestFileSimpleChains(t *testing.T) {
 	require.Equal(t, "/tmp/foo/bar", mkdir.Path)
 
 	action = f.Actions[1]
-	require.Equal(t, 3, int(action.Input))
+	require.Equal(t, 1, int(action.Input))
 	require.Equal(t, -1, int(action.SecondaryInput))
 	require.Equal(t, -1, int(action.Output))
 
 	rm = action.Action.(*pb.FileAction_Rm).Rm
 	require.Equal(t, "/tmp/abc", rm.Path)
 
 	action = f.Actions[2]
-	require.Equal(t, 4, int(action.Input))
+	require.Equal(t, 2, int(action.Input))
 	require.Equal(t, -1, int(action.SecondaryInput))
 	require.Equal(t, 0, int(action.Output))
 
@@ -314,7 +360,7 @@ func TestFileCopyFromAction(t *testing.T) {
 	require.Equal(t, 0700, int(mkdir.Mode))
 
 	action = f.Actions[1]
-	require.Equal(t, 3, int(action.Input))
+	require.Equal(t, 1, int(action.Input))
 	require.Equal(t, -1, int(action.SecondaryInput))
 	require.Equal(t, -1, int(action.Output))
 
@@ -326,7 +372,7 @@ func TestFileCopyFromAction(t *testing.T) {
 
 	action = f.Actions[2]
 	require.Equal(t, 0, int(action.Input))
-	require.Equal(t, 4, int(action.SecondaryInput))
+	require.Equal(t, 2, int(action.SecondaryInput))
 	require.Equal(t, 0, int(action.Output))
 
 	copy := action.Action.(*pb.FileAction_Copy).Copy
@@ -420,7 +466,7 @@ func TestFilePipeline(t *testing.T) {
 	require.Equal(t, 0700, int(mkdir.Mode))
 
 	action = f.Actions[1]
-	require.Equal(t, 4, int(action.Input))
+	require.Equal(t, 2, int(action.Input))
 	require.Equal(t, -1, int(action.SecondaryInput))
 	require.Equal(t, -1, int(action.Output))
 
@@ -432,7 +478,7 @@ func TestFilePipeline(t *testing.T) {
 
 	action = f.Actions[2]
 	require.Equal(t, 0, int(action.Input))
-	require.Equal(t, 5, int(action.SecondaryInput))
+	require.Equal(t, 3, int(action.SecondaryInput))
 	require.Equal(t, -1, int(action.Output))
 	require.Equal(t, arr[4].Inputs[1].Digest, op.Inputs[0].Digest)
 
@@ -442,7 +488,7 @@ func TestFilePipeline(t *testing.T) {
 	require.Equal(t, "/out/baz", copy.Dest)
 
 	action = f.Actions[3]
-	require.Equal(t, 6, int(action.Input))
+	require.Equal(t, 4, int(action.Input))
 	require.Equal(t, -1, int(action.SecondaryInput))
 	require.Equal(t, 0, int(action.Output))
 

solver/llbsolver/file/backend.go

@@ -118,10 +118,10 @@ func docopy(ctx context.Context, src, dest string, action pb.FileActionCopy) err
 	return nil
 }
 
-type FileBackend struct {
+type Backend struct {
 }
 
-func (fb *FileBackend) Mkdir(ctx context.Context, m fileoptypes.Mount, action pb.FileActionMkDir) error {
+func (fb *Backend) Mkdir(ctx context.Context, m fileoptypes.Mount, action pb.FileActionMkDir) error {
 	mnt, ok := m.(*Mount)
 	if !ok {
 		return errors.Errorf("invalid mount type %T", m)
@@ -137,26 +137,37 @@ func (fb *FileBackend) Mkdir(ctx context.Context, m fileoptypes.Mount, action pb
 	return mkdir(ctx, dir, action)
 }
 
-func (fb *FileBackend) Mkfile(ctx context.Context, m fileoptypes.Mount, action pb.FileActionMkFile) error {
+func (fb *Backend) Mkfile(ctx context.Context, m fileoptypes.Mount, action pb.FileActionMkFile) error {
 	mnt, ok := m.(*Mount)
 	if !ok {
 		return errors.Errorf("invalid mount type %T", m)
 	}
 
-	_ = mnt
+	lm := snapshot.LocalMounter(mnt.m)
+	dir, err := lm.Mount()
+	if err != nil {
+		return err
+	}
+	defer lm.Unmount()
 
-	return errors.Errorf("mkfile not implemented")
+	return mkfile(ctx, dir, action)
 }
-func (fb *FileBackend) Rm(ctx context.Context, m fileoptypes.Mount, action pb.FileActionRm) error {
+func (fb *Backend) Rm(ctx context.Context, m fileoptypes.Mount, action pb.FileActionRm) error {
 	mnt, ok := m.(*Mount)
 	if !ok {
 		return errors.Errorf("invalid mount type %T", m)
 	}
-	_ = mnt
 
-	return errors.Errorf("rm not implemented")
+	lm := snapshot.LocalMounter(mnt.m)
+	dir, err := lm.Mount()
+	if err != nil {
+		return err
+	}
+	defer lm.Unmount()
+
+	return rm(ctx, dir, action)
 }
-func (fb *FileBackend) Copy(ctx context.Context, m1 fileoptypes.Mount, m2 fileoptypes.Mount, action pb.FileActionCopy) error {
+func (fb *Backend) Copy(ctx context.Context, m1 fileoptypes.Mount, m2 fileoptypes.Mount, action pb.FileActionCopy) error {
 	mnt1, ok := m1.(*Mount)
 	if !ok {
 		return errors.Errorf("invalid mount type %T", m1)
@@ -166,14 +177,19 @@ func (fb *FileBackend) Copy(ctx context.Context, m1 fileoptypes.Mount, m2 fileop
 		return errors.Errorf("invalid mount type %T", m2)
 	}
 
-	_ = mnt1
-	_ = mnt2
-	return errors.Errorf("copy not implemented")
-}
+	lm := snapshot.LocalMounter(mnt1.m)
+	src, err := lm.Mount()
+	if err != nil {
+		return err
+	}
+	defer lm.Unmount()
 
-// type Backend interface {
-// 	Mkdir(context.Context, Mount, pb.FileActionMkDir) error
-// 	Mkfile(context.Context, Mount, pb.FileActionMkFile) error
-// 	Rm(context.Context, Mount, pb.FileActionRm) error
-// 	Copy(context.Context, Mount, Mount, pb.FileActionCopy) error
-// }
+	lm2 := snapshot.LocalMounter(mnt2.m)
+	dest, err := lm2.Mount()
+	if err != nil {
+		return err
+	}
+	defer lm2.Unmount()
+
+	return docopy(ctx, src, dest, action)
+}

solver/llbsolver/file/refmanager.go

@@ -9,13 +9,17 @@ import (
 	"github.com/pkg/errors"
 )
 
+func NewRefManager(cm cache.Manager) *RefManager {
+	return &RefManager{cm: cm}
+}
+
 type RefManager struct {
 	cm cache.Manager
 }
 
 func (rm *RefManager) Prepare(ctx context.Context, ref fileoptypes.Ref, readonly bool) (fileoptypes.Mount, error) {
 	ir, ok := ref.(cache.ImmutableRef)
-	if !ok {
+	if !ok && ref != nil {
 		return nil, errors.Errorf("invalid ref type: %T", ref)
 	}
 
@@ -43,7 +47,7 @@ func (rm *RefManager) Commit(ctx context.Context, mount fileoptypes.Mount) (file
 	if !ok {
 		return nil, errors.Errorf("invalid mount type %T", mount)
 	}
-	if err := m.Release(context.TODO()); err != nil {
+	if err := m.m.Release(); err != nil {
 		return nil, err
 	}
 	if m.mr == nil {