Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

llb: fileop implementation #809

Merged
merged 25 commits into from
Mar 18, 2019
Merged
Changes from 1 commit
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
5b4841f
llb: initial fileop implementation
tonistiigi Jan 23, 2019
431d11d
llb: add timestamp override to fileop
tonistiigi Feb 1, 2019
89e6614
solver: change uid to uint
tonistiigi Feb 1, 2019
a443cff
fileop: resolve review comments
tonistiigi Feb 5, 2019
9fb1f09
llbsolver: fileop base
tonistiigi Feb 8, 2019
b2b0e3d
fileop: add release support and tests
tonistiigi Feb 20, 2019
2be999b
fileop: llbsolver implementation
tonistiigi Feb 21, 2019
81a5fa5
llbsolver: fileop implementation
tonistiigi Feb 27, 2019
4ffd797
fileop: connect with contenthash
tonistiigi Mar 3, 2019
171feaa
vendor: add fsutil copy package
tonistiigi Mar 15, 2019
7210bf6
fileop: add chown support
tonistiigi Mar 15, 2019
8a4674b
fileop: add dockerfile support
tonistiigi Mar 8, 2019
0d17ac3
fileop: updates with new fsutil copy pkg
tonistiigi Mar 9, 2019
33955c9
vendor: revendor new packages
tonistiigi Mar 11, 2019
bb11c8c
dockerfile: empty wildcard regression test
tonistiigi Mar 10, 2019
39ba2ed
dockerfile: regression test for workdir creation
tonistiigi Mar 10, 2019
369f748
dockerfile: regression test for wildcard unpack
tonistiigi Mar 10, 2019
c4ef668
dockerfile: regression test for existing dest dir perms
tonistiigi Mar 11, 2019
c57e5b2
dockerfile: add regression test for wildcard cache
tonistiigi Mar 10, 2019
637bec7
dockerfile: make fileop default
tonistiigi Mar 11, 2019
f38d971
dockerfile: add matrix testing for non-fileop versions
tonistiigi Mar 11, 2019
a16b47f
integration: disable gc
tonistiigi Mar 11, 2019
c6149da
fileop: review fixes
tonistiigi Mar 11, 2019
6fd30e7
dockerfile: add more tests to fix fileop
Mar 15, 2019
5effbff
dockerfile: improve symlinks copy tests
tonistiigi Mar 16, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
llb: add timestamp override to fileop
Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
tonistiigi committed Mar 15, 2019
commit 431d11dda35b90ea231beaceb6de588361f565ab
42 changes: 37 additions & 5 deletions client/llb/fileop.go
Original file line number Diff line number Diff line change
@@ -4,6 +4,7 @@ import (
_ "crypto/sha256"
"os"
"path"
"time"

"github.com/moby/buildkit/solver/pb"
digest "github.com/opencontainers/go-digest"
@@ -149,6 +150,7 @@ func (a *fileActionMkdir) toProtoAction(parent string, base pb.InputIndex) pb.Is
Mode: int32(a.mode & 0777),
MakeParents: a.info.MakeParents,
Owner: a.info.ChownOpt.marshal(base),
Timestamp: marshalTime(a.info.CreatedTime),
},
}
}
@@ -180,6 +182,7 @@ func WithParents(b bool) MkdirOption {
type MkdirInfo struct {
MakeParents bool
ChownOpt *ChownOpt
CreatedTime *time.Time
}

func (mi *MkdirInfo) SetMkdirOption(mi2 *MkdirInfo) {
@@ -260,7 +263,8 @@ type MkfileOption interface {
}

type MkfileInfo struct {
ChownOpt *ChownOpt
ChownOpt *ChownOpt
CreatedTime *time.Time
}

func (mi *MkfileInfo) SetMkfileOption(mi2 *MkfileInfo) {
@@ -279,10 +283,11 @@ type fileActionMkfile struct {
func (a *fileActionMkfile) toProtoAction(parent string, base pb.InputIndex) pb.IsFileAction {
return &pb.FileAction_Mkfile{
Mkfile: &pb.FileActionMkFile{
Path: normalizePath(parent, a.file),
Mode: int32(a.mode & 0777),
Data: a.dt,
Owner: a.info.ChownOpt.marshal(base),
Path: normalizePath(parent, a.file),
Mode: int32(a.mode & 0777),
Data: a.dt,
Owner: a.info.ChownOpt.marshal(base),
Timestamp: marshalTime(a.info.CreatedTime),
},
}
}
@@ -391,6 +396,7 @@ type CopyInfo struct {
AllowWildcard bool
AllowEmptyWildcard bool
ChownOpt *ChownOpt
CreatedTime *time.Time
}

func (mi *CopyInfo) SetCopyOption(mi2 *CopyInfo) {
@@ -418,6 +424,7 @@ func (a *fileActionCopy) toProtoAction(parent string, base pb.InputIndex) pb.IsF
DirCopyContents: a.info.CopyDirContentsOnly,
AttemptUnpack: a.info.AttemptUnpack,
CreateDestPath: a.info.CreateDestPath,
Timestamp: marshalTime(a.info.CreatedTime),
}
if a.info.Mode != nil {
c.Mode = int32(*a.info.Mode)
@@ -441,6 +448,31 @@ func (c *fileActionCopy) sourcePath() string {
return p
}

type CreatedTime time.Time

func WithCreatedTime(t time.Time) CreatedTime {
return CreatedTime(t)
}

func (c CreatedTime) SetMkdirOption(mi *MkdirInfo) {
mi.CreatedTime = (*time.Time)(&c)
}

func (c CreatedTime) SetMkfileOption(mi *MkfileInfo) {
mi.CreatedTime = (*time.Time)(&c)
}

func (c CreatedTime) SetCopyOption(mi *CopyInfo) {
mi.CreatedTime = (*time.Time)(&c)
}

func marshalTime(t *time.Time) int64 {
if t == nil {
return -1
}
return t.UnixNano()
}

type FileOp struct {
MarshalCache
action *FileAction
46 changes: 46 additions & 0 deletions client/llb/fileop_test.go
Original file line number Diff line number Diff line change
@@ -2,6 +2,7 @@ package llb

import (
"testing"
"time"

"github.com/moby/buildkit/solver/pb"
digest "github.com/opencontainers/go-digest"
@@ -39,6 +40,7 @@ func TestFileMkdir(t *testing.T) {

require.Equal(t, "/foo", mkdir.Path)
require.Equal(t, 0700, int(mkdir.Mode))
require.Equal(t, int64(-1), mkdir.Timestamp)
}

func TestFileMkdirChain(t *testing.T) {
@@ -126,6 +128,7 @@ func TestFileMkfile(t *testing.T) {
require.Equal(t, "/foo", mkdir.Path)
require.Equal(t, 0700, int(mkdir.Mode))
require.Equal(t, "data", string(mkdir.Data))
require.Equal(t, int64(-1), mkdir.Timestamp)
}

func TestFileRm(t *testing.T) {
@@ -270,6 +273,7 @@ func TestFileCopy(t *testing.T) {

require.Equal(t, "/etc/foo", copy.Src)
require.Equal(t, "/tmp/bar", copy.Dest)
require.Equal(t, int64(-1), copy.Timestamp)
}

func TestFileCopyFromAction(t *testing.T) {
@@ -560,6 +564,48 @@ func TestFileCopyOwner(t *testing.T) {
require.Equal(t, -1, int(copy.Owner.Group.Input))
}

func TestFileCreatedTime(t *testing.T) {
t.Parallel()

dt := time.Now()
dt2 := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC)
dt3 := time.Date(2019, time.November, 10, 23, 0, 0, 0, time.UTC)

st := Image("foo").File(
Mkdir("/foo", 0700, WithCreatedTime(dt)).
Mkfile("bar", 0600, []byte{}, WithCreatedTime(dt2)).
Copy(Scratch(), "src", "dst", WithCreatedTime(dt3)))
def, err := st.Marshal()

require.NoError(t, err)

m, arr := parseDef(t, def.Def)
require.Equal(t, 3, len(arr))

dgst, idx := last(t, arr)
require.Equal(t, 0, idx)
require.Equal(t, m[dgst], arr[1])

f := arr[1].Op.(*pb.Op_File).File
require.Equal(t, len(arr[1].Inputs), 1)
require.Equal(t, m[arr[1].Inputs[0].Digest], arr[0])
require.Equal(t, 0, int(arr[1].Inputs[0].Index))

require.Equal(t, 3, len(f.Actions))

action := f.Actions[0]
mkdir := action.Action.(*pb.FileAction_Mkdir).Mkdir
require.Equal(t, dt.UnixNano(), mkdir.Timestamp)

action = f.Actions[1]
mkfile := action.Action.(*pb.FileAction_Mkfile).Mkfile
require.Equal(t, dt2.UnixNano(), mkfile.Timestamp)

action = f.Actions[2]
copy := action.Action.(*pb.FileAction_Copy).Copy
require.Equal(t, dt3.UnixNano(), copy.Timestamp)
}

func parseDef(t *testing.T, def [][]byte) (map[digest.Digest]pb.Op, []pb.Op) {
m := map[digest.Digest]pb.Op{}
arr := make([]pb.Op, 0, len(def))
Loading