Skip to content

Commit

Permalink
exporter: add tar exporter
Browse files Browse the repository at this point in the history
Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
  • Loading branch information
tonistiigi committed Mar 27, 2019
1 parent 33bb70c commit d08bff3
Show file tree
Hide file tree
Showing 13 changed files with 407 additions and 30 deletions.
8 changes: 8 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,14 @@ The local client will copy the files directly to the client. This is useful if B
buildctl build ... --output type=local,dest=path/to/output-dir
```
Tar exporter is similar to local exporter but transfers the files through a tarball.
```
buildctl build ... --output type=tar,dest=out.tar
buildctl build ... --output type=tar > out.tar
```
##### Exporting built image to Docker
```
Expand Down
1 change: 1 addition & 0 deletions client/exporters.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package client
const (
ExporterImage = "image"
ExporterLocal = "local"
ExporterTar = "tar"
ExporterOCI = "oci"
ExporterDocker = "docker"
)
2 changes: 1 addition & 1 deletion client/solve.go
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
return nil, errors.New("output directory is required for local exporter")
}
s.Allow(filesync.NewFSSyncTargetDir(ex.OutputDir))
case ExporterOCI, ExporterDocker:
case ExporterOCI, ExporterDocker, ExporterTar:
if ex.OutputDir != "" {
return nil, errors.Errorf("output directory %s is not supported by %s exporter", ex.OutputDir, ex.Type)
}
Expand Down
4 changes: 2 additions & 2 deletions cmd/buildctl/build/output.go
Original file line number Diff line number Diff line change
Expand Up @@ -95,8 +95,8 @@ func resolveExporterDest(exporter, dest string) (io.WriteCloser, string, error)
return nil, "", errors.New("output directory is required for local exporter")
}
return nil, dest, nil
case client.ExporterOCI, client.ExporterDocker:
if dest != "" {
case client.ExporterOCI, client.ExporterDocker, client.ExporterTar:
if dest != "" && dest != "-" {
fi, err := os.Stat(dest)
if err != nil && !os.IsNotExist(err) {
return nil, "", errors.Wrapf(err, "invalid destination file: %s", dest)
Expand Down
7 changes: 5 additions & 2 deletions exporter/local/export.go
Original file line number Diff line number Diff line change
Expand Up @@ -93,10 +93,13 @@ func (e *localExporterInstance) Export(ctx context.Context, inp exporter.Source)
lbl := "copying files"
if isMap {
lbl += " " + k
fs = fsutil.SubDirFS(fs, fstypes.Stat{
fs, err = fsutil.SubDirFS([]fsutil.Dir{{FS: fs, Stat: fstypes.Stat{
Mode: uint32(os.ModeDir | 0755),
Path: strings.Replace(k, "/", "_", -1),
})
}}})
if err != nil {
return err
}
}

progress := newProgressHandler(ctx, lbl)
Expand Down
155 changes: 155 additions & 0 deletions exporter/tar/export.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
package local

import (
"context"
"io/ioutil"
"os"
"strings"
"time"

"github.com/moby/buildkit/cache"
"github.com/moby/buildkit/exporter"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/session/filesync"
"github.com/moby/buildkit/snapshot"
"github.com/moby/buildkit/util/progress"
"github.com/pkg/errors"
"github.com/tonistiigi/fsutil"
fstypes "github.com/tonistiigi/fsutil/types"
)

type Opt struct {
SessionManager *session.Manager
}

type localExporter struct {
opt Opt
// session manager
}

func New(opt Opt) (exporter.Exporter, error) {
le := &localExporter{opt: opt}
return le, nil
}

func (e *localExporter) Resolve(ctx context.Context, opt map[string]string) (exporter.ExporterInstance, error) {
id := session.FromContext(ctx)
if id == "" {
return nil, errors.New("could not access local files without session")
}

timeoutCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()

caller, err := e.opt.SessionManager.Get(timeoutCtx, id)
if err != nil {
return nil, err
}

li := &localExporterInstance{localExporter: e, caller: caller}
return li, nil
}

type localExporterInstance struct {
*localExporter
caller session.Caller
}

func (e *localExporterInstance) Name() string {
return "exporting to client"
}

func (e *localExporterInstance) Export(ctx context.Context, inp exporter.Source) (map[string]string, error) {
var defers []func()

defer func() {
for i := len(defers) - 1; i >= 0; i-- {
defers[i]()
}
}()

getDir := func(ctx context.Context, k string, ref cache.ImmutableRef) (*fsutil.Dir, error) {
var src string
var err error
if ref == nil {
src, err = ioutil.TempDir("", "buildkit")
if err != nil {
return nil, err
}
defers = append(defers, func() { os.RemoveAll(src) })
} else {
mount, err := ref.Mount(ctx, true)
if err != nil {
return nil, err
}

lm := snapshot.LocalMounter(mount)

src, err = lm.Mount()
if err != nil {
return nil, err
}
defers = append(defers, func() { lm.Unmount() })
}

return &fsutil.Dir{
FS: fsutil.NewFS(src, nil),
Stat: fstypes.Stat{
Mode: uint32(os.ModeDir | 0755),
Path: strings.Replace(k, "/", "_", -1),
},
}, nil
}

var fs fsutil.FS

if len(inp.Refs) > 0 {
dirs := make([]fsutil.Dir, 0, len(inp.Refs))
for k, ref := range inp.Refs {
d, err := getDir(ctx, k, ref)
if err != nil {
return nil, err
}
dirs = append(dirs, *d)
}
var err error
fs, err = fsutil.SubDirFS(dirs)
if err != nil {
return nil, err
}
} else {
d, err := getDir(ctx, "", inp.Ref)
if err != nil {
return nil, err
}
fs = d.FS
}

w, err := filesync.CopyFileWriter(ctx, e.caller)
if err != nil {
return nil, err
}
report := oneOffProgress(ctx, "sending tarball")
if err := fsutil.WriteTar(ctx, fs, w); err != nil {
w.Close()
return nil, report(err)
}
return nil, report(w.Close())
}

func oneOffProgress(ctx context.Context, id string) func(err error) error {
pw, _, _ := progress.FromContext(ctx)
now := time.Now()
st := progress.Status{
Started: &now,
}
pw.Write(id, st)
return func(err error) error {
// TODO: set error on status
now := time.Now()
st.Completed = &now
pw.Write(id, st)
pw.Close()
return err
}
}
86 changes: 86 additions & 0 deletions frontend/dockerfile/dockerfile_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"context"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/http/httptest"
Expand Down Expand Up @@ -84,6 +85,7 @@ var allTests = []integration.Test{
testCopyChownExistingDir,
testCopyWildcardCache,
testDockerignoreOverride,
testTarExporter,
}

var fileOpTests = []integration.Test{
Expand Down Expand Up @@ -235,6 +237,84 @@ RUN [ "$(cat testfile)" == "contents0" ]
require.NoError(t, err)
}

func testTarExporter(t *testing.T, sb integration.Sandbox) {
f := getFrontend(t, sb)

dockerfile := []byte(`
FROM scratch AS stage-linux
COPY foo forlinux
FROM scratch AS stage-darwin
COPY bar fordarwin
FROM stage-$TARGETOS
`)

dir, err := tmpdir(
fstest.CreateFile("Dockerfile", dockerfile, 0600),
fstest.CreateFile("foo", []byte("data"), 0600),
fstest.CreateFile("bar", []byte("data2"), 0600),
)
require.NoError(t, err)
defer os.RemoveAll(dir)

c, err := client.New(context.TODO(), sb.Address())
require.NoError(t, err)
defer c.Close()

buf := &bytes.Buffer{}
_, err = f.Solve(context.TODO(), c, client.SolveOpt{
Exports: []client.ExportEntry{
{
Type: client.ExporterTar,
Output: &nopWriteCloser{buf},
},
},
LocalDirs: map[string]string{
builder.DefaultLocalNameDockerfile: dir,
builder.DefaultLocalNameContext: dir,
},
}, nil)
require.NoError(t, err)

m, err := testutil.ReadTarToMap(buf.Bytes(), false)
require.NoError(t, err)

mi, ok := m["forlinux"]
require.Equal(t, true, ok)
require.Equal(t, "data", string(mi.Data))

// repeat multi-platform
buf = &bytes.Buffer{}
_, err = f.Solve(context.TODO(), c, client.SolveOpt{
Exports: []client.ExportEntry{
{
Type: client.ExporterTar,
Output: &nopWriteCloser{buf},
},
},
FrontendAttrs: map[string]string{
"platform": "linux/amd64,darwin/amd64",
},
LocalDirs: map[string]string{
builder.DefaultLocalNameDockerfile: dir,
builder.DefaultLocalNameContext: dir,
},
}, nil)
require.NoError(t, err)

m, err = testutil.ReadTarToMap(buf.Bytes(), false)
require.NoError(t, err)

mi, ok = m["linux_amd64/forlinux"]
require.Equal(t, true, ok)
require.Equal(t, "data", string(mi.Data))

mi, ok = m["darwin_amd64/fordarwin"]
require.Equal(t, true, ok)
require.Equal(t, "data2", string(mi.Data))
}

func testWorkdirCreatesDir(t *testing.T, sb integration.Sandbox) {
f := getFrontend(t, sb)

Expand Down Expand Up @@ -3824,3 +3904,9 @@ func getFileOp(t *testing.T, sb integration.Sandbox) bool {
require.True(t, ok)
return vv
}

type nopWriteCloser struct {
io.Writer
}

func (nopWriteCloser) Close() error { return nil }
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ require (
github.com/sirupsen/logrus v1.3.0
github.com/stretchr/testify v1.3.0
github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8 // indirect
github.com/tonistiigi/fsutil v0.0.0-20190319020005-1bdbf124ad49
github.com/tonistiigi/fsutil v0.0.0-20190327153851-3bbb99cdbd76
github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea
github.com/uber/jaeger-client-go v0.0.0-20180103221425-e02c85f9069e
github.com/uber/jaeger-lib v1.2.1 // indirect
Expand Down
5 changes: 3 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -128,8 +128,9 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8 h1:zLV6q4e8Jv9EHjNg/iHfzwDkCve6Ua5jCygptrtXHvI=
github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
github.com/tonistiigi/fsutil v0.0.0-20190319020005-1bdbf124ad49 h1:UFQ7uDVXIH4fFfOb+fISgTl8Ukk0CkGQudHQh980l+0=
github.com/tonistiigi/fsutil v0.0.0-20190319020005-1bdbf124ad49/go.mod h1:pzh7kdwkDRh+Bx8J30uqaKJ1M4QrSH/um8fcIXeM8rc=
github.com/tonistiigi/fsutil v0.0.0-20190327055408-8468a1ee4798 h1:VVioEQ8N7rEuRkdHRFsub7qz0nnHpdWkwabkpfXrV/w=
github.com/tonistiigi/fsutil v0.0.0-20190327055408-8468a1ee4798/go.mod h1:pzh7kdwkDRh+Bx8J30uqaKJ1M4QrSH/um8fcIXeM8rc=
github.com/tonistiigi/fsutil v0.0.0-20190327153851-3bbb99cdbd76/go.mod h1:pzh7kdwkDRh+Bx8J30uqaKJ1M4QrSH/um8fcIXeM8rc=
github.com/tonistiigi/go-immutable-radix v0.0.0-20170803185627-826af9ccf0fe h1:pd7hrFSqUPxYS9IB+UMG1AB/8EXGXo17ssx0bSQ5L6Y=
github.com/tonistiigi/go-immutable-radix v0.0.0-20170803185627-826af9ccf0fe/go.mod h1:/+MCh11CJf2oz0BXmlmqyopK/ad1rKkcOXPoYuPCJYU=
github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea h1:SXhTLE6pb6eld/v/cCndK0AMpt1wiVFb/YYmqB3/QG0=
Expand Down
Loading

0 comments on commit d08bff3

Please sign in to comment.