oci/cas/dir: Load blob URI from oci-layout

With this change, users can configure their blob storage once at init
time with an optional --blob-uri.  Most other commands (which do not
need path -> blob conversion) can load the blob location from the
oci-layout layout file (the 1.1.0 format is in flight with [1,2]).
The only other user-facing change is to 'umoci gc', which gains a
--digest-regexp.  Folks who customized their blob URI will need to
supply --digest-regexp to reverse whichever blob URI they're using.

This seems like a more convenient interface to me than requiring all
callers to provide the custom blob location [3].  And it is more
powerful as well, allowing users to shard their blob storage [4],
etc. if they feel moved to do so.

[1]: xiekeyang/oci-discovery#20
[2]: https://github.com/wking/image-spec/blob/ref-engine-discovery-layout/image-layout.md
[3]: https://github.com/openSUSE/umoci/pull/190
[4]: opencontainers/image-spec#449

Signed-off-by: W. Trevor King <wking@tremily.us>

cmd/umoci/gc.go

@@ -18,10 +18,13 @@
 package main
 
 import (
+	"regexp"
+
 	"github.com/openSUSE/umoci/oci/cas/dir"
 	"github.com/openSUSE/umoci/oci/casext"
 	"github.com/pkg/errors"
 	"github.com/urfave/cli"
+	casDir "github.com/wking/casengine/dir"
 	"golang.org/x/net/context"
 )
 
@@ -35,6 +38,12 @@ Where "<image-path>" is the path to the OCI image.
 This command will do a mark-and-sweep garbage collection of the provided OCI
 image, only retaining blobs which can be reached by a descriptor path from the
 root set of references. All other blobs will be removed.`,
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:  "digest-regexp",
+			Usage: "regular expression for calculating the digest from a filesystem path.  This is required if your oci-layout declares an oci-cas-template-v1 CAS engine (e.g. via 'umoci init --blob-uri ...')",
+		},
+	},
 
 	// create modifies an image layout.
 	Category: "layout",
@@ -52,8 +61,22 @@ root set of references. All other blobs will be removed.`,
 func gc(ctx *cli.Context) error {
 	imagePath := ctx.App.Metadata["--image-path"].(string)
 
+	var getDigest casDir.GetDigest
+	if ctx.IsSet("digest-regexp") {
+		getDigestRegexp, err := regexp.Compile(ctx.String("digest-regexp"))
+		if err != nil {
+			return errors.Wrap(err, "compile digest-regexp")
+		}
+
+		regexpGetDigest := &casDir.RegexpGetDigest{
+			Regexp: getDigestRegexp,
+		}
+
+		getDigest = regexpGetDigest.GetDigest
+	}
+
 	// Get a reference to the CAS.
-	engine, err := dir.Open(imagePath)
+	engine, err := dir.OpenWithDigestLister(imagePath, getDigest)
 	if err != nil {
 		return errors.Wrap(err, "open CAS")
 	}

cmd/umoci/init.go

@@ -38,6 +38,13 @@ The new OCI image does not contain any references or blobs, but those can be
 created through the use of umoci-new(1), umoci-tag(1) and other similar
 commands.`,
 
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:  "blob-uri",
+			Usage: "URI Template for storing blobs, interpreted with the image path as the base URI.  Defaults to blobs/{algorithm}/{encoded}",
+		},
+	},
+
 	// create modifies an image layout.
 	Category: "layout",
 
@@ -54,7 +61,7 @@ func initLayout(ctx *cli.Context) error {
 		return errors.Wrap(err, "image layout creation")
 	}
 
-	if err := dir.Create(imagePath); err != nil {
+	if err := dir.Create(imagePath, ctx.String("blob-uri")); err != nil {
 		return errors.Wrap(err, "image layout creation")
 	}
 

doc/man/umoci-gc.1.md

@@ -7,6 +7,7 @@ umoci gc - Garbage collects all unreferenced OCI image blobs
 # SYNOPSIS
 **umoci gc**
 **--layout**=*image*
+[**--digest-regexp**=*regexp*]
 
 # DESCRIPTION
 Conduct a mark-and-sweep garbage collection of the provided OCI image, only
@@ -20,6 +21,18 @@ The global options are defined in **umoci**(1).
   The OCI image layout to be garbage collected. *image* must be a path to a
   valid OCI image.
 
+**--digest-regexp**=*regexp*
+  A regular expression for calculating the digest from a filesystem
+  path.  This is required if your oci-layout declares an
+  `oci-cas-template-v1` CAS engine.  For example, if you created the
+  image with:
+
+    umoci init --blob-uri file:///path/to/my/blobs/{algorithm}/{encoded:2}/{encoded}
+
+  Then you should set *regexp* to:
+
+    ^.*/(?P<algorithm>[a-z0-9+._-]+)/[a-zA-Z0-9=_-]{1,2}/(?P<encoded>[a-zA-Z0-9=_-]{1,})$
+
 # EXAMPLE
 
 The following deletes a tag from an OCI image and clean conducts a garbage

doc/man/umoci-init.1.md

@@ -7,6 +7,7 @@ umoci init - Create a new OCI image layout
 # SYNOPSIS
 **umoci init**
 **--layout**=*image*
+[**--blob-uri**=*template*
 
 # DESCRIPTION
 Creates a new OCI image layout. The new OCI image does not contain any new
@@ -21,6 +22,11 @@ The global options are defined in **umoci**(1).
   The path where the OCI image layout will be created. The path must not exist
   already or **umoci-init**(1) will return an error.
 
+**--blob-uri**=*template*
+  The URI Template for retrieving digests.  Relative URIs will be
+  resolved with the image path as the base URI.  For more details,
+  see the [OCI CAS Template Protocol][cas-template].
+
 # EXAMPLE
 
 The following creates a brand new OCI image layout and then creates a blank tag
@@ -33,3 +39,5 @@ for further manipulation with **umoci-repack**(1) and **umoci-config**(1).
 
 # SEE ALSO
 **umoci**(1), **umoci-new**(1)
+
+[cas-template]: https://github.com/xiekeyang/oci-discovery/blob/0be7eae246ae9a975a76ca209c045043f0793572/cas-template.md

hack/vendor.sh

@@ -135,6 +135,7 @@ clone github.com/jtacoma/uritemplates v1.0.0
 clone github.com/vbatts/go-mtree 005af4d18f8ab74174ce23565be732a3101cf316
 clone github.com/sirupsen/logrus v1.0.3
 clone github.com/wking/casengine 3ed08888a9365a2753ab8b809b7efb286566fe8d
+clone github.com/xiekeyang/oci-discovery 17aaa9ee7538d1db09b5f142ed319e06dee7407e
 clone golang.org/x/net 45e771701b814666a7eb299e6c7a57d0b1799e91 https://github.com/golang/net
 # Used purely for testing.
 clone github.com/mohae/deepcopy 491d3605edfb866af34a48075bd4355ac1bf46ca

mutate/mutate_test.go

@@ -47,7 +47,7 @@ const (
 
 func setup(t *testing.T, dir string) (cas.Engine, ispec.Descriptor) {
 	dir = filepath.Join(dir, "image")
-	if err := casdir.Create(dir); err != nil {
+	if err := casdir.Create(dir, ""); err != nil {
 		t.Fatal(err)
 	}
 

oci/cas/cas.go

@@ -67,6 +67,10 @@ type Engine interface {
 	// CAS returns the casengine.Engine backing this engine.
 	CAS() (casEngine casengine.Engine)
 
+	// DigestListerEngine returns the casengine.DigestListerEngine
+	// backing this engine, or nil if no such engine exists.
+	DigestListerEngine() (casEngine casengine.DigestListerEngine)
+
 	// PutBlob adds a new blob to the image. This is idempotent; a nil error
 	// means that "the content is stored at DIGEST" without implying "because
 	// of this PutBlob() call".
@@ -106,7 +110,7 @@ type Engine interface {
 
 	// ListBlobs returns the set of blob digests stored in the image.
 	//
-	// Deprecated: Use CAS().Digests instead.
+	// Deprecated: Use DigestListerEngine().Digests instead.
 	ListBlobs(ctx context.Context) (digests []digest.Digest, err error)
 
 	// Clean executes a garbage collection of any non-blob garbage in the store

oci/cas/dir/cas_test.go

@@ -19,14 +19,18 @@ package dir
 
 import (
 	"bytes"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"regexp"
 	"testing"
 
 	"github.com/openSUSE/umoci/oci/cas"
+	"github.com/opencontainers/go-digest"
 	"github.com/pkg/errors"
+	"github.com/wking/casengine/dir"
 	"golang.org/x/net/context"
 )
 
@@ -43,7 +47,8 @@ func TestCreateLayout(t *testing.T) {
 	defer os.RemoveAll(root)
 
 	image := filepath.Join(root, "image")
-	if err := Create(image); err != nil {
+
+	if err := Create(image, ""); err != nil {
 		t.Fatalf("unexpected error creating image: %+v", err)
 	}
 
@@ -66,7 +71,7 @@ func TestCreateLayout(t *testing.T) {
 	}
 
 	// We should get an error if we try to create a new image atop an old one.
-	if err := Create(image); err == nil {
+	if err := Create(image, ""); err == nil {
 		t.Errorf("expected to get a cowardly no-clobber error!")
 	}
 }
@@ -81,7 +86,7 @@ func TestEngineBlob(t *testing.T) {
 	defer os.RemoveAll(root)
 
 	image := filepath.Join(root, "image")
-	if err := Create(image); err != nil {
+	if err := Create(image, ""); err != nil {
 		t.Fatalf("unexpected error creating image: %+v", err)
 	}
 
@@ -214,7 +219,7 @@ func TestEngineValidate(t *testing.T) {
 	if err := os.Remove(image); err != nil {
 		t.Fatal(err)
 	}
-	if err := Create(image); err != nil {
+	if err := Create(image, ""); err != nil {
 		t.Fatalf("unexpected error creating image: %+v", err)
 	}
 	if err := os.RemoveAll(filepath.Join(image, blobDirectory)); err != nil {
@@ -234,7 +239,7 @@ func TestEngineValidate(t *testing.T) {
 	if err := os.Remove(image); err != nil {
 		t.Fatal(err)
 	}
-	if err := Create(image); err != nil {
+	if err := Create(image, ""); err != nil {
 		t.Fatalf("unexpected error creating image: %+v", err)
 	}
 	if err := os.RemoveAll(filepath.Join(image, blobDirectory)); err != nil {
@@ -257,7 +262,7 @@ func TestEngineValidate(t *testing.T) {
 	if err := os.Remove(image); err != nil {
 		t.Fatal(err)
 	}
-	if err := Create(image); err != nil {
+	if err := Create(image, ""); err != nil {
 		t.Fatalf("unexpected error creating image: %+v", err)
 	}
 	if err := os.RemoveAll(filepath.Join(image, indexFile)); err != nil {
@@ -277,7 +282,7 @@ func TestEngineValidate(t *testing.T) {
 	if err := os.Remove(image); err != nil {
 		t.Fatal(err)
 	}
-	if err := Create(image); err != nil {
+	if err := Create(image, ""); err != nil {
 		t.Fatalf("unexpected error creating image: %+v", err)
 	}
 	if err := os.RemoveAll(filepath.Join(image, indexFile)); err != nil {
@@ -300,3 +305,73 @@ func TestEngineValidate(t *testing.T) {
 		engine.Close()
 	}
 }
+
+func TestEngineURITemplate(t *testing.T) {
+	ctx := context.Background()
+
+	root, err := ioutil.TempDir("", "umoci-TestEngineURITemplate")
+	if err != nil {
+		t.Fatal(err)
+	}
+	//defer os.RemoveAll(root)
+
+	image := filepath.Join(root, "image")
+
+	if filepath.Separator != '/' {
+		t.Fatalf("CAS URI Template initialization is not implemented for filepath.Separator %q", filepath.Separator)
+	}
+
+	if err := Create(image, fmt.Sprintf("file://%s/blobs/{algorithm}/{encoded:2}/{encoded}", root)); err != nil {
+		t.Fatalf("unexpected error creating image: %+v", err)
+	}
+
+	getDigestRegexp, err := regexp.Compile(`^.*/blobs/(?P<algorithm>[a-z0-9+._-]+)/[a-zA-Z0-9=_-]{1,2}/(?P<encoded>[a-zA-Z0-9=_-]{1,})$`)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	getDigest := &dir.RegexpGetDigest{
+		Regexp: getDigestRegexp,
+	}
+
+	engine, err := OpenWithDigestLister(image, getDigest.GetDigest)
+	if err != nil {
+		t.Fatalf("unexpected error opening image: %+v", err)
+	}
+	defer engine.Close()
+
+	bytesIn := []byte("Hello, World!")
+	dig, err := engine.CAS().Put(ctx, digest.SHA256, bytes.NewReader(bytesIn))
+	if err != nil {
+		t.Errorf("Put: unexpected error: %+v", err)
+	}
+
+	reader, err := engine.CAS().Get(ctx, dig)
+	if err != nil {
+		t.Errorf("Get: unexpected error: %+v", err)
+	}
+	defer reader.Close()
+
+	gotBytes, err := ioutil.ReadAll(reader)
+	if err != nil {
+		t.Errorf("Get: failed to ReadAll: %+v", err)
+	}
+	if !bytes.Equal(bytesIn, gotBytes) {
+		t.Errorf("Get: bytes did not match: expected=%s got=%s", string(bytesIn), string(gotBytes))
+	}
+
+	path := filepath.Join(root, "blobs", digest.SHA256.String(), "df", "dffd6021bb2bd5b0af676290809ec3a53191dd81c7f70a4b28688a362182986f")
+	reader, err = os.Open(path)
+	if err != nil {
+		t.Error(err)
+	}
+	defer reader.Close()
+
+	gotBytes, err = ioutil.ReadAll(reader)
+	if err != nil {
+		t.Errorf("Open: failed to ReadAll: %+v", err)
+	}
+	if !bytes.Equal(bytesIn, gotBytes) {
+		t.Errorf("Open: bytes did not match: expected=%s got=%s", string(bytesIn), string(gotBytes))
+	}
+}