Skip to content

Commit

Permalink
feat: support artifact discovering in OCI image layout (oras-project#765
Browse files Browse the repository at this point in the history
)

Signed-off-by: Billy Zha <jinzha1@microsoft.com>
  • Loading branch information
qweeah authored and Terry Howe committed Feb 2, 2023
1 parent 57ce9b7 commit dcfb2e2
Show file tree
Hide file tree
Showing 3 changed files with 262 additions and 27 deletions.
44 changes: 17 additions & 27 deletions cmd/oras/discover.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,8 @@ import (

"gopkg.in/yaml.v3"
"oras.land/oras-go/v2"
"oras.land/oras-go/v2/registry/remote"
"oras.land/oras/cmd/oras/internal/errors"
"oras.land/oras/cmd/oras/internal/option"
"oras.land/oras/internal/graph"

"github.com/need-being/go-tree"
"github.com/opencontainers/image-spec/specs-go"
Expand All @@ -36,10 +35,9 @@ import (

type discoverOptions struct {
option.Common
option.Remote
option.Platform
option.Target

targetRef string
artifactType string
outputType string
}
Expand Down Expand Up @@ -70,13 +68,17 @@ Example - Discover all the referrers of manifest with annotations, displayed in
Example - Discover referrers with type 'test-artifact' of manifest 'hello:v1' in registry 'localhost:5000':
oras discover --artifact-type test-artifact localhost:5000/hello:v1
Example - Discover referrers of the manifest tagged 'v1' in an OCI layout folder 'layout-dir':
oras discover --oci-layout layout-dir:v1
oras discover --oci-layout -v -o tree layout-dir:v1
`,
Args: cobra.ExactArgs(1),
PreRunE: func(cmd *cobra.Command, args []string) error {
opts.RawReference = args[0]
return option.Parse(&opts)
},
RunE: func(cmd *cobra.Command, args []string) error {
opts.targetRef = args[0]
return runDiscover(opts)
},
}
Expand All @@ -90,32 +92,32 @@ Example - Discover referrers with type 'test-artifact' of manifest 'hello:v1' in

func runDiscover(opts discoverOptions) error {
ctx, _ := opts.SetLoggerLevel()
repo, err := opts.NewRepository(opts.targetRef, opts.Common)
repo, err := opts.NewReadonlyTarget(ctx, opts.Common)
if err != nil {
return err
}
if repo.Reference.Reference == "" {
return errors.NewErrInvalidReference(repo.Reference)
if err := opts.EnsureReferenceNotEmpty(); err != nil {
return err
}

// discover artifacts
resolveOpts := oras.DefaultResolveOptions
resolveOpts.TargetPlatform = opts.Platform.Platform
desc, err := oras.Resolve(ctx, repo, repo.Reference.Reference, resolveOpts)
desc, err := oras.Resolve(ctx, repo, opts.Reference, resolveOpts)
if err != nil {
return err
}

if opts.outputType == "tree" {
root := tree.New(repo.Reference.String())
root := tree.New(opts.Reference)
err = fetchAllReferrers(ctx, repo, desc, opts.artifactType, root, &opts)
if err != nil {
return err
}
return tree.Print(root)
}

refs, err := fetchReferrers(ctx, repo, desc, opts.artifactType)
refs, err := graph.Referrers(ctx, repo, desc, opts.artifactType)
if err != nil {
return err
}
Expand All @@ -124,9 +126,9 @@ func runDiscover(opts discoverOptions) error {
}

if n := len(refs); n > 1 {
fmt.Println("Discovered", n, "artifacts referencing", repo.Reference)
fmt.Println("Discovered", n, "artifacts referencing", opts.Reference)
} else {
fmt.Println("Discovered", n, "artifact referencing", repo.Reference)
fmt.Println("Discovered", n, "artifact referencing", opts.Reference)
}
fmt.Println("Digest:", desc.Digest)
if len(refs) > 0 {
Expand All @@ -136,20 +138,8 @@ func runDiscover(opts discoverOptions) error {
return nil
}

func fetchReferrers(ctx context.Context, repo *remote.Repository, desc ocispec.Descriptor, artifactType string) ([]ocispec.Descriptor, error) {
results := []ocispec.Descriptor{}
err := repo.Referrers(ctx, desc, artifactType, func(referrers []ocispec.Descriptor) error {
results = append(results, referrers...)
return nil
})
if err != nil {
return nil, err
}
return results, nil
}

func fetchAllReferrers(ctx context.Context, repo *remote.Repository, desc ocispec.Descriptor, artifactType string, node *tree.Node, opts *discoverOptions) error {
results, err := fetchReferrers(ctx, repo, desc, artifactType)
func fetchAllReferrers(ctx context.Context, repo oras.ReadOnlyGraphTarget, desc ocispec.Descriptor, artifactType string, node *tree.Node, opts *discoverOptions) error {
results, err := graph.Referrers(ctx, repo, desc, artifactType)
if err != nil {
return err
}
Expand Down
71 changes: 71 additions & 0 deletions internal/graph/graph.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,9 @@ import (
"encoding/json"

ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"oras.land/oras-go/v2"
"oras.land/oras-go/v2/content"
"oras.land/oras-go/v2/registry"
"oras.land/oras/internal/docker"
)

Expand Down Expand Up @@ -59,3 +61,72 @@ func Successors(ctx context.Context, fetcher content.Fetcher, node ocispec.Descr
}
return
}

// Referrers returns referrer nodes of desc in target.
func Referrers(ctx context.Context, target oras.ReadOnlyGraphTarget, desc ocispec.Descriptor, artifactType string) ([]ocispec.Descriptor, error) {
var results []ocispec.Descriptor
if repo, ok := target.(registry.ReferrerLister); ok {
// get referrers directly
err := repo.Referrers(ctx, desc, artifactType, func(referrers []ocispec.Descriptor) error {
results = append(results, referrers...)
return nil
})
if err != nil {
return nil, err
}
return results, nil
}

// find matched referrers in all predecessors
predecessors, err := target.Predecessors(ctx, desc)
if err != nil {
return nil, err
}
for _, node := range predecessors {
switch node.MediaType {
case ocispec.MediaTypeArtifactManifest:
fetched, err := fetchBytes(ctx, target, node)
if err != nil {
return nil, err
}
var artifact ocispec.Artifact
if err := json.Unmarshal(fetched, &artifact); err != nil {
return nil, err
}
if artifact.Subject == nil || !content.Equal(*artifact.Subject, desc) {
continue
}
node.ArtifactType = artifact.ArtifactType
node.Annotations = artifact.Annotations
case ocispec.MediaTypeImageManifest:
fetched, err := fetchBytes(ctx, target, node)
if err != nil {
return nil, err
}
var image ocispec.Manifest
if err := json.Unmarshal(fetched, &image); err != nil {
return nil, err
}
if image.Subject == nil || !content.Equal(*image.Subject, desc) {
continue
}
node.ArtifactType = image.Config.MediaType
node.Annotations = image.Annotations
default:
continue
}
if node.ArtifactType != "" && (artifactType == "" || artifactType == node.ArtifactType) {
results = append(results, node)
}
}
return results, nil
}

func fetchBytes(ctx context.Context, fetcher content.Fetcher, desc ocispec.Descriptor) ([]byte, error) {
rc, err := fetcher.Fetch(ctx, desc)
if err != nil {
return nil, err
}
defer rc.Close()
return content.ReadAll(rc, desc)
}
174 changes: 174 additions & 0 deletions internal/graph/graph_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,174 @@
/*
Copyright The ORAS Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package graph

import (
"bytes"
"context"
"encoding/json"
"errors"
"reflect"
"testing"

"github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"oras.land/oras-go/v2"
"oras.land/oras-go/v2/content/memory"
)

type errLister struct {
oras.ReadOnlyGraphTarget
}

func (e *errLister) Referrers(ctx context.Context, desc ocispec.Descriptor, artifactType string, fn func(referrers []ocispec.Descriptor) error) error {
return errors.New("")
}

type refLister struct {
referrers []ocispec.Descriptor
oras.ReadOnlyGraphTarget
}

func (m *refLister) Referrers(ctx context.Context, desc ocispec.Descriptor, artifactType string, fn func(referrers []ocispec.Descriptor) error) error {
return fn(m.referrers)
}

type predecessorFinder struct {
*memory.Store
}

func TestReferrers(t *testing.T) {
ctx := context.Background()
var blobs [][]byte
var descs []ocispec.Descriptor
appendBlob := func(mediaType string, blob []byte) {
blobs = append(blobs, blob)
descs = append(descs, ocispec.Descriptor{
MediaType: mediaType,
Digest: digest.FromBytes(blob),
Size: int64(len(blob)),
})
}
generateImage := func(subject *ocispec.Descriptor, annotations map[string]string, config ocispec.Descriptor, layers ...ocispec.Descriptor) {
manifest := ocispec.Manifest{
Subject: subject,
Config: config,
Layers: layers,
Annotations: annotations,
}
manifestJSON, err := json.Marshal(manifest)
if err != nil {
t.Fatal(err)
}
appendBlob(ocispec.MediaTypeImageManifest, manifestJSON)
}
generateArtifact := func(artifactType string, subject *ocispec.Descriptor, annotations map[string]string, blobs ...ocispec.Descriptor) {
manifest := ocispec.Artifact{
Subject: subject,
Blobs: blobs,
Annotations: annotations,
ArtifactType: artifactType,
}
manifestJSON, err := json.Marshal(manifest)
if err != nil {
t.Fatal(err)
}
appendBlob(ocispec.MediaTypeArtifactManifest, manifestJSON)
}
generateIndex := func(manifests ...ocispec.Descriptor) {
index := ocispec.Index{
Manifests: manifests,
}
manifestJSON, err := json.Marshal(index)
if err != nil {
t.Fatal(err)
}
appendBlob(ocispec.MediaTypeImageIndex, manifestJSON)
}
const (
subject = iota
imgConfig
image
artifact
index
)
anno := map[string]string{"test": "foo"}
appendBlob(ocispec.MediaTypeArtifactManifest, []byte("subject content"))
imageType := "test.image"
appendBlob(imageType, []byte("config content"))
generateImage(&descs[subject], anno, descs[imgConfig])
imageDesc := descs[image]
imageDesc.Annotations = anno
imageDesc.ArtifactType = imageType
artifactType := "test.artifact"
generateArtifact(artifactType, &descs[subject], anno)
generateIndex(descs[subject])
artifactDesc := descs[artifact]
artifactDesc.Annotations = anno
artifactDesc.ArtifactType = artifactType

referrers := []ocispec.Descriptor{descs[image], descs[image]}
memory := memory.New()
for i := range descs {
memory.Push(ctx, descs[i], bytes.NewReader(blobs[i]))
}
finder := &predecessorFinder{Store: memory}

type args struct {
ctx context.Context
target oras.ReadOnlyGraphTarget
desc ocispec.Descriptor
artifactType string
}
tests := []struct {
name string
args args
want []ocispec.Descriptor
wantErr bool
}{
{"should fail when a referrer lister failed to get referrers", args{ctx, &errLister{}, ocispec.Descriptor{}, ""}, nil, true},
{"should return referrers when target is a referrer lister", args{ctx, &refLister{referrers: referrers}, ocispec.Descriptor{}, ""}, referrers, false},
{"should return nil for index node", args{ctx, finder, descs[index], ""}, nil, false},
{"should return nil for config node", args{ctx, finder, descs[imgConfig], ""}, nil, false},
{"should find filtered image referrer", args{ctx, finder, descs[subject], imageType}, []ocispec.Descriptor{imageDesc}, false},
{"should find filtered artifact referrer", args{ctx, finder, descs[subject], artifactType}, []ocispec.Descriptor{artifactDesc}, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := Referrers(tt.args.ctx, tt.args.target, tt.args.desc, tt.args.artifactType)
if (err != nil) != tt.wantErr {
t.Errorf("Referrers() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("Referrers() = %v, want %v", got, tt.want)
}
})
}

t.Run("should find referrers in predecessors", func(t *testing.T) {
want1 := []ocispec.Descriptor{artifactDesc, imageDesc}
want2 := []ocispec.Descriptor{imageDesc, artifactDesc}
got, err := Referrers(ctx, finder, descs[subject], "")
if err != nil {
t.Errorf("Referrers() error = %v", err)
return
}
if !reflect.DeepEqual(got, want1) && !reflect.DeepEqual(got, want2) {
t.Errorf("Referrers() = %v, want %v", got, want1)
}
})
}

0 comments on commit dcfb2e2

Please sign in to comment.