Skip to content

Commit

Permalink
fix: fix bug when copying recursively with certain platforms (#818)
Browse files Browse the repository at this point in the history 
Signed-off-by: Billy Zha <jinzha1@microsoft.com>
  • Loading branch information
@qweeah
qweeah committed Feb 23, 2023
1 parent 2d89bc8 commit e8bc5ac
Show file tree
Hide file tree
Showing 19 changed files with 684 additions and 247 deletions.
27 changes: 22 additions & 5 deletions cmd/oras/cp.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,13 @@ import (
"strings"
"sync"

"github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/spf13/cobra"
"oras.land/oras-go/v2"
"oras.land/oras/cmd/oras/internal/display"
"oras.land/oras/cmd/oras/internal/option"
"oras.land/oras/internal/graph"
)

type copyOptions struct {
Expand Down Expand Up @@ -117,6 +119,7 @@ func runCopy(opts copyOptions) error {
committed := &sync.Map{}
extendedCopyOptions := oras.DefaultExtendedCopyOptions
extendedCopyOptions.Concurrency = opts.concurrency
extendedCopyOptions.FindPredecessors = graph.FindReferrerPredecessors
extendedCopyOptions.PreCopy = display.StatusPrinter("Copying", opts.Verbose)
extendedCopyOptions.PostCopy = func(ctx context.Context, desc ocispec.Descriptor) error {
committed.Store(desc.Digest.String(), desc.Annotations[ocispec.AnnotationTitle])
Expand All @@ -131,9 +134,10 @@ func runCopy(opts copyOptions) error {
}

var desc ocispec.Descriptor
if ref := opts.To.Reference; ref == "" {
// push to the destination with digest only if no tag specified
desc, err = src.Resolve(ctx, opts.From.Reference)
rOpts := oras.DefaultResolveOptions
rOpts.TargetPlatform = opts.Platform.Platform
if dstRef := opts.To.Reference; dstRef == "" {
desc, err = oras.Resolve(ctx, src, opts.From.Reference, rOpts)
if err != nil {
return err
}
Expand All @@ -144,21 +148,34 @@ func runCopy(opts copyOptions) error {
}
} else {
if opts.recursive {
desc, err = oras.ExtendedCopy(ctx, src, opts.From.Reference, dst, opts.To.Reference, extendedCopyOptions)
srcRef := opts.From.Reference
if rOpts.TargetPlatform != nil {
// resolve source reference to specified platform
desc, err := oras.Resolve(ctx, src, opts.From.Reference, rOpts)
if err != nil {
return err
}
srcRef = desc.Digest.String()
}
desc, err = oras.ExtendedCopy(ctx, src, srcRef, dst, dstRef, extendedCopyOptions)
} else {
copyOptions := oras.CopyOptions{
CopyGraphOptions: extendedCopyOptions.CopyGraphOptions,
}
if opts.Platform.Platform != nil {
copyOptions.WithTargetPlatform(opts.Platform.Platform)
}
desc, err = oras.Copy(ctx, src, opts.From.Reference, dst, opts.To.Reference, copyOptions)
desc, err = oras.Copy(ctx, src, opts.From.Reference, dst, dstRef, copyOptions)
}
}
if err != nil {
return err
}

if from, err := digest.Parse(opts.From.Reference); err == nil && from != desc.Digest {
// correct source digest
opts.From.RawReference = fmt.Sprintf("%s@%s", opts.From.Path, desc.Digest.String())
}
fmt.Println("Copied", opts.From.AnnotatedReference(), "=>", opts.To.AnnotatedReference())

if len(opts.extraRefs) != 0 {
Expand Down
30 changes: 28 additions & 2 deletions internal/graph/graph.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ import (
"encoding/json"

ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"oras.land/oras-go/v2"
"oras.land/oras-go/v2/content"
"oras.land/oras-go/v2/registry"
"oras.land/oras/internal/docker"
Expand Down Expand Up @@ -63,7 +62,7 @@ func Successors(ctx context.Context, fetcher content.Fetcher, node ocispec.Descr
}

// Referrers returns referrer nodes of desc in target.
func Referrers(ctx context.Context, target oras.ReadOnlyGraphTarget, desc ocispec.Descriptor, artifactType string) ([]ocispec.Descriptor, error) {
func Referrers(ctx context.Context, target content.ReadOnlyGraphStorage, desc ocispec.Descriptor, artifactType string) ([]ocispec.Descriptor, error) {
var results []ocispec.Descriptor
if repo, ok := target.(registry.ReferrerLister); ok {
// get referrers directly
Expand Down Expand Up @@ -122,6 +121,33 @@ func Referrers(ctx context.Context, target oras.ReadOnlyGraphTarget, desc ocispe
return results, nil
}

// FindReferrerPredecessors returns referrer nodes of desc in target.
func FindReferrerPredecessors(ctx context.Context, src content.ReadOnlyGraphStorage, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
var results []ocispec.Descriptor
if repo, ok := src.(registry.ReferrerLister); ok {
// get referrers directly
err := repo.Referrers(ctx, desc, "", func(referrers []ocispec.Descriptor) error {
results = append(results, referrers...)
return nil
})
if err != nil {
return nil, err
}
return results, nil
}
predecessors, err := src.Predecessors(ctx, desc)
if err != nil {
return nil, err
}
for _, node := range predecessors {
switch node.MediaType {
case ocispec.MediaTypeArtifactManifest, ocispec.MediaTypeImageManifest:
results = append(results, node)
}
}
return results, nil
}

func fetchBytes(ctx context.Context, fetcher content.Fetcher, desc ocispec.Descriptor) ([]byte, error) {
rc, err := fetcher.Fetch(ctx, desc)
if err != nil {
Expand Down
231 changes: 231 additions & 0 deletions internal/graph/graph_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,9 @@ import (
"github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"oras.land/oras-go/v2"
"oras.land/oras-go/v2/content"
"oras.land/oras-go/v2/content/memory"
"oras.land/oras/internal/docker"
)

type errLister struct {
Expand All @@ -37,6 +39,14 @@ func (e *errLister) Referrers(ctx context.Context, desc ocispec.Descriptor, arti
return errors.New("")
}

type errFinder struct {
oras.ReadOnlyGraphTarget
}

func (e *errFinder) Predecessors(ctx context.Context, node ocispec.Descriptor) ([]ocispec.Descriptor, error) {
return nil, errors.New("")
}

type refLister struct {
referrers []ocispec.Descriptor
oras.ReadOnlyGraphTarget
Expand All @@ -50,6 +60,10 @@ type predecessorFinder struct {
*memory.Store
}

type fetcher struct {
content.Fetcher
}

func TestReferrers(t *testing.T) {
ctx := context.Background()
var blobs [][]byte
Expand Down Expand Up @@ -172,3 +186,220 @@ func TestReferrers(t *testing.T) {
}
})
}

func TestSuccessors(t *testing.T) {
var blobs [][]byte
var descs []ocispec.Descriptor
appendBlob := func(mediaType string, blob []byte) {
blobs = append(blobs, blob)
descs = append(descs, ocispec.Descriptor{
MediaType: mediaType,
Digest: digest.FromBytes(blob),
Size: int64(len(blob)),
})
}
generateImage := func(subject *ocispec.Descriptor, mediaType string, config ocispec.Descriptor, layers ...ocispec.Descriptor) {
manifest := ocispec.Manifest{
MediaType: mediaType,
Subject: subject,
Config: config,
Layers: layers,
}
manifestJSON, err := json.Marshal(manifest)
if err != nil {
t.Fatal(err)
}
appendBlob(mediaType, manifestJSON)
}
generateArtifact := func(artifactType string, subject *ocispec.Descriptor, blobs ...ocispec.Descriptor) {
manifest := ocispec.Artifact{
MediaType: ocispec.MediaTypeArtifactManifest,
Subject: subject,
Blobs: blobs,
}
manifestJSON, err := json.Marshal(manifest)
if err != nil {
t.Fatal(err)
}
appendBlob(ocispec.MediaTypeArtifactManifest, manifestJSON)
}
generateIndex := func(manifests ...ocispec.Descriptor) {
index := ocispec.Index{
Manifests: manifests,
}
manifestJSON, err := json.Marshal(index)
if err != nil {
t.Fatal(err)
}
appendBlob(ocispec.MediaTypeImageIndex, manifestJSON)
}
const (
subject = iota
config
ociImage
dockerImage
artifact
index
)
appendBlob(ocispec.MediaTypeArtifactManifest, []byte("subject content"))
imageType := "test.image"
appendBlob(imageType, []byte("config content"))
generateImage(&descs[subject], ocispec.MediaTypeImageManifest, descs[config])
generateImage(&descs[subject], docker.MediaTypeManifest, descs[config])
artifactType := "test.artifact"
generateArtifact(artifactType, &descs[subject])
generateIndex(descs[subject])
memory := memory.New()
ctx := context.Background()
for i := range descs {
memory.Push(ctx, descs[i], bytes.NewReader(blobs[i]))
}
fetcher := &fetcher{Fetcher: memory}

type args struct {
ctx context.Context
fetcher content.Fetcher
node ocispec.Descriptor
}
tests := []struct {
name string
args args
wantNodes []ocispec.Descriptor
wantSubject *ocispec.Descriptor
wantConfig *ocispec.Descriptor
wantErr bool
}{
{"should failed to get non-existent artifact", args{ctx, fetcher, ocispec.Descriptor{MediaType: ocispec.MediaTypeArtifactManifest}}, nil, nil, nil, true},
{"should failed to get non-existent OCI image", args{ctx, fetcher, ocispec.Descriptor{MediaType: ocispec.MediaTypeImageManifest}}, nil, nil, nil, true},
{"should failed to get non-existent docker image", args{ctx, fetcher, ocispec.Descriptor{MediaType: docker.MediaTypeManifest}}, nil, nil, nil, true},
{"should get success of a docker image", args{ctx, fetcher, descs[dockerImage]}, nil, &descs[subject], &descs[config], false},
{"should get success of an OCI image", args{ctx, fetcher, descs[ociImage]}, nil, &descs[subject], &descs[config], false},
{"should get success of an artifact", args{ctx, fetcher, descs[artifact]}, nil, &descs[subject], nil, false},
{"should get success of an index", args{ctx, fetcher, descs[index]}, []ocispec.Descriptor{descs[subject]}, nil, nil, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gotNodes, gotSubject, gotConfig, err := Successors(tt.args.ctx, tt.args.fetcher, tt.args.node)
if (err != nil) != tt.wantErr {
t.Errorf("Successors() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(gotNodes, tt.wantNodes) {
t.Errorf("Successors() gotNodes = %v, want %v", gotNodes, tt.wantNodes)
}
if !reflect.DeepEqual(gotSubject, tt.wantSubject) {
t.Errorf("Successors() gotSubject = %v, want %v", gotSubject, tt.wantSubject)
}
if !reflect.DeepEqual(gotConfig, tt.wantConfig) {
t.Errorf("Successors() gotConfig = %v, want %v", gotConfig, tt.wantConfig)
}
})
}
}

func TestFindReferrerPredecessors(t *testing.T) {
ctx := context.Background()
var blobs [][]byte
var descs []ocispec.Descriptor
appendBlob := func(mediaType string, blob []byte) {
blobs = append(blobs, blob)
descs = append(descs, ocispec.Descriptor{
MediaType: mediaType,
Digest: digest.FromBytes(blob),
Size: int64(len(blob)),
})
}
generateImage := func(subject *ocispec.Descriptor, annotations map[string]string, config ocispec.Descriptor, layers ...ocispec.Descriptor) {
manifest := ocispec.Manifest{
Subject: subject,
Config: config,
Layers: layers,
Annotations: annotations,
}
manifestJSON, err := json.Marshal(manifest)
if err != nil {
t.Fatal(err)
}
appendBlob(ocispec.MediaTypeImageManifest, manifestJSON)
}
generateArtifact := func(artifactType string, subject *ocispec.Descriptor, annotations map[string]string, blobs ...ocispec.Descriptor) {
manifest := ocispec.Artifact{
Subject: subject,
Blobs: blobs,
Annotations: annotations,
ArtifactType: artifactType,
}
manifestJSON, err := json.Marshal(manifest)
if err != nil {
t.Fatal(err)
}
appendBlob(ocispec.MediaTypeArtifactManifest, manifestJSON)
}
generateIndex := func(manifests ...ocispec.Descriptor) {
index := ocispec.Index{
Manifests: manifests,
}
manifestJSON, err := json.Marshal(index)
if err != nil {
t.Fatal(err)
}
appendBlob(ocispec.MediaTypeImageIndex, manifestJSON)
}
const (
subject = iota
imgConfig
image
artifact
)
var anno map[string]string
appendBlob(ocispec.MediaTypeArtifactManifest, []byte("subject content"))
imageType := "test.image"
appendBlob(imageType, []byte("config content"))
generateImage(&descs[subject], anno, descs[imgConfig])
imageDesc := descs[image]
imageDesc.Annotations = anno
imageDesc.ArtifactType = imageType
artifactType := "test.artifact"
generateArtifact(artifactType, &descs[subject], anno)
generateIndex(descs[subject])
artifactDesc := descs[artifact]
artifactDesc.Annotations = anno
artifactDesc.ArtifactType = artifactType

referrers := []ocispec.Descriptor{descs[image], descs[image]}
memory := memory.New()
for i := range descs {
memory.Push(ctx, descs[i], bytes.NewReader(blobs[i]))
}
finder := &predecessorFinder{Store: memory}
type args struct {
ctx context.Context
src content.ReadOnlyGraphStorage
desc ocispec.Descriptor
}
tests := []struct {
name string
args args
want []ocispec.Descriptor
wantErr bool
}{

{"should failed to get referrers", args{ctx, &errLister{}, ocispec.Descriptor{}}, nil, true},
{"should failed to get predecessor", args{ctx, &errFinder{}, ocispec.Descriptor{}}, nil, true},
{"should return referrers when target is a referrer lister", args{ctx, &refLister{referrers: referrers}, ocispec.Descriptor{}}, referrers, false},
{"should return image for config node", args{ctx, finder, descs[imgConfig]}, []ocispec.Descriptor{descs[image]}, false},
{"should return image and artifact for subject node", args{ctx, finder, descs[subject]}, []ocispec.Descriptor{descs[image], descs[artifact]}, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := FindReferrerPredecessors(tt.args.ctx, tt.args.src, tt.args.desc)
if (err != nil) != tt.wantErr {
t.Errorf("FindReferrerPredecessors() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("FindReferrerPredecessors() = %v, want %v", got, tt.want)
}
})
}
}
9 changes: 9 additions & 0 deletions test/e2e/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,15 @@ graph TD;
D1["test.sbom.file(image)"] -- subject --> C1
D2["test.signature.file(image)"] -- subject --> D1
end
subgraph "file: artifacts_index.tar.gz"
direction TB
F0>tag: multi]-..->F1[oci index]
F1--linux/amd64-->F2[oci image]
F1--linux/arm64-->F3[oci image]
F1--linux/arm/v7-->F4[oci image]
G1["referrer.index(image)"] -- subject --> F1
G2["referrer.image(image)"] -- subject --> F2
end
end
```

Expand Down
Loading

0 comments on commit e8bc5ac

Please sign in to comment.