Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: support artifact discovering in OCI image layout #765

Merged
merged 20 commits into from
Jan 29, 2023
Merged

feat: support artifact discovering in OCI image layout #765

Changes from 11 commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
9d2cc94
feat: support discover
qweeah Jan 28, 2023
e67be05
add index support in OCI image layout
qweeah Jan 28, 2023
14abb31
align discover experience to remote registry
qweeah Jan 28, 2023
6c10352
bug fix
qweeah Jan 28, 2023
19014c4
bug fix
qweeah Jan 28, 2023
b1d4c02
add example
qweeah Jan 28, 2023
f0fc6b8
add error case
qweeah Jan 28, 2023
bf77ecc
fix build error
qweeah Jan 28, 2023
2454cdd
Revert "fix build error"
qweeah Jan 28, 2023
032e7aa
Revert "add error case"
qweeah Jan 28, 2023
875f179
code clean
qweeah Jan 28, 2023
c36fd71
Merge remote-tracking branch 'origin_src/main' into oci-discover
qweeah Jan 29, 2023
01d1f05
resolve comments
qweeah Jan 29, 2023
eef812a
move referrers func to a dedicated package
qweeah Jan 29, 2023
560c390
add unit tests
qweeah Jan 29, 2023
da80966
add unit test
qweeah Jan 29, 2023
b5a3112
add unit test
qweeah Jan 29, 2023
243fa5b
add unit test with memory store
qweeah Jan 29, 2023
764310a
code clean
qweeah Jan 29, 2023
3271b20
resolve comments
qweeah Jan 29, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 63 additions & 20 deletions cmd/oras/discover.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@ import (

"gopkg.in/yaml.v3"
"oras.land/oras-go/v2"
"oras.land/oras-go/v2/content"
"oras.land/oras-go/v2/registry/remote"
"oras.land/oras/cmd/oras/internal/errors"
"oras.land/oras/cmd/oras/internal/option"

"github.com/need-being/go-tree"
Expand All @@ -36,10 +36,9 @@ import (

type discoverOptions struct {
option.Common
option.Remote
option.Platform
option.Target

targetRef string
artifactType string
outputType string
}
Expand Down Expand Up @@ -70,13 +69,17 @@ Example - Discover all the referrers of manifest with annotations, displayed in

Example - Discover referrers with type 'test-artifact' of manifest 'hello:v1' in registry 'localhost:5000':
oras discover --artifact-type test-artifact localhost:5000/hello:v1

Example - Discover referrers of the manifest tagged 'v1' in an OCI layout folder 'layout-dir':
oras discover layout-dir:v1
oras discover -v -o tree layout-dir:v1
qweeah marked this conversation as resolved.
Show resolved Hide resolved
`,
Args: cobra.ExactArgs(1),
PreRunE: func(cmd *cobra.Command, args []string) error {
opts.RawReference = args[0]
return option.Parse(&opts)
},
RunE: func(cmd *cobra.Command, args []string) error {
opts.targetRef = args[0]
return runDiscover(opts)
},
}
Expand All @@ -90,24 +93,24 @@ Example - Discover referrers with type 'test-artifact' of manifest 'hello:v1' in

func runDiscover(opts discoverOptions) error {
ctx, _ := opts.SetLoggerLevel()
repo, err := opts.NewRepository(opts.targetRef, opts.Common)
repo, err := opts.NewReadonlyTarget(ctx, opts.Common)
if err != nil {
return err
}
if repo.Reference.Reference == "" {
return errors.NewErrInvalidReference(repo.Reference)
if err := opts.EnsureReferenceNotEmpty(); err != nil {
return err
}

// discover artifacts
resolveOpts := oras.DefaultResolveOptions
resolveOpts.TargetPlatform = opts.Platform.Platform
desc, err := oras.Resolve(ctx, repo, repo.Reference.Reference, resolveOpts)
desc, err := oras.Resolve(ctx, repo, opts.Reference, resolveOpts)
if err != nil {
return err
}

if opts.outputType == "tree" {
root := tree.New(repo.Reference.String())
root := tree.New(opts.Reference)
err = fetchAllReferrers(ctx, repo, desc, opts.artifactType, root, &opts)
if err != nil {
return err
Expand All @@ -124,9 +127,9 @@ func runDiscover(opts discoverOptions) error {
}

if n := len(refs); n > 1 {
fmt.Println("Discovered", n, "artifacts referencing", repo.Reference)
fmt.Println("Discovered", n, "artifacts referencing", opts.Reference)
} else {
fmt.Println("Discovered", n, "artifact referencing", repo.Reference)
fmt.Println("Discovered", n, "artifact referencing", opts.Reference)
}
fmt.Println("Digest:", desc.Digest)
if len(refs) > 0 {
Expand All @@ -136,19 +139,59 @@ func runDiscover(opts discoverOptions) error {
return nil
}

func fetchReferrers(ctx context.Context, repo *remote.Repository, desc ocispec.Descriptor, artifactType string) ([]ocispec.Descriptor, error) {
results := []ocispec.Descriptor{}
err := repo.Referrers(ctx, desc, artifactType, func(referrers []ocispec.Descriptor) error {
results = append(results, referrers...)
return nil
})
if err != nil {
return nil, err
func fetchReferrers(ctx context.Context, target oras.ReadOnlyGraphTarget, desc ocispec.Descriptor, artifactType string) ([]ocispec.Descriptor, error) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we put this function into a package so that we can have unit tests?

var results []ocispec.Descriptor
if repo, ok := target.(*remote.Repository); ok {
qweeah marked this conversation as resolved.
Show resolved Hide resolved
// get referrers directly
err := repo.Referrers(ctx, desc, artifactType, func(referrers []ocispec.Descriptor) error {
results = append(results, referrers...)
return nil
})
if err != nil {
return nil, err
}
} else {
// find matched referrers in all predecessors
predecessors, err := target.Predecessors(ctx, desc)
qweeah marked this conversation as resolved.
Show resolved Hide resolved
if err != nil {
return nil, err
}
for _, node := range predecessors {
var fetched []byte
if rc, err := target.Fetch(ctx, node); err != nil {
qweeah marked this conversation as resolved.
Show resolved Hide resolved
return nil, err
} else {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The else enclosing can be deleted.

Copy link
Contributor Author

@qweeah qweeah Jan 29, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If else is deleted, then rc will be undefined.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It reminds me that I can actually use https://pkg.go.dev/oras.land/oras-go/v2#FetchBytes to avoid reading from rc myself

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

rc is not deferred to be closed.

fetched, err = content.ReadAll(rc, node)
if err != nil {
return nil, err
}
}
qweeah marked this conversation as resolved.
Show resolved Hide resolved
var isMatch = func(got *ocispec.Descriptor, want ocispec.Descriptor) bool {
return got != nil && content.Equal(*got, want)
}
qweeah marked this conversation as resolved.
Show resolved Hide resolved
switch node.MediaType {
case ocispec.MediaTypeArtifactManifest:
var artifact ocispec.Artifact
json.Unmarshal(fetched, &artifact)
qweeah marked this conversation as resolved.
Show resolved Hide resolved
if isMatch(artifact.Subject, desc) {
node.ArtifactType = artifact.ArtifactType
}
case ocispec.MediaTypeImageManifest:
var image ocispec.Manifest
json.Unmarshal(fetched, &image)
qweeah marked this conversation as resolved.
Show resolved Hide resolved
if isMatch(image.Subject, desc) {
node.ArtifactType = image.Config.MediaType
}
qweeah marked this conversation as resolved.
Show resolved Hide resolved
}
if node.ArtifactType != "" && (artifactType == "" || artifactType == node.ArtifactType) {
results = append(results, node)
}
}
}
return results, nil
qweeah marked this conversation as resolved.
Show resolved Hide resolved
}

func fetchAllReferrers(ctx context.Context, repo *remote.Repository, desc ocispec.Descriptor, artifactType string, node *tree.Node, opts *discoverOptions) error {
func fetchAllReferrers(ctx context.Context, repo oras.ReadOnlyGraphTarget, desc ocispec.Descriptor, artifactType string, node *tree.Node, opts *discoverOptions) error {
results, err := fetchReferrers(ctx, repo, desc, artifactType)
if err != nil {
return err
Expand Down