Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

refactor: remove AutoRemoveReferrers flag #683

Merged
merged 2 commits into from
Jan 26, 2024
Merged

refactor: remove AutoRemoveReferrers flag #683

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6456913
feat: remove AutoRemoveReferrers flag
wangxiaoxuan273 Jan 19, 2024
78c40fd
resolved comments
wangxiaoxuan273 Jan 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 12 additions & 19 deletions content/oci/oci.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,19 +56,13 @@ type Store struct {
// - Default value: true.
AutoSaveIndex bool

// AutoGC controls if the OCI store will automatically clean newly produced
// dangling (unreferenced) blobs during Delete() operation. For example the
// blobs whose manifests have been deleted. Tagged manifests will not be
// deleted.
// AutoGC controls if the OCI store will automatically clean dangling
// (unreferenced) blobs created by the Delete() operation. This includes the
// referrers and the unreferenced successor blobs of the deleted content.
// Tagged manifests will not be deleted.
// - Default value: true.
AutoGC bool

// AutoDeleteReferrers controls if the OCI store will automatically delete its
// referrers when a manifest is deleted. When set to true, the referrers will
// be deleted even if they exist in index.json.
// - Default value: true.
AutoDeleteReferrers bool

root string
indexPath string
index *ocispec.Index
Expand Down Expand Up @@ -102,14 +96,13 @@ func NewWithContext(ctx context.Context, root string) (*Store, error) {
}

store := &Store{
AutoSaveIndex: true,
AutoGC: true,
AutoDeleteReferrers: true,
root: rootAbs,
indexPath: filepath.Join(rootAbs, ocispec.ImageIndexFile),
storage: storage,
tagResolver: resolver.NewMemory(),
graph: graph.NewMemory(),
AutoSaveIndex: true,
AutoGC: true,
root: rootAbs,
indexPath: filepath.Join(rootAbs, ocispec.ImageIndexFile),
storage: storage,
tagResolver: resolver.NewMemory(),
graph: graph.NewMemory(),
}

if err := ensureDir(filepath.Join(rootAbs, ocispec.ImageBlobsDir)); err != nil {
Expand Down Expand Up @@ -177,7 +170,7 @@ func (s *Store) Delete(ctx context.Context, target ocispec.Descriptor) error {
deleteQueue = deleteQueue[1:]

// get referrers if applicable
if s.AutoDeleteReferrers && descriptor.IsManifest(head) {
if s.AutoGC && descriptor.IsManifest(head) {
referrers, err := registry.Referrers(ctx, &unsafeStore{s}, head, "")
if err != nil {
return err
Expand Down
313 changes: 1 addition & 312 deletions content/oci/oci_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2263,7 +2263,7 @@ func TestStore_PredecessorsAndDelete(t *testing.T) {
}
}

func TestStore_DeleteWithAutoGCAndAutoRemoveReferrers(t *testing.T) {
func TestStore_DeleteWithAutoGC(t *testing.T) {
tempDir := t.TempDir()
s, err := New(tempDir)
if err != nil {
Expand Down Expand Up @@ -2417,317 +2417,6 @@ func TestStore_DeleteWithAutoGCAndAutoRemoveReferrers(t *testing.T) {
}
}

func TestStore_DeleteDisableAutoRemoveReferrers(t *testing.T) {
tempDir := t.TempDir()
s, err := New(tempDir)
if err != nil {
t.Fatal("New() error =", err)
}
s.AutoDeleteReferrers = false
ctx := context.Background()

// generate test content
var blobs [][]byte
var descs []ocispec.Descriptor
appendBlob := func(mediaType string, blob []byte) {
blobs = append(blobs, blob)
descs = append(descs, ocispec.Descriptor{
MediaType: mediaType,
Digest: digest.FromBytes(blob),
Size: int64(len(blob)),
})
}
generateManifest := func(config ocispec.Descriptor, subject *ocispec.Descriptor, layers ...ocispec.Descriptor) {
manifest := ocispec.Manifest{
Config: config,
Subject: subject,
Layers: layers,
}
manifestJSON, err := json.Marshal(manifest)
if err != nil {
t.Fatal(err)
}
appendBlob(ocispec.MediaTypeImageManifest, manifestJSON)
}
generateIndex := func(manifests ...ocispec.Descriptor) {
index := ocispec.Index{
Manifests: manifests,
}
indexJSON, err := json.Marshal(index)
if err != nil {
t.Fatal(err)
}
appendBlob(ocispec.MediaTypeImageIndex, indexJSON)
}

appendBlob(ocispec.MediaTypeImageConfig, []byte("config")) // Blob 0
appendBlob(ocispec.MediaTypeImageLayer, []byte("foo")) // Blob 1
appendBlob(ocispec.MediaTypeImageLayer, []byte("bar")) // Blob 2
appendBlob(ocispec.MediaTypeImageLayer, []byte("hello")) // Blob 3
generateManifest(descs[0], nil, descs[1]) // Blob 4
generateManifest(descs[0], nil, descs[2]) // Blob 5
generateManifest(descs[0], nil, descs[3]) // Blob 6
generateIndex(descs[4:6]...) // Blob 7
generateIndex(descs[6]) // Blob 8
appendBlob(ocispec.MediaTypeImageLayer, []byte("world")) // Blob 9
generateManifest(descs[0], &descs[6], descs[9]) // Blob 10
generateManifest(descs[0], &descs[10], descs[2]) // Blob 11

eg, egCtx := errgroup.WithContext(ctx)
for i := range blobs {
eg.Go(func(i int) func() error {
return func() error {
err := s.Push(egCtx, descs[i], bytes.NewReader(blobs[i]))
if err != nil {
return fmt.Errorf("failed to push test content to src: %d: %v", i, err)
}
return nil
}
}(i))
}
if err := eg.Wait(); err != nil {
t.Fatal(err)
}

// delete blob 4 and verify the result
if err := s.Delete(egCtx, descs[4]); err != nil {
t.Fatal(err)
}

// blob 1 and 4 are now deleted, and other blobs are still present
notPresent := []ocispec.Descriptor{descs[1], descs[4]}
for _, node := range notPresent {
if exists, _ := s.Exists(egCtx, node); exists {
t.Errorf("%v should not exist in store", node)
}
}
stillPresent := []ocispec.Descriptor{descs[0], descs[2], descs[3], descs[5], descs[6], descs[7], descs[8], descs[9], descs[10], descs[11]}
for _, node := range stillPresent {
if exists, _ := s.Exists(egCtx, node); !exists {
t.Errorf("%v should exist in store", node)
}
}

// delete blob 8 and verify the result
if err := s.Delete(egCtx, descs[8]); err != nil {
t.Fatal(err)
}

// blob 1, 4 and 8 are now deleted, and other blobs are still present
notPresent = []ocispec.Descriptor{descs[1], descs[4], descs[8]}
for _, node := range notPresent {
if exists, _ := s.Exists(egCtx, node); exists {
t.Errorf("%v should not exist in store", node)
}
}
stillPresent = []ocispec.Descriptor{descs[0], descs[2], descs[3], descs[5], descs[6], descs[7], descs[9], descs[10], descs[11]}
for _, node := range stillPresent {
if exists, _ := s.Exists(egCtx, node); !exists {
t.Errorf("%v should exist in store", node)
}
}

// delete blob 6 and verify the result
if err := s.Delete(egCtx, descs[6]); err != nil {
t.Fatal(err)
}

// blob 1, 3, 4, 6, 8 are now deleted, and other blobs are still present
notPresent = []ocispec.Descriptor{descs[1], descs[3], descs[4], descs[6], descs[8]}
for _, node := range notPresent {
if exists, _ := s.Exists(egCtx, node); exists {
t.Errorf("%v should not exist in store", node)
}
}
stillPresent = []ocispec.Descriptor{descs[0], descs[2], descs[5], descs[7], descs[9], descs[10], descs[11]}
for _, node := range stillPresent {
if exists, _ := s.Exists(egCtx, node); !exists {
t.Errorf("%v should exist in store", node)
}
}

// verify predecessors information
wants := [][]ocispec.Descriptor{
{descs[5], descs[10], descs[11]}, // Blob 0
nil, // Blob 1
{descs[5], descs[11]}, // Blob 2
nil, // Blob 3
{descs[7]}, // Blob 4's predecessor is descs[7], even though blob 4 no longer exist
{descs[7]}, // Blob 5
{descs[10]}, // Blob 6's predecessor is descs[10], even though blob 6 no longer exist
nil, // Blob 7
nil, // Blob 8
{descs[10]}, // Blob 9
{descs[11]}, // Blob 10
nil, // Blob 11
}
for i, want := range wants {
predecessors, err := s.Predecessors(ctx, descs[i])
if err != nil {
t.Errorf("Store.Predecessors(%d) error = %v", i, err)
}
if !equalDescriptorSet(predecessors, want) {
t.Errorf("Store.Predecessors(%d) = %v, want %v", i, predecessors, want)
}
}
}

func TestStore_DeleteDisableAutoGCAndAutoRemoveReferrers(t *testing.T) {
tempDir := t.TempDir()
s, err := New(tempDir)
if err != nil {
t.Fatal("New() error =", err)
}
s.AutoDeleteReferrers = false
s.AutoGC = false
ctx := context.Background()

// generate test content
var blobs [][]byte
var descs []ocispec.Descriptor
appendBlob := func(mediaType string, blob []byte) {
blobs = append(blobs, blob)
descs = append(descs, ocispec.Descriptor{
MediaType: mediaType,
Digest: digest.FromBytes(blob),
Size: int64(len(blob)),
})
}
generateManifest := func(config ocispec.Descriptor, subject *ocispec.Descriptor, layers ...ocispec.Descriptor) {
manifest := ocispec.Manifest{
Config: config,
Subject: subject,
Layers: layers,
}
manifestJSON, err := json.Marshal(manifest)
if err != nil {
t.Fatal(err)
}
appendBlob(ocispec.MediaTypeImageManifest, manifestJSON)
}
generateIndex := func(manifests ...ocispec.Descriptor) {
index := ocispec.Index{
Manifests: manifests,
}
indexJSON, err := json.Marshal(index)
if err != nil {
t.Fatal(err)
}
appendBlob(ocispec.MediaTypeImageIndex, indexJSON)
}

appendBlob(ocispec.MediaTypeImageConfig, []byte("config")) // Blob 0
appendBlob(ocispec.MediaTypeImageLayer, []byte("foo")) // Blob 1
appendBlob(ocispec.MediaTypeImageLayer, []byte("bar")) // Blob 2
appendBlob(ocispec.MediaTypeImageLayer, []byte("hello")) // Blob 3
generateManifest(descs[0], nil, descs[1]) // Blob 4
generateManifest(descs[0], nil, descs[2]) // Blob 5
generateManifest(descs[0], nil, descs[3]) // Blob 6
generateIndex(descs[4:6]...) // Blob 7
generateIndex(descs[6]) // Blob 8
appendBlob(ocispec.MediaTypeImageLayer, []byte("world")) // Blob 9
generateManifest(descs[0], &descs[6], descs[9]) // Blob 10
generateManifest(descs[0], &descs[10], descs[2]) // Blob 11

eg, egCtx := errgroup.WithContext(ctx)
for i := range blobs {
eg.Go(func(i int) func() error {
return func() error {
err := s.Push(egCtx, descs[i], bytes.NewReader(blobs[i]))
if err != nil {
return fmt.Errorf("failed to push test content to src: %d: %v", i, err)
}
return nil
}
}(i))
}
if err := eg.Wait(); err != nil {
t.Fatal(err)
}

// delete blob 4 and verify the result
if err := s.Delete(egCtx, descs[4]); err != nil {
t.Fatal(err)
}

// blob 4 is now deleted, and other blobs are still present
notPresent := []ocispec.Descriptor{descs[4]}
for _, node := range notPresent {
if exists, _ := s.Exists(egCtx, node); exists {
t.Errorf("%v should not exist in store", node)
}
}
stillPresent := []ocispec.Descriptor{descs[0], descs[1], descs[2], descs[3], descs[5], descs[6], descs[7], descs[8], descs[9], descs[10], descs[11]}
for _, node := range stillPresent {
if exists, _ := s.Exists(egCtx, node); !exists {
t.Errorf("%v should exist in store", node)
}
}

// delete blob 8 and verify the result
if err := s.Delete(egCtx, descs[8]); err != nil {
t.Fatal(err)
}

// blob 4 and 8 are now deleted, and other blobs are still present
notPresent = []ocispec.Descriptor{descs[4], descs[8]}
for _, node := range notPresent {
if exists, _ := s.Exists(egCtx, node); exists {
t.Errorf("%v should not exist in store", node)
}
}
stillPresent = []ocispec.Descriptor{descs[0], descs[1], descs[2], descs[3], descs[5], descs[6], descs[7], descs[9], descs[10], descs[11]}
for _, node := range stillPresent {
if exists, _ := s.Exists(egCtx, node); !exists {
t.Errorf("%v should exist in store", node)
}
}

// delete blob 6 and verify the result
if err := s.Delete(egCtx, descs[6]); err != nil {
t.Fatal(err)
}

// blob 4, 6, 8 are now deleted, and other blobs are still present
notPresent = []ocispec.Descriptor{descs[4], descs[6], descs[8]}
for _, node := range notPresent {
if exists, _ := s.Exists(egCtx, node); exists {
t.Errorf("%v should not exist in store", node)
}
}
stillPresent = []ocispec.Descriptor{descs[0], descs[1], descs[2], descs[3], descs[5], descs[7], descs[9], descs[10], descs[11]}
for _, node := range stillPresent {
if exists, _ := s.Exists(egCtx, node); !exists {
t.Errorf("%v should exist in store", node)
}
}

// verify predecessors information
wants := [][]ocispec.Descriptor{
{descs[5], descs[10], descs[11]}, // Blob 0
nil, // Blob 1
{descs[5], descs[11]}, // Blob 2
nil, // Blob 3
{descs[7]}, // Blob 4's predecessor is descs[7], even though blob 4 no longer exist
{descs[7]}, // Blob 5
{descs[10]}, // Blob 6's predecessor is descs[10], even though blob 6 no longer exist
nil, // Blob 7
nil, // Blob 8
{descs[10]}, // Blob 9
{descs[11]}, // Blob 10
nil, // Blob 11
}
for i, want := range wants {
predecessors, err := s.Predecessors(ctx, descs[i])
if err != nil {
t.Errorf("Store.Predecessors(%d) error = %v", i, err)
}
if !equalDescriptorSet(predecessors, want) {
t.Errorf("Store.Predecessors(%d) = %v, want %v", i, predecessors, want)
}
}
}

func TestStore_Untag(t *testing.T) {
content := []byte("test delete")
desc := ocispec.Descriptor{
Expand Down
Loading