Skip to content

Commit 3510d7e

Browse files
KN4CK3RKN4CK3R
authored
Fix container blob mount (#22226)
1 parent 1aba53d commit 3510d7e

File tree

5 files changed

+120
-71
lines changed

5 files changed

+120
-71
lines changed

‎models/packages/container/search.go

+10
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ type BlobSearchOptions struct {
2525
Digest string
2626
Tag string
2727
IsManifest bool
28+
Repository string
2829
}
2930

3031
func (opts *BlobSearchOptions) toConds() builder.Cond {
@@ -53,6 +54,15 @@ func (opts *BlobSearchOptions) toConds() builder.Cond {
5354

5455
cond = cond.And(builder.In("package_file.id", builder.Select("package_property.ref_id").Where(propsCond).From("package_property")))
5556
}
57+
if opts.Repository != "" {
58+
var propsCond builder.Cond = builder.Eq{
59+
"package_property.ref_type": packages.PropertyTypePackage,
60+
"package_property.name": container_module.PropertyRepository,
61+
"package_property.value": opts.Repository,
62+
}
63+
64+
cond = cond.And(builder.In("package.id", builder.Select("package_property.ref_id").Where(propsCond).From("package_property")))
65+
}
5666

5767
return cond
5868
}

‎routers/api/packages/container/blob.go

+76-53
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,60 @@ func saveAsPackageBlob(hsr packages_module.HashedSizeReader, pi *packages_servic
3333

3434
contentStore := packages_module.NewContentStore()
3535

36+
uploadVersion, err := getOrCreateUploadVersion(pi)
37+
if err != nil {
38+
return nil, err
39+
}
40+
41+
err = db.WithTx(db.DefaultContext, func(ctx context.Context) error {
42+
pb, exists, err = packages_model.GetOrInsertBlob(ctx, pb)
43+
if err != nil {
44+
log.Error("Error inserting package blob: %v", err)
45+
return err
46+
}
47+
// FIXME: Workaround to be removed in v1.20
48+
// https://github.com/go-gitea/gitea/issues/19586
49+
if exists {
50+
err = contentStore.Has(packages_module.BlobHash256Key(pb.HashSHA256))
51+
if err != nil && (errors.Is(err, util.ErrNotExist) || errors.Is(err, os.ErrNotExist)) {
52+
log.Debug("Package registry inconsistent: blob %s does not exist on file system", pb.HashSHA256)
53+
exists = false
54+
}
55+
}
56+
if !exists {
57+
if err := contentStore.Save(packages_module.BlobHash256Key(pb.HashSHA256), hsr, hsr.Size()); err != nil {
58+
log.Error("Error saving package blob in content store: %v", err)
59+
return err
60+
}
61+
}
62+
63+
return createFileForBlob(ctx, uploadVersion, pb)
64+
})
65+
if err != nil {
66+
if !exists {
67+
if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil {
68+
log.Error("Error deleting package blob from content store: %v", err)
69+
}
70+
}
71+
return nil, err
72+
}
73+
74+
return pb, nil
75+
}
76+
77+
// mountBlob mounts the specific blob to a different package
78+
func mountBlob(pi *packages_service.PackageInfo, pb *packages_model.PackageBlob) error {
79+
uploadVersion, err := getOrCreateUploadVersion(pi)
80+
if err != nil {
81+
return err
82+
}
83+
84+
return db.WithTx(db.DefaultContext, func(ctx context.Context) error {
85+
return createFileForBlob(ctx, uploadVersion, pb)
86+
})
87+
}
88+
89+
func getOrCreateUploadVersion(pi *packages_service.PackageInfo) (*packages_model.PackageVersion, error) {
3690
var uploadVersion *packages_model.PackageVersion
3791

3892
// FIXME: Replace usage of mutex with database transaction
@@ -83,66 +137,35 @@ func saveAsPackageBlob(hsr packages_module.HashedSizeReader, pi *packages_servic
83137
return nil
84138
})
85139
uploadVersionMutex.Unlock()
86-
if err != nil {
87-
return nil, err
88-
}
89-
90-
err = db.WithTx(db.DefaultContext, func(ctx context.Context) error {
91-
pb, exists, err = packages_model.GetOrInsertBlob(ctx, pb)
92-
if err != nil {
93-
log.Error("Error inserting package blob: %v", err)
94-
return err
95-
}
96-
// FIXME: Workaround to be removed in v1.20
97-
// https://github.com/go-gitea/gitea/issues/19586
98-
if exists {
99-
err = contentStore.Has(packages_module.BlobHash256Key(pb.HashSHA256))
100-
if err != nil && (errors.Is(err, util.ErrNotExist) || errors.Is(err, os.ErrNotExist)) {
101-
log.Debug("Package registry inconsistent: blob %s does not exist on file system", pb.HashSHA256)
102-
exists = false
103-
}
104-
}
105-
if !exists {
106-
if err := contentStore.Save(packages_module.BlobHash256Key(pb.HashSHA256), hsr, hsr.Size()); err != nil {
107-
log.Error("Error saving package blob in content store: %v", err)
108-
return err
109-
}
110-
}
111140

112-
filename := strings.ToLower(fmt.Sprintf("sha256_%s", pb.HashSHA256))
141+
return uploadVersion, err
142+
}
113143

114-
pf := &packages_model.PackageFile{
115-
VersionID: uploadVersion.ID,
116-
BlobID: pb.ID,
117-
Name: filename,
118-
LowerName: filename,
119-
CompositeKey: packages_model.EmptyFileKey,
120-
}
121-
if pf, err = packages_model.TryInsertFile(ctx, pf); err != nil {
122-
if err == packages_model.ErrDuplicatePackageFile {
123-
return nil
124-
}
125-
log.Error("Error inserting package file: %v", err)
126-
return err
127-
}
144+
func createFileForBlob(ctx context.Context, pv *packages_model.PackageVersion, pb *packages_model.PackageBlob) error {
145+
filename := strings.ToLower(fmt.Sprintf("sha256_%s", pb.HashSHA256))
128146

129-
if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeFile, pf.ID, container_module.PropertyDigest, digestFromPackageBlob(pb)); err != nil {
130-
log.Error("Error setting package file property: %v", err)
131-
return err
147+
pf := &packages_model.PackageFile{
148+
VersionID: pv.ID,
149+
BlobID: pb.ID,
150+
Name: filename,
151+
LowerName: filename,
152+
CompositeKey: packages_model.EmptyFileKey,
153+
}
154+
var err error
155+
if pf, err = packages_model.TryInsertFile(ctx, pf); err != nil {
156+
if err == packages_model.ErrDuplicatePackageFile {
157+
return nil
132158
}
159+
log.Error("Error inserting package file: %v", err)
160+
return err
161+
}
133162

134-
return nil
135-
})
136-
if err != nil {
137-
if !exists {
138-
if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil {
139-
log.Error("Error deleting package blob from content store: %v", err)
140-
}
141-
}
142-
return nil, err
163+
if _, err := packages_model.InsertProperty(ctx, packages_model.PropertyTypeFile, pf.ID, container_module.PropertyDigest, digestFromPackageBlob(pb)); err != nil {
164+
log.Error("Error setting package file property: %v", err)
165+
return err
143166
}
144167

145-
return pb, nil
168+
return nil
146169
}
147170

148171
func deleteBlob(ownerID int64, image, digest string) error {

‎routers/api/packages/container/container.go

+7-2
Original file line numberDiff line numberDiff line change
@@ -195,10 +195,15 @@ func InitiateUploadBlob(ctx *context.Context) {
195195
from := ctx.FormTrim("from")
196196
if mount != "" {
197197
blob, _ := workaroundGetContainerBlob(ctx, &container_model.BlobSearchOptions{
198-
Image: from,
199-
Digest: mount,
198+
Repository: from,
199+
Digest: mount,
200200
})
201201
if blob != nil {
202+
if err := mountBlob(&packages_service.PackageInfo{Owner: ctx.Package.Owner, Name: image}, blob.Blob); err != nil {
203+
apiError(ctx, http.StatusInternalServerError, err)
204+
return
205+
}
206+
202207
setResponseHeaders(ctx.Resp, &containerHeaders{
203208
Location: fmt.Sprintf("/v2/%s/%s/blobs/%s", ctx.Package.Owner.LowerName, image, mount),
204209
ContentDigest: mount,

‎templates/package/settings.tmpl

+1-1
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@
5151
{{.locale.Tr "packages.settings.delete"}}
5252
</div>
5353
<div class="content">
54-
<div class="ui warning message text left">
54+
<div class="ui warning message text left word-break">
5555
{{.locale.Tr "packages.settings.delete.notice" .PackageDescriptor.Package.Name .PackageDescriptor.Version.Version}}
5656
</div>
5757
<form class="ui form" action="{{.Link}}" method="post">

‎tests/integration/api_packages_container_test.go

+26-15
Original file line numberDiff line numberDiff line change
@@ -256,6 +256,32 @@ func TestPackageContainer(t *testing.T) {
256256
})
257257
})
258258

259+
t.Run("UploadBlob/Mount", func(t *testing.T) {
260+
defer tests.PrintCurrentTest(t)()
261+
262+
req := NewRequest(t, "POST", fmt.Sprintf("%s/blobs/uploads?mount=%s", url, unknownDigest))
263+
addTokenAuthHeader(req, userToken)
264+
MakeRequest(t, req, http.StatusAccepted)
265+
266+
req = NewRequest(t, "POST", fmt.Sprintf("%s/blobs/uploads?mount=%s", url, blobDigest))
267+
addTokenAuthHeader(req, userToken)
268+
resp := MakeRequest(t, req, http.StatusCreated)
269+
270+
assert.Equal(t, fmt.Sprintf("/v2/%s/%s/blobs/%s", user.Name, image, blobDigest), resp.Header().Get("Location"))
271+
assert.Equal(t, blobDigest, resp.Header().Get("Docker-Content-Digest"))
272+
273+
req = NewRequest(t, "POST", fmt.Sprintf("%s/blobs/uploads?mount=%s&from=%s", url, unknownDigest, "unknown/image"))
274+
addTokenAuthHeader(req, userToken)
275+
MakeRequest(t, req, http.StatusAccepted)
276+
277+
req = NewRequest(t, "POST", fmt.Sprintf("%s/blobs/uploads?mount=%s&from=%s/%s", url, blobDigest, user.Name, image))
278+
addTokenAuthHeader(req, userToken)
279+
resp = MakeRequest(t, req, http.StatusCreated)
280+
281+
assert.Equal(t, fmt.Sprintf("/v2/%s/%s/blobs/%s", user.Name, image, blobDigest), resp.Header().Get("Location"))
282+
assert.Equal(t, blobDigest, resp.Header().Get("Docker-Content-Digest"))
283+
})
284+
259285
for _, tag := range tags {
260286
t.Run(fmt.Sprintf("[Tag:%s]", tag), func(t *testing.T) {
261287
t.Run("UploadManifest", func(t *testing.T) {
@@ -444,21 +470,6 @@ func TestPackageContainer(t *testing.T) {
444470
assert.Equal(t, indexManifestDigest, pd.Files[0].Properties.GetByName(container_module.PropertyDigest))
445471
})
446472

447-
t.Run("UploadBlob/Mount", func(t *testing.T) {
448-
defer tests.PrintCurrentTest(t)()
449-
450-
req := NewRequest(t, "POST", fmt.Sprintf("%s/blobs/uploads?mount=%s", url, unknownDigest))
451-
addTokenAuthHeader(req, userToken)
452-
MakeRequest(t, req, http.StatusAccepted)
453-
454-
req = NewRequest(t, "POST", fmt.Sprintf("%s/blobs/uploads?mount=%s", url, blobDigest))
455-
addTokenAuthHeader(req, userToken)
456-
resp := MakeRequest(t, req, http.StatusCreated)
457-
458-
assert.Equal(t, fmt.Sprintf("/v2/%s/%s/blobs/%s", user.Name, image, blobDigest), resp.Header().Get("Location"))
459-
assert.Equal(t, blobDigest, resp.Header().Get("Docker-Content-Digest"))
460-
})
461-
462473
t.Run("HeadBlob", func(t *testing.T) {
463474
defer tests.PrintCurrentTest(t)()
464475

0 commit comments

Comments
 (0)