Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix merging artifact chunks error when minio storage basepath is set #28555

Merged
merged 4 commits into from
Dec 21, 2023
Merged

Fix merging artifact chunks error when minio storage basepath is set #28555

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
f2b702a
fix: trim minio basepath when merging chunks
fuxiaohei Dec 20, 2023
b70a3cc
fix: update artifact chunk basename to ignore directory value in stor…
fuxiaohei Dec 20, 2023
3644975
update filepath to handle file's path from storage
fuxiaohei Dec 20, 2023
2e72950
Merge branch 'main' into fix/artifact-upload-minio
GiteaBot Dec 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 9 additions & 4 deletions routers/api/actions/artifacts_chunks.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,11 @@ func saveUploadChunk(st storage.ObjectStorage, ctx *ArtifactContext,
contentRange := ctx.Req.Header.Get("Content-Range")
start, end, length := int64(0), int64(0), int64(0)
if _, err := fmt.Sscanf(contentRange, "bytes %d-%d/%d", &start, &end, &length); err != nil {
log.Warn("parse content range error: %v, content-range: %s", err, contentRange)
return -1, fmt.Errorf("parse content range error: %v", err)
}
// build chunk store path
storagePath := fmt.Sprintf("tmp%d/%d-%d-%d.chunk", runID, artifact.ID, start, end)
storagePath := fmt.Sprintf("tmp%d/%d-%d-%d-%d.chunk", runID, runID, artifact.ID, start, end)
// use io.TeeReader to avoid reading all body to md5 sum.
// it writes data to hasher after reading end
// if hash is not matched, delete the read-end result
Expand Down Expand Up @@ -58,6 +59,7 @@ func saveUploadChunk(st storage.ObjectStorage, ctx *ArtifactContext,
}

type chunkFileItem struct {
RunID int64
ArtifactID int64
Start int64
End int64
Expand All @@ -67,9 +69,12 @@ type chunkFileItem struct {
func listChunksByRunID(st storage.ObjectStorage, runID int64) (map[int64][]*chunkFileItem, error) {
storageDir := fmt.Sprintf("tmp%d", runID)
var chunks []*chunkFileItem
if err := st.IterateObjects(storageDir, func(path string, obj storage.Object) error {
item := chunkFileItem{Path: path}
if _, err := fmt.Sscanf(path, filepath.Join(storageDir, "%d-%d-%d.chunk"), &item.ArtifactID, &item.Start, &item.End); err != nil {
if err := st.IterateObjects(storageDir, func(fpath string, obj storage.Object) error {
baseName := filepath.Base(fpath)
fuxiaohei marked this conversation as resolved.
Show resolved Hide resolved
// when read chunks from storage, it only contains storage dir and basename,
// no matter the subdirectory setting in storage config
item := chunkFileItem{Path: storageDir + "/" + baseName}
if _, err := fmt.Sscanf(baseName, "%d-%d-%d-%d.chunk", &item.RunID, &item.ArtifactID, &item.Start, &item.End); err != nil {
return fmt.Errorf("parse content range error: %v", err)
}
chunks = append(chunks, &item)
Expand Down