Implement actions artifacts (#22738)

Implement action artifacts server api.

This change is used for supporting
https://github.com/actions/upload-artifact and
https://github.com/actions/download-artifact in gitea actions. It can
run sample workflow from doc
https://docs.github.com/en/actions/using-workflows/storing-workflow-data-as-artifacts.
The api design is inspired by
https://github.com/nektos/act/blob/master/pkg/artifacts/server.go and
includes some changes from gitea internal structs and methods.

Actions artifacts contains two parts:

- Gitea server api and storage (this pr implement basic design without
some complex cases supports)
- Runner communicate with gitea server api (in comming)

Old pr #22345 is outdated after
actions merged. I create new pr from main branch.


![897f7694-3e0f-4f7c-bb4b-9936624ead45](https://user-images.githubusercontent.com/2142787/219382371-eb3cf810-e4e0-456b-a8ff-aecc2b1a1032.jpeg)

Add artifacts list in actions workflow page.

models/actions/artifact.go

@@ -0,0 +1,122 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+// This artifact server is inspired by https://github.com/nektos/act/blob/master/pkg/artifacts/server.go.
+// It updates url setting and uses ObjectStore to handle artifacts persistence.
+
+package actions
+
+import (
+	"context"
+	"errors"
+
+	"code.gitea.io/gitea/models/db"
+	"code.gitea.io/gitea/modules/timeutil"
+	"code.gitea.io/gitea/modules/util"
+)
+
+const (
+	// ArtifactStatusUploadPending is the status of an artifact upload that is pending
+	ArtifactStatusUploadPending = 1
+	// ArtifactStatusUploadConfirmed is the status of an artifact upload that is confirmed
+	ArtifactStatusUploadConfirmed = 2
+	// ArtifactStatusUploadError is the status of an artifact upload that is errored
+	ArtifactStatusUploadError = 3
+)
+
+func init() {
+	db.RegisterModel(new(ActionArtifact))
+}
+
+// ActionArtifact is a file that is stored in the artifact storage.
+type ActionArtifact struct {
+	ID                 int64 `xorm:"pk autoincr"`
+	RunID              int64 `xorm:"index UNIQUE(runid_name)"` // The run id of the artifact
+	RunnerID           int64
+	RepoID             int64 `xorm:"index"`
+	OwnerID            int64
+	CommitSHA          string
+	StoragePath        string             // The path to the artifact in the storage
+	FileSize           int64              // The size of the artifact in bytes
+	FileCompressedSize int64              // The size of the artifact in bytes after gzip compression
+	ContentEncoding    string             // The content encoding of the artifact
+	ArtifactPath       string             // The path to the artifact when runner uploads it
+	ArtifactName       string             `xorm:"UNIQUE(runid_name)"` // The name of the artifact when runner uploads it
+	Status             int64              `xorm:"index"`              // The status of the artifact, uploading, expired or need-delete
+	CreatedUnix        timeutil.TimeStamp `xorm:"created"`
+	UpdatedUnix        timeutil.TimeStamp `xorm:"updated index"`
+}
+
+// CreateArtifact create a new artifact with task info or get same named artifact in the same run
+func CreateArtifact(ctx context.Context, t *ActionTask, artifactName string) (*ActionArtifact, error) {
+	if err := t.LoadJob(ctx); err != nil {
+		return nil, err
+	}
+	artifact, err := getArtifactByArtifactName(ctx, t.Job.RunID, artifactName)
+	if errors.Is(err, util.ErrNotExist) {
+		artifact := &ActionArtifact{
+			RunID:     t.Job.RunID,
+			RunnerID:  t.RunnerID,
+			RepoID:    t.RepoID,
+			OwnerID:   t.OwnerID,
+			CommitSHA: t.CommitSHA,
+			Status:    ArtifactStatusUploadPending,
+		}
+		if _, err := db.GetEngine(ctx).Insert(artifact); err != nil {
+			return nil, err
+		}
+		return artifact, nil
+	} else if err != nil {
+		return nil, err
+	}
+	return artifact, nil
+}
+
+func getArtifactByArtifactName(ctx context.Context, runID int64, name string) (*ActionArtifact, error) {
+	var art ActionArtifact
+	has, err := db.GetEngine(ctx).Where("run_id = ? AND artifact_name = ?", runID, name).Get(&art)
+	if err != nil {
+		return nil, err
+	} else if !has {
+		return nil, util.ErrNotExist
+	}
+	return &art, nil
+}
+
+// GetArtifactByID returns an artifact by id
+func GetArtifactByID(ctx context.Context, id int64) (*ActionArtifact, error) {
+	var art ActionArtifact
+	has, err := db.GetEngine(ctx).ID(id).Get(&art)
+	if err != nil {
+		return nil, err
+	} else if !has {
+		return nil, util.ErrNotExist
+	}
+
+	return &art, nil
+}
+
+// UpdateArtifactByID updates an artifact by id
+func UpdateArtifactByID(ctx context.Context, id int64, art *ActionArtifact) error {
+	art.ID = id
+	_, err := db.GetEngine(ctx).ID(id).AllCols().Update(art)
+	return err
+}
+
+// ListArtifactsByRunID returns all artifacts of a run
+func ListArtifactsByRunID(ctx context.Context, runID int64) ([]*ActionArtifact, error) {
+	arts := make([]*ActionArtifact, 0, 10)
+	return arts, db.GetEngine(ctx).Where("run_id=?", runID).Find(&arts)
+}
+
+// ListUploadedArtifactsByRunID returns all uploaded artifacts of a run
+func ListUploadedArtifactsByRunID(ctx context.Context, runID int64) ([]*ActionArtifact, error) {
+	arts := make([]*ActionArtifact, 0, 10)
+	return arts, db.GetEngine(ctx).Where("run_id=? AND status=?", runID, ArtifactStatusUploadConfirmed).Find(&arts)
+}
+
+// ListArtifactsByRepoID returns all artifacts of a repo
+func ListArtifactsByRepoID(ctx context.Context, repoID int64) ([]*ActionArtifact, error) {
+	arts := make([]*ActionArtifact, 0, 10)
+	return arts, db.GetEngine(ctx).Where("repo_id=?", repoID).Find(&arts)
+}

models/fixtures/access_token.yml

@@ -30,4 +30,4 @@
   token_last_eight: 69d28c91
   created_unix: 946687980
   updated_unix: 946687980
-#commented out tokens so you can see what they are in plaintext
+#commented out tokens so you can see what they are in plaintext

models/fixtures/action_run.yml

@@ -0,0 +1,19 @@
+-
+  id: 791
+  title: "update actions"
+  repo_id: 4
+  owner_id: 1
+  workflow_id: "artifact.yaml"
+  index: 187
+  trigger_user_id: 1
+  ref: "refs/heads/master"
+  commit_sha: "c2d72f548424103f01ee1dc02889c1e2bff816b0"
+  event: "push"
+  is_fork_pull_request: 0
+  status: 1
+  started: 1683636528
+  stopped: 1683636626
+  created: 1683636108
+  updated: 1683636626
+  need_approval: 0
+  approved_by: 0

models/fixtures/action_run_job.yml

@@ -0,0 +1,14 @@
+-
+  id: 192
+  run_id: 791
+  repo_id: 4
+  owner_id: 1
+  commit_sha: c2d72f548424103f01ee1dc02889c1e2bff816b0
+  is_fork_pull_request: 0
+  name: job_2
+  attempt: 1
+  job_id: job_2
+  task_id: 47
+  status: 1
+  started: 1683636528
+  stopped: 1683636626

models/fixtures/action_task.yml

@@ -0,0 +1,20 @@
+-
+  id: 47
+  job_id: 192
+  attempt: 3
+  runner_id: 1
+  status: 6 # 6 is the status code for "running", running task can upload artifacts
+  started: 1683636528
+  stopped: 1683636626
+  repo_id: 4
+  owner_id: 1
+  commit_sha: c2d72f548424103f01ee1dc02889c1e2bff816b0
+  is_fork_pull_request: 0
+  token_hash: 6d8ef48297195edcc8e22c70b3020eaa06c52976db67d39b4260c64a69a2cc1508825121b7b8394e48e00b1bf8718b2a867e
+  token_salt: jVuKnSPGgy
+  token_last_eight: eeb1a71a
+  log_filename: artifact-test2/2f/47.log
+  log_in_storage: 1
+  log_length: 707
+  log_size: 90179
+  log_expired: 0

models/migrations/migrations.go

@@ -491,6 +491,8 @@ var migrations = []Migration{
 	NewMigration("Add ArchivedUnix Column", v1_20.AddArchivedUnixToRepository),
 	// v256 -> v257
 	NewMigration("Add is_internal column to package", v1_20.AddIsInternalColumnToPackage),
+	// v257 -> v258
+	NewMigration("Add Actions Artifact table", v1_20.CreateActionArtifactTable),
 }
 
 // GetCurrentDBVersion returns the current db version

models/migrations/v1_20/v257.go

@@ -0,0 +1,33 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package v1_20 //nolint
+
+import (
+	"code.gitea.io/gitea/modules/timeutil"
+
+	"xorm.io/xorm"
+)
+
+func CreateActionArtifactTable(x *xorm.Engine) error {
+	// ActionArtifact is a file that is stored in the artifact storage.
+	type ActionArtifact struct {
+		ID                 int64 `xorm:"pk autoincr"`
+		RunID              int64 `xorm:"index UNIQUE(runid_name)"` // The run id of the artifact
+		RunnerID           int64
+		RepoID             int64 `xorm:"index"`
+		OwnerID            int64
+		CommitSHA          string
+		StoragePath        string             // The path to the artifact in the storage
+		FileSize           int64              // The size of the artifact in bytes
+		FileCompressedSize int64              // The size of the artifact in bytes after gzip compression
+		ContentEncoding    string             // The content encoding of the artifact
+		ArtifactPath       string             // The path to the artifact when runner uploads it
+		ArtifactName       string             `xorm:"UNIQUE(runid_name)"` // The name of the artifact when runner uploads it
+		Status             int64              `xorm:"index"`              // The status of the artifact
+		CreatedUnix        timeutil.TimeStamp `xorm:"created"`
+		UpdatedUnix        timeutil.TimeStamp `xorm:"updated index"`
+	}
+
+	return x.Sync(new(ActionArtifact))
+}

models/repo.go

@@ -59,6 +59,12 @@ func DeleteRepository(doer *user_model.User, uid, repoID int64) error {
 		return fmt.Errorf("find actions tasks of repo %v: %w", repoID, err)
 	}
 
+	// Query the artifacts of this repo, they will be needed after they have been deleted to remove artifacts files in ObjectStorage
+	artifacts, err := actions_model.ListArtifactsByRepoID(ctx, repoID)
+	if err != nil {
+		return fmt.Errorf("list actions artifacts of repo %v: %w", repoID, err)
+	}
+
 	// In case is a organization.
 	org, err := user_model.GetUserByID(ctx, uid)
 	if err != nil {
@@ -164,6 +170,7 @@ func DeleteRepository(doer *user_model.User, uid, repoID int64) error {
 		&actions_model.ActionRunJob{RepoID: repoID},
 		&actions_model.ActionRun{RepoID: repoID},
 		&actions_model.ActionRunner{RepoID: repoID},
+		&actions_model.ActionArtifact{RepoID: repoID},
 	); err != nil {
 		return fmt.Errorf("deleteBeans: %w", err)
 	}
@@ -336,6 +343,14 @@ func DeleteRepository(doer *user_model.User, uid, repoID int64) error {
 		}
 	}
 
+	// delete actions artifacts in ObjectStorage after the repo have already been deleted
+	for _, art := range artifacts {
+		if err := storage.ActionsArtifacts.Delete(art.StoragePath); err != nil {
+			log.Error("remove artifact file %q: %v", art.StoragePath, err)
+			// go on
+		}
+	}
+
 	return nil
 }
 

models/unittest/testdb.go

@@ -126,7 +126,7 @@ func MainTest(m *testing.M, testOpts *TestOptions) {
 
 	setting.Packages.Storage.Path = filepath.Join(setting.AppDataPath, "packages")
 
-	setting.Actions.Storage.Path = filepath.Join(setting.AppDataPath, "actions_log")
+	setting.Actions.LogStorage.Path = filepath.Join(setting.AppDataPath, "actions_log")
 
 	setting.Git.HomePath = filepath.Join(setting.AppDataPath, "home")
 

modules/setting/actions.go

@@ -10,7 +10,8 @@ import (
 // Actions settings
 var (
 	Actions = struct {
-		Storage           // how the created logs should be stored
+		LogStorage        Storage // how the created logs should be stored
+		ArtifactStorage   Storage // how the created artifacts should be stored
 		Enabled           bool
 		DefaultActionsURL string `ini:"DEFAULT_ACTIONS_URL"`
 	}{
@@ -25,5 +26,9 @@ func loadActionsFrom(rootCfg ConfigProvider) {
 		log.Fatal("Failed to map Actions settings: %v", err)
 	}
 
-	Actions.Storage = getStorage(rootCfg, "actions_log", "", nil)
+	actionsSec := rootCfg.Section("actions.artifacts")
+	storageType := actionsSec.Key("STORAGE_TYPE").MustString("")
+
+	Actions.LogStorage = getStorage(rootCfg, "actions_log", "", nil)
+	Actions.ArtifactStorage = getStorage(rootCfg, "actions_artifacts", storageType, actionsSec)
 }

modules/storage/storage.go

@@ -128,6 +128,8 @@ var (
 
 	// Actions represents actions storage
 	Actions ObjectStorage = uninitializedStorage
+	// Actions Artifacts represents actions artifacts storage
+	ActionsArtifacts ObjectStorage = uninitializedStorage
 )
 
 // Init init the stoarge
@@ -212,9 +214,14 @@ func initPackages() (err error) {
 func initActions() (err error) {
 	if !setting.Actions.Enabled {
 		Actions = discardStorage("Actions isn't enabled")
+		ActionsArtifacts = discardStorage("ActionsArtifacts isn't enabled")
 		return nil
 	}
-	log.Info("Initialising Actions storage with type: %s", setting.Actions.Storage.Type)
-	Actions, err = NewStorage(setting.Actions.Storage.Type, &setting.Actions.Storage)
+	log.Info("Initialising Actions storage with type: %s", setting.Actions.LogStorage.Type)
+	if Actions, err = NewStorage(setting.Actions.LogStorage.Type, &setting.Actions.LogStorage); err != nil {
+		return err
+	}
+	log.Info("Initialising ActionsArtifacts storage with type: %s", setting.Actions.ArtifactStorage.Type)
+	ActionsArtifacts, err = NewStorage(setting.Actions.ArtifactStorage.Type, &setting.Actions.ArtifactStorage)
 	return err
 }

options/locale/locale_en-US.ini

@@ -114,6 +114,8 @@ unknown = Unknown
 
 rss_feed = RSS Feed
 
+artifacts = Artifacts
+
 concept_system_global = Global
 concept_user_individual = Individual
 concept_code_repository = Repository