Add mirror last_updated_at to model and dataset detail API (#167)

* Add mirror last_updated_at to model and dataset detail API * Use transaction to update mirror and repository status * Optimize code
OpenCSGs · Nov 7, 2024 · a6987fd · a6987fd
1 parent 6230ca0
commit a6987fd
Show file tree

Hide file tree

Showing 12 changed files with 108 additions and 64 deletions.
diff --git a/builder/store/database/dataset.go b/builder/store/database/dataset.go
@@ -149,6 +149,7 @@ func (s *DatasetStore) FindByPath(ctx context.Context, namespace string, repoPat
 		NewSelect().
 		Model(resDataset).
 		Relation("Repository.User").
+		Relation("Repository.Mirror").
 		Where("repository.path =?", fmt.Sprintf("%s/%s", namespace, repoPath)).
 		Scan(ctx)
 	if err != nil {

diff --git a/builder/store/database/mirror.go b/builder/store/database/mirror.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/uptrace/bun"
 	"opencsg.com/csghub-server/common/types"
 )
 
@@ -283,3 +284,18 @@ func (s *MirrorStore) IndexWithPagination(ctx context.Context, per, page int) (m
 
 	return
 }
+
+func (s *MirrorStore) UpdateMirrorAndRepository(ctx context.Context, mirror *Mirror, repo *Repository) error {
+	err := s.db.Operator.Core.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
+		_, err := tx.NewUpdate().Model(mirror).WherePK().Exec(ctx)
+		if err != nil {
+			return fmt.Errorf("failed to update mirror: %v", err)
+		}
+		_, err = tx.NewUpdate().Model(repo).WherePK().Exec(ctx)
+		if err != nil {
+			return fmt.Errorf("failed to update repository: %v", err)
+		}
+		return nil
+	})
+	return err
+}
diff --git a/builder/store/database/model.go b/builder/store/database/model.go
@@ -174,6 +174,7 @@ func (s *ModelStore) FindByPath(ctx context.Context, namespace string, name stri
 		NewSelect().
 		Model(resModel).
 		Relation("Repository.User").
+		Relation("Repository.Mirror").
 		Where("repository.path =?", fmt.Sprintf("%s/%s", namespace, name)).
 		Limit(1).
 		Scan(ctx)

diff --git a/common/types/dataset.go b/common/types/dataset.go
@@ -18,27 +18,28 @@ type UpdateDatasetReq struct {
 }
 
 type Dataset struct {
-	ID            int64                `json:"id,omitempty"`
-	Name          string               `json:"name"`
-	Nickname      string               `json:"nickname"`
-	Description   string               `json:"description"`
-	Likes         int64                `json:"likes"`
-	Downloads     int64                `json:"downloads"`
-	Path          string               `json:"path"`
-	RepositoryID  int64                `json:"repository_id"`
-	Repository    Repository           `json:"repository"`
-	Private       bool                 `json:"private"`
-	User          User                 `json:"user"`
-	Tags          []RepoTag            `json:"tags"`
-	Readme        string               `json:"readme"`
-	DefaultBranch string               `json:"default_branch"`
-	CreatedAt     time.Time            `json:"created_at"`
-	UpdatedAt     time.Time            `json:"updated_at"`
-	UserLikes     bool                 `json:"user_likes"`
-	Source        RepositorySource     `json:"source"`
-	SyncStatus    RepositorySyncStatus `json:"sync_status"`
-	License       string               `json:"license"`
-	CanWrite      bool                 `json:"can_write"`
-	CanManage     bool                 `json:"can_manage"`
-	Namespace     *Namespace           `json:"namespace"`
+	ID                  int64                `json:"id,omitempty"`
+	Name                string               `json:"name"`
+	Nickname            string               `json:"nickname"`
+	Description         string               `json:"description"`
+	Likes               int64                `json:"likes"`
+	Downloads           int64                `json:"downloads"`
+	Path                string               `json:"path"`
+	RepositoryID        int64                `json:"repository_id"`
+	Repository          Repository           `json:"repository"`
+	Private             bool                 `json:"private"`
+	User                User                 `json:"user"`
+	Tags                []RepoTag            `json:"tags"`
+	Readme              string               `json:"readme"`
+	DefaultBranch       string               `json:"default_branch"`
+	CreatedAt           time.Time            `json:"created_at"`
+	UpdatedAt           time.Time            `json:"updated_at"`
+	UserLikes           bool                 `json:"user_likes"`
+	Source              RepositorySource     `json:"source"`
+	SyncStatus          RepositorySyncStatus `json:"sync_status"`
+	License             string               `json:"license"`
+	CanWrite            bool                 `json:"can_write"`
+	CanManage           bool                 `json:"can_manage"`
+	Namespace           *Namespace           `json:"namespace"`
+	MirrorLastUpdatedAt time.Time            `json:"mirror_last_updated_at"`
 }
diff --git a/common/types/model.go b/common/types/model.go
@@ -154,17 +154,18 @@ type Model struct {
 	// widget UI style: generation,chat
 	WidgetType ModelWidgetType `json:"widget_type" example:"generation"`
 	// url to interact with the model
-	Status          string               `json:"status" example:"RUNNING"`
-	UserLikes       bool                 `json:"user_likes"`
-	Source          RepositorySource     `json:"source"`
-	SyncStatus      RepositorySyncStatus `json:"sync_status"`
-	EnableInference bool                 `json:"enable_inference"`
-	EnableFinetune  bool                 `json:"enable_finetune"`
-	BaseModel       string               `json:"base_model"`
-	License         string               `json:"license"`
-	CanWrite        bool                 `json:"can_write"`
-	CanManage       bool                 `json:"can_manage"`
-	Namespace       *Namespace           `json:"namespace"`
+	Status              string               `json:"status" example:"RUNNING"`
+	UserLikes           bool                 `json:"user_likes"`
+	Source              RepositorySource     `json:"source"`
+	SyncStatus          RepositorySyncStatus `json:"sync_status"`
+	EnableInference     bool                 `json:"enable_inference"`
+	EnableFinetune      bool                 `json:"enable_finetune"`
+	BaseModel           string               `json:"base_model"`
+	License             string               `json:"license"`
+	CanWrite            bool                 `json:"can_write"`
+	CanManage           bool                 `json:"can_manage"`
+	Namespace           *Namespace           `json:"namespace"`
+	MirrorLastUpdatedAt time.Time            `json:"mirror_last_updated_at"`
 }
 
 type SDKModelInfo struct {

diff --git a/component/dataset.go b/component/dataset.go
@@ -430,16 +430,17 @@ func (c *DatasetComponent) Show(ctx context.Context, namespace, name, currentUse
 			Email:    dataset.Repository.User.Email,
 			Avatar:   dataset.Repository.User.Avatar,
 		},
-		Private:    dataset.Repository.Private,
-		CreatedAt:  dataset.CreatedAt,
-		UpdatedAt:  dataset.Repository.UpdatedAt,
-		UserLikes:  likeExists,
-		Source:     dataset.Repository.Source,
-		SyncStatus: dataset.Repository.SyncStatus,
-		License:    dataset.Repository.License,
-		CanWrite:   permission.CanWrite,
-		CanManage:  permission.CanAdmin,
-		Namespace:  ns,
+		Private:             dataset.Repository.Private,
+		CreatedAt:           dataset.CreatedAt,
+		UpdatedAt:           dataset.Repository.UpdatedAt,
+		UserLikes:           likeExists,
+		Source:              dataset.Repository.Source,
+		SyncStatus:          dataset.Repository.SyncStatus,
+		License:             dataset.Repository.License,
+		MirrorLastUpdatedAt: dataset.Repository.Mirror.LastUpdatedAt,
+		CanWrite:            permission.CanWrite,
+		CanManage:           permission.CanAdmin,
+		Namespace:           ns,
 	}
 
 	return resDataset, nil

diff --git a/component/model.go b/component/model.go
@@ -411,12 +411,13 @@ func (c *ModelComponent) Show(ctx context.Context, namespace, name, currentUser
 		CreatedAt: model.CreatedAt,
 		UpdatedAt: model.Repository.UpdatedAt,
 		// TODO:default to ModelWidgetTypeGeneration, need to config later
-		WidgetType: types.ModelWidgetTypeGeneration,
-		UserLikes:  likeExists,
-		Source:     model.Repository.Source,
-		SyncStatus: model.Repository.SyncStatus,
-		BaseModel:  model.BaseModel,
-		License:    model.Repository.License,
+		WidgetType:          types.ModelWidgetTypeGeneration,
+		UserLikes:           likeExists,
+		Source:              model.Repository.Source,
+		SyncStatus:          model.Repository.SyncStatus,
+		BaseModel:           model.BaseModel,
+		License:             model.Repository.License,
+		MirrorLastUpdatedAt: model.Repository.Mirror.LastUpdatedAt,
 
 		CanWrite:  permission.CanWrite,
 		CanManage: permission.CanAdmin,

diff --git a/docs/docs.go b/docs/docs.go
@@ -16643,6 +16643,9 @@ const docTemplate = `{
                 "likes": {
                     "type": "integer"
                 },
+                "mirror_last_updated_at": {
+                    "type": "string"
+                },
                 "name": {
                     "type": "string"
                 },
@@ -17087,6 +17090,9 @@ const docTemplate = `{
                 "likes": {
                     "type": "integer"
                 },
+                "mirror_last_updated_at": {
+                    "type": "string"
+                },
                 "name": {
                     "type": "string"
                 },

diff --git a/docs/swagger.json b/docs/swagger.json
@@ -16632,6 +16632,9 @@
                 "likes": {
                     "type": "integer"
                 },
+                "mirror_last_updated_at": {
+                    "type": "string"
+                },
                 "name": {
                     "type": "string"
                 },
@@ -17076,6 +17079,9 @@
                 "likes": {
                     "type": "integer"
                 },
+                "mirror_last_updated_at": {
+                    "type": "string"
+                },
                 "name": {
                     "type": "string"
                 },

diff --git a/docs/swagger.yaml b/docs/swagger.yaml
@@ -1268,6 +1268,8 @@ definitions:
         type: string
       likes:
         type: integer
+      mirror_last_updated_at:
+        type: string
       name:
         type: string
       namespace:
@@ -1570,6 +1572,8 @@ definitions:
         type: string
       likes:
         type: integer
+      mirror_last_updated_at:
+        type: string
       name:
         type: string
       namespace:

diff --git a/mirror/lfssyncer/minio.go b/mirror/lfssyncer/minio.go
@@ -10,6 +10,7 @@ import (
 	"net/url"
 	"path/filepath"
 	"sync"
+	"time"
 
 	"github.com/minio/minio-go/v7"
 	"opencsg.com/csghub-server/builder/store/database"
@@ -98,19 +99,21 @@ func (w *MinioLFSSyncWorker) worker(id int) {
 		err = w.SyncLfs(ctx, id, mirror)
 		if err != nil {
 			repo.SyncStatus = types.SyncStatusFailed
-			_, repoErr := w.repoStore.UpdateRepo(ctx, *repo)
-			if repoErr != nil {
-				slog.Error("fail to update repo sync status to failed: %w", slog.Any("error", err))
+			mirror.LastMessage = err.Error()
+			err = w.mirrorStore.UpdateMirrorAndRepository(ctx, mirror, repo)
+			if err != nil {
+				slog.Error("fail to update mirror and repository", slog.Int("workerId", id), slog.Any("error", err))
 			}
 			slog.Error("fail to sync lfs", slog.Int("workerId", id), slog.String("error", err.Error()))
 			continue
 		}
-
 		repo.SyncStatus = types.SyncStatusCompleted
-		_, err = w.repoStore.UpdateRepo(ctx, *repo)
+		mirror.LastUpdatedAt = time.Now()
+		err = w.mirrorStore.UpdateMirrorAndRepository(ctx, mirror, repo)
 		if err != nil {
-			slog.Error("fail to update repo sync status to complete: %w", slog.Any("error", err))
+			slog.Error("fail to update mirror and repository", slog.Int("workerId", id), slog.Any("error", err))
 		}
+		slog.Info("sync lfs completed", slog.Int("workerId", id))
 	}
 }
 

diff --git a/mirror/reposyncer/local_woker.go b/mirror/reposyncer/local_woker.go
@@ -7,6 +7,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"time"
 
 	"opencsg.com/csghub-server/builder/git"
 	"opencsg.com/csghub-server/builder/git/gitserver"
@@ -125,6 +126,12 @@ func (w *LocalMirrorWoker) SyncRepo(ctx context.Context, task queue.MirrorTask)
 	err = w.git.MirrorSync(ctx, req)
 
 	if err != nil {
+		mirror.Status = types.MirrorFailed
+		mirror.LastMessage = fmt.Sprintf("failed mirror remote repo in git server:%s", err.Error())
+		updateErr := w.mirrorStore.Update(ctx, mirror)
+		if updateErr != nil {
+			return fmt.Errorf("failed to update mirror: %w", updateErr)
+		}
 		return fmt.Errorf("failed mirror remote repo in git server: %v", err)
 	}
 	slog.Info("Mirror remote repo in git server successfully", "repo_type", mirror.Repository.RepositoryType, "namespace", namespace, "name", name)
@@ -158,15 +165,10 @@ func (w *LocalMirrorWoker) SyncRepo(ctx context.Context, task queue.MirrorTask)
 	if err != nil {
 		mirror.Status = types.MirrorIncomplete
 		mirror.LastMessage = err.Error()
-		err = w.mirrorStore.Update(ctx, mirror)
-		if err != nil {
-			return fmt.Errorf("failed to update mirror: %w", err)
-		}
-
 		mirror.Repository.SyncStatus = types.SyncStatusFailed
-		_, err = w.repoStore.UpdateRepo(ctx, *mirror.Repository)
+		err = w.mirrorStore.UpdateMirrorAndRepository(ctx, mirror, mirror.Repository)
 		if err != nil {
-			return fmt.Errorf("failed to update repo sync status to failed: %w", err)
+			return fmt.Errorf("failed to update mirror and repository: %w", err)
 		}
 		return fmt.Errorf("failed to sync lfs files: %v", err)
 	}
@@ -181,7 +183,8 @@ func (w *LocalMirrorWoker) SyncRepo(ctx context.Context, task queue.MirrorTask)
 	} else {
 		mirror.Status = types.MirrorFinished
 	}
-
+	// Update mirror last updated at
+	mirror.LastUpdatedAt = time.Now()
 	err = w.mirrorStore.Update(ctx, mirror)
 	if err != nil {
 		return fmt.Errorf("failed to update mirror: %w", err)