refactor sensitive check into a new moderation service; introduct Tem…

…poral workflow to run

.gitignore

@@ -19,4 +19,7 @@ run_server_local.sh
 run_user_local.sh
 cmd/csghub-server/__debug_*
 run_dataviewer_local.sh
+
+.data
+/data
 */*/_api_test/

api/handler/code.go

@@ -20,15 +20,19 @@ func NewCodeHandler(config *config.Config) (*CodeHandler, error) {
 	if err != nil {
 		return nil, err
 	}
+	sc, err := component.NewSensitiveComponent(config)
+	if err != nil {
+		return nil, fmt.Errorf("error creating sensitive component:%w", err)
+	}
 	return &CodeHandler{
 		c:  tc,
-		sc: component.NewSensitiveComponent(config),
+		sc: sc,
 	}, nil
 }
 
 type CodeHandler struct {
 	c  *component.CodeComponent
-	sc component.SensitiveChecker
+	sc *component.SensitiveComponent
 }
 
 // CreateCode   godoc
@@ -57,7 +61,7 @@ func (h *CodeHandler) Create(ctx *gin.Context) {
 		return
 	}
 
-	_, err := h.sc.CheckRequest(ctx, req)
+	_, err := h.sc.CheckRequestV2(ctx, req)
 	if err != nil {
 		slog.Error("failed to check sensitive request", slog.Any("error", err))
 		httpbase.BadRequest(ctx, fmt.Errorf("sensitive check failed: %w", err).Error())
@@ -166,7 +170,7 @@ func (h *CodeHandler) Update(ctx *gin.Context) {
 		return
 	}
 
-	_, err := h.sc.CheckRequest(ctx, req)
+	_, err := h.sc.CheckRequestV2(ctx, req)
 	if err != nil {
 		slog.Error("failed to check sensitive request", slog.Any("error", err))
 		httpbase.BadRequest(ctx, fmt.Errorf("sensitive check failed: %w", err).Error())

api/handler/collection.go

@@ -21,15 +21,19 @@ func NewCollectionHandler(cfg *config.Config) (*CollectionHandler, error) {
 	if err != nil {
 		return nil, err
 	}
+	sc, err := component.NewSensitiveComponent(cfg)
+	if err != nil {
+		return nil, fmt.Errorf("error creating sensitive component:%w", err)
+	}
 	return &CollectionHandler{
 		cc: cc,
-		sc: component.NewSensitiveComponent(cfg),
+		sc: sc,
 	}, nil
 }
 
 type CollectionHandler struct {
 	cc *component.CollectionComponent
-	sc component.SensitiveChecker
+	sc *component.SensitiveComponent
 }
 
 // GetCollections godoc
@@ -100,7 +104,7 @@ func (c *CollectionHandler) Create(ctx *gin.Context) {
 		return
 	}
 
-	_, err := c.sc.CheckRequest(ctx, req)
+	_, err := c.sc.CheckRequestV2(ctx, req)
 	if err != nil {
 		slog.Error("failed to check sensitive request", slog.Any("error", err))
 		httpbase.BadRequest(ctx, fmt.Errorf("sensitive check failed: %w", err).Error())
@@ -172,7 +176,7 @@ func (c *CollectionHandler) UpdateCollection(ctx *gin.Context) {
 		return
 	}
 
-	_, err := c.sc.CheckRequest(ctx, req)
+	_, err := c.sc.CheckRequestV2(ctx, req)
 	if err != nil {
 		slog.Error("failed to check sensitive request", slog.Any("error", err))
 		httpbase.BadRequest(ctx, fmt.Errorf("sensitive check failed: %w", err).Error())

api/handler/dataset.go

@@ -23,15 +23,19 @@ func NewDatasetHandler(config *config.Config) (*DatasetHandler, error) {
 	if err != nil {
 		return nil, err
 	}
+	sc, err := component.NewSensitiveComponent(config)
+	if err != nil {
+		return nil, fmt.Errorf("error creating sensitive component:%w", err)
+	}
 	return &DatasetHandler{
 		c:  tc,
-		sc: component.NewSensitiveComponent(config),
+		sc: sc,
 	}, nil
 }
 
 type DatasetHandler struct {
 	c  *component.DatasetComponent
-	sc component.SensitiveChecker
+	sc *component.SensitiveComponent
 }
 
 // CreateDataset   godoc
@@ -59,7 +63,7 @@ func (h *DatasetHandler) Create(ctx *gin.Context) {
 		httpbase.BadRequest(ctx, err.Error())
 		return
 	}
-	_, err := h.sc.CheckRequest(ctx, req)
+	_, err := h.sc.CheckRequestV2(ctx, req)
 	if err != nil {
 		slog.Error("failed to check sensitive request", slog.Any("error", err))
 		httpbase.BadRequest(ctx, fmt.Errorf("sensitive check failed: %w", err).Error())
@@ -168,7 +172,7 @@ func (h *DatasetHandler) Update(ctx *gin.Context) {
 		return
 	}
 
-	_, err := h.sc.CheckRequest(ctx, req)
+	_, err := h.sc.CheckRequestV2(ctx, req)
 	if err != nil {
 		slog.Error("failed to check sensitive request", slog.Any("error", err))
 		httpbase.BadRequest(ctx, fmt.Errorf("sensitive check failed: %w", err).Error())

api/handler/discussion.go

@@ -16,12 +16,15 @@ import (
 
 type DiscussionHandler struct {
 	c  *component.DiscussionComponent
-	sc component.SensitiveChecker
+	sc *component.SensitiveComponent
 }
 
 func NewDiscussionHandler(cfg *config.Config) (*DiscussionHandler, error) {
 	c := component.NewDiscussionComponent()
-	sc := component.NewSensitiveComponent(cfg)
+	sc, err := component.NewSensitiveComponent(cfg)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create sensitive component: %w", err)
+	}
 	return &DiscussionHandler{
 		c:  c,
 		sc: sc,

api/handler/model.go

@@ -22,7 +22,10 @@ func NewModelHandler(config *config.Config) (*ModelHandler, error) {
 	if err != nil {
 		return nil, err
 	}
-	sc := component.NewSensitiveComponent(config)
+	sc, err := component.NewSensitiveComponent(config)
+	if err != nil {
+		return nil, fmt.Errorf("error creating sensitive component:%w", err)
+	}
 	return &ModelHandler{
 		c:  uc,
 		sc: sc,
@@ -31,7 +34,7 @@ func NewModelHandler(config *config.Config) (*ModelHandler, error) {
 
 type ModelHandler struct {
 	c  *component.ModelComponent
-	sc component.SensitiveChecker
+	sc *component.SensitiveComponent
 }
 
 // GetVisiableModels godoc
@@ -123,7 +126,7 @@ func (h *ModelHandler) Create(ctx *gin.Context) {
 	}
 	req.Username = currentUser
 
-	_, err := h.sc.CheckRequest(ctx, req)
+	_, err := h.sc.CheckRequestV2(ctx, req)
 	if err != nil {
 		slog.Error("failed to check sensitive request", slog.Any("error", err))
 		httpbase.BadRequest(ctx, fmt.Errorf("sensitive check failed: %w", err).Error())
@@ -168,7 +171,7 @@ func (h *ModelHandler) Update(ctx *gin.Context) {
 		return
 	}
 
-	_, err := h.sc.CheckRequest(ctx, req)
+	_, err := h.sc.CheckRequestV2(ctx, req)
 	if err != nil {
 		slog.Error("failed to check sensitive request", slog.Any("error", err))
 		httpbase.BadRequest(ctx, fmt.Errorf("sensitive check failed: %w", err).Error())
@@ -631,7 +634,7 @@ func (h *ModelHandler) DeployDedicated(ctx *gin.Context) {
 		return
 	}
 
-	_, err = h.sc.CheckRequest(ctx, &req)
+	_, err = h.sc.CheckRequestV2(ctx, &req)
 	if err != nil {
 		slog.Error("failed to check sensitive request", slog.Any("error", err))
 		httpbase.BadRequest(ctx, fmt.Errorf("sensitive check failed: %w", err).Error())

api/handler/prompt.go

@@ -21,17 +21,21 @@ import (
 
 type PromptHandler struct {
 	pc *component.PromptComponent
-	sc component.SensitiveChecker
+	sc *component.SensitiveComponent
 }
 
 func NewPromptHandler(cfg *config.Config) (*PromptHandler, error) {
 	promptComp, err := component.NewPromptComponent(cfg)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create PromptComponent: %w", err)
 	}
+	sc, err := component.NewSensitiveComponent(cfg)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create SensitiveComponent: %w", err)
+	}
 	return &PromptHandler{
 		pc: promptComp,
-		sc: component.NewSensitiveComponent(cfg),
+		sc: sc,
 	}, nil
 }
 
@@ -949,7 +953,7 @@ func (h *PromptHandler) Create(ctx *gin.Context) {
 		httpbase.BadRequest(ctx, err.Error())
 		return
 	}
-	_, err := h.sc.CheckRequest(ctx, req)
+	_, err := h.sc.CheckRequestV2(ctx, req)
 	if err != nil {
 		slog.Error("failed to check sensitive request", slog.Any("error", err))
 		httpbase.BadRequest(ctx, fmt.Errorf("sensitive check failed: %w", err).Error())
@@ -998,7 +1002,7 @@ func (h *PromptHandler) Update(ctx *gin.Context) {
 		return
 	}
 
-	_, err := h.sc.CheckRequest(ctx, req)
+	_, err := h.sc.CheckRequestV2(ctx, req)
 	if err != nil {
 		slog.Error("failed to check sensitive request", slog.Any("error", err))
 		httpbase.BadRequest(ctx, fmt.Errorf("sensitive check failed: %w", err).Error())

api/handler/repo.go

@@ -335,6 +335,8 @@ func (h *RepoHandler) FileInfo(ctx *gin.Context) {
 	httpbase.OK(ctx, file)
 }
 
+// Deprecated: use ResolveDownload instead
+//
 // DownloadRepoFile godoc
 // @Security     ApiKey or JWT
 // @Summary      Download a repo file [Depricated: use 'resolve' api instead]

api/handler/sensitive.go

@@ -1,6 +1,7 @@
 package handler
 
 import (
+	"fmt"
 	"log/slog"
 
 	"github.com/gin-gonic/gin"
@@ -10,13 +11,17 @@ import (
 )
 
 type SensitiveHandler struct {
-	c component.SensitiveChecker
+	c *component.SensitiveComponent
 }
 
-func NewSensitiveHandler(cfg *config.Config) *SensitiveHandler {
-	return &SensitiveHandler{
-		c: component.NewSensitiveComponent(cfg),
+func NewSensitiveHandler(cfg *config.Config) (*SensitiveHandler, error) {
+	sc, err := component.NewSensitiveComponent(cfg)
+	if err != nil {
+		return nil, fmt.Errorf("error creating sensitive component:%w", err)
 	}
+	return &SensitiveHandler{
+		c: sc,
+	}, nil
 }
 
 func (h *SensitiveHandler) Text(ctx *gin.Context) {

api/handler/space.go

@@ -17,19 +17,23 @@ import (
 )
 
 func NewSpaceHandler(config *config.Config) (*SpaceHandler, error) {
-	sc, err := component.NewSpaceComponent(config)
+	c, err := component.NewSpaceComponent(config)
 	if err != nil {
 		return nil, err
 	}
+	ssc, err := component.NewSensitiveComponent(config)
+	if err != nil {
+		return nil, fmt.Errorf("error creating sensitive component:%w", err)
+	}
 	return &SpaceHandler{
-		c:  sc,
-		sc: component.NewSensitiveComponent(config),
+		c:   c,
+		ssc: ssc,
 	}, nil
 }
 
 type SpaceHandler struct {
-	c  *component.SpaceComponent
-	sc component.SensitiveChecker
+	c   *component.SpaceComponent
+	ssc *component.SensitiveComponent
 }
 
 // GetAllSpaces   godoc
@@ -151,7 +155,7 @@ func (h *SpaceHandler) Create(ctx *gin.Context) {
 		httpbase.BadRequest(ctx, err.Error())
 		return
 	}
-	_, err := h.sc.CheckRequest(ctx, &req)
+	_, err := h.ssc.CheckRequestV2(ctx, &req)
 	if err != nil {
 		slog.Error("failed to check sensitive request", slog.Any("error", err))
 		httpbase.BadRequest(ctx, fmt.Errorf("sensitive check failed: %w", err).Error())
@@ -196,7 +200,7 @@ func (h *SpaceHandler) Update(ctx *gin.Context) {
 		httpbase.BadRequest(ctx, err.Error())
 		return
 	}
-	_, err := h.sc.CheckRequest(ctx, req)
+	_, err := h.ssc.CheckRequestV2(ctx, req)
 	if err != nil {
 		slog.Error("failed to check sensitive request", slog.Any("error", err))
 		httpbase.BadRequest(ctx, fmt.Errorf("sensitive check failed: %w", err).Error())

api/handler/sshkey.go

@@ -17,15 +17,19 @@ func NewSSHKeyHandler(config *config.Config) (*SSHKeyHandler, error) {
 	if err != nil {
 		return nil, err
 	}
+	sc, err := component.NewSensitiveComponent(config)
+	if err != nil {
+		return nil, fmt.Errorf("error creating sensitive component:%w", err)
+	}
 	return &SSHKeyHandler{
 		c:  oc,
-		sc: component.NewSensitiveComponent(config),
+		sc: sc,
 	}, nil
 }
 
 type SSHKeyHandler struct {
 	c  *component.SSHKeyComponent
-	sc component.SensitiveChecker
+	sc *component.SensitiveComponent
 }
 
 // CreateUserSSHKey godoc
@@ -49,7 +53,7 @@ func (h *SSHKeyHandler) Create(ctx *gin.Context) {
 		return
 	}
 
-	_, err := h.sc.CheckRequest(ctx, &req)
+	_, err := h.sc.CheckRequestV2(ctx, &req)
 	if err != nil {
 		slog.Error("failed to check sensitive request", slog.Any("error", err))
 		httpbase.BadRequest(ctx, fmt.Errorf("sensitive check failed: %w", err).Error())

api/router/api.go

@@ -240,7 +240,10 @@ func NewRouter(config *config.Config, enableSwagger bool) (*gin.Engine, error) {
 	apiGroup.GET("/callback/casdoor", userProxyHandler.Proxy)
 	// Sensive check
 	if config.SensitiveCheck.Enable {
-		sensitiveCtrl := handler.NewSensitiveHandler(config)
+		sensitiveCtrl, err := handler.NewSensitiveHandler(config)
+		if err != nil {
+			return nil, fmt.Errorf("error creating sensitive handler:%w", err)
+		}
 		apiGroup.POST("/sensitive/text", sensitiveCtrl.Text)
 		apiGroup.POST("/sensitive/image", sensitiveCtrl.Image)
 	}