ISSUE-1913-2: Changes to support custom scheduled jobs

deepfence_server/apiDocs/operation.go

@@ -758,6 +758,9 @@ func (d *OpenApiDocs) AddSettingsOperations() {
 	d.AddOperation("addScheduledTask", http.MethodPost, "/deepfence/scheduled-task",
 		"Add scheduled task", "Add scheduled task",
 		http.StatusNoContent, []string{tagSettings}, bearerToken, new(AddScheduledTaskRequest), nil)
+	d.AddOperation("deleteCustomScheduledTask", http.MethodDelete, "/deepfence/scheduled-task/{id}",
+		"Delete Custom Schedule task", "Delete Custom Schedule task",
+		http.StatusNoContent, []string{tagSettings}, bearerToken, new(ScheduleJobId), nil)
 
 	// Database upload
 	d.AddOperation("uploadVulnerabilityDatabase", http.MethodPut, "/deepfence/database/vulnerability",

deepfence_server/handler/scheduled_tasks.go

@@ -46,6 +46,21 @@ func (h *Handler) UpdateScheduledTask(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(http.StatusNoContent)
 }
 
+func (h *Handler) DeleteCustomScheduledTask(w http.ResponseWriter, r *http.Request) {
+	id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
+	if err != nil {
+		h.respondError(&BadDecoding{err}, w)
+		return
+	}
+	defer r.Body.Close()
+	err = model.DeleteCustomSchedule(r.Context(), id)
+	if err != nil {
+		h.respondError(err, w)
+		return
+	}
+	w.WriteHeader(http.StatusNoContent)
+}
+
 func (h *Handler) AddScheduledTask(w http.ResponseWriter, r *http.Request) {
 	defer r.Body.Close()
 

deepfence_server/model/scheduled_tasks.go

@@ -31,11 +31,20 @@ var (
 )
 
 type AddScheduledTaskRequest struct {
-	NodeType    string `json:"node_type"`
-	Action      string `json:"action"`
+	Action      string `json:"action" validate:"required,oneof=SecretScan VulnerabilityScan MalwareScan ComplianceScan CloudComplianceScan" required:"true" enum:"SecretScan,VulnerabilityScan,MalwareScan,ComplianceScan,CloudComplianceScan"`
 	Description string `json:"description"`
 	CronExpr    string `json:"cron_expr"`
-	Filters     string `json:"filters"`
+	ScheduleTaskPayload
+}
+
+type ScheduleTaskPayload struct {
+	ScanTriggerCommon
+	ScanConfigLanguages []VulnerabilityScanConfigLanguage `json:"scan_config" required:"true"`
+	ComplianceBenchmarkTypes
+}
+
+type ScheduleJobId struct {
+	ID int64 `path:"id"`
 }
 
 type UpdateScheduledTaskRequest struct {
@@ -76,15 +85,29 @@ func UpdateScheduledTask(ctx context.Context, id int64, updateScheduledTask Upda
 	})
 }
 
+func DeleteCustomSchedule(ctx context.Context, id int64) error {
+	pgClient, err := directory.PostgresClient(ctx)
+	if err != nil {
+		return err
+	}
+	return pgClient.DeleteCustomSchedule(ctx, id)
+}
+
 func AddScheduledTask(ctx context.Context, req AddScheduledTaskRequest) error {
 	pgClient, err := directory.PostgresClient(ctx)
 	if err != nil {
 		return err
 	}
-	payload := make(map[string]string)
-	payload["node_type"] = req.NodeType
-	payload["filters"] = req.Filters
-	payloadJson, _ := json.Marshal(payload)
+
+	payload := ScheduleTaskPayload{}
+	payload.NodeIds = req.NodeIds
+	payload.Filters = req.Filters
+	payload.ScanConfigLanguages = req.ScanConfigLanguages
+	payload.BenchmarkTypes = req.BenchmarkTypes
+	payloadJson, err := json.Marshal(payload)
+	if err != nil {
+		return err
+	}
 
 	params := postgresqlDb.CreateScheduleParams{}
 	params.Action = req.Action

deepfence_server/router/router.go

@@ -487,7 +487,9 @@ func SetupRoutes(r *chi.Mux, serverPort string, serveOpenapiDocs bool, ingestC c
 			r.Route("/scheduled-task", func(r chi.Router) {
 				r.Get("/", dfHandler.AuthHandler(ResourceAllUsers, PermissionRead, dfHandler.GetScheduledTask))
 				r.Patch("/{id}", dfHandler.AuthHandler(ResourceAllUsers, PermissionWrite, dfHandler.UpdateScheduledTask))
-				r.Post("/", dfHandler.AuthHandler(ResourceIntegration, PermissionWrite, dfHandler.AddScheduledTask))
+				r.Delete("/{id}", dfHandler.AuthHandler(ResourceAllUsers, PermissionDelete, dfHandler.DeleteCustomScheduledTask))
+
+				r.Post("/", dfHandler.AuthHandler(ResourceAllUsers, PermissionWrite, dfHandler.AddScheduledTask))
 			})
 
 			// Integration

deepfence_utils/postgresql/queries.sql

@@ -637,4 +637,10 @@ WHERE id = $6;
 -- name: DeleteSchedule :exec
 DELETE
 FROM scheduler
-WHERE id = $1;
+WHERE id = $1;
+
+-- name: DeleteCustomSchedule :exec
+DELETE
+FROM scheduler
+WHERE id = $1
+	AND is_system='f';

deepfence_worker/cronjobs/scheduled_tasks.go

@@ -3,6 +3,7 @@ package cronjobs
 import (
 	"context"
 	"encoding/json"
+	"strings"
 
 	"github.com/deepfence/ThreatMapper/deepfence_server/handler"
 	"github.com/deepfence/ThreatMapper/deepfence_server/model"
@@ -25,17 +26,24 @@ func RunScheduledTasks(ctx context.Context, task *asynq.Task) error {
 	}
 
 	log.Info().Msgf("RunScheduledTasks: %s", messagePayload["description"])
+	log.Info().Msgf("RunScheduledTasks: %v", messagePayload)
 
 	scheduleId := int64(messagePayload["id"].(float64))
 	jobStatus := "Success"
-	err := runScheduledTasks(ctx, messagePayload)
+	isSystem := messagePayload["is_system"].(bool)
+	var err error
+	if isSystem {
+		err = runSystemScheduledTasks(ctx, messagePayload)
+	} else {
+		err = runCustomScheduledTasks(ctx, messagePayload)
+	}
 	if err != nil {
 		jobStatus = err.Error()
-		log.Error().Msg("runScheduledTasks: " + err.Error())
+		log.Error().Msg("RunScheduledTasks: " + err.Error())
 	}
 	err = saveJobStatus(ctx, scheduleId, jobStatus)
 	if err != nil {
-		log.Error().Msg("runScheduledTasks saveJobStatus: " + err.Error())
+		log.Error().Msg("RunScheduledTasks saveJobStatus: " + err.Error())
 	}
 	return nil
 }
@@ -48,43 +56,17 @@ var (
 	}
 )
 
-func runScheduledTasks(ctx context.Context, messagePayload map[string]interface{}) error {
+func runSystemScheduledTasks(ctx context.Context, messagePayload map[string]interface{}) error {
 	payload := messagePayload["payload"].(map[string]interface{})
 	nodeType := payload["node_type"].(string)
 
-	var searchFilter reporters_search.SearchFilter
-	_, ok := payload["filters"]
-	if ok {
-		filters := payload["filters"].(string)
-		err := json.Unmarshal([]byte(filters), &searchFilter)
-		if err != nil {
-			log.Error().Msgf("Error Unmarshaling filter: %v", err)
-			return err
-		}
-
-		fieldsValues := searchFilter.Filters.ContainsFilter.FieldsValues
-		if fieldsValues == nil {
-			fieldsValues = make(map[string][]interface{})
-		}
-
-		if _, ok := fieldsValues["pseudo"]; !ok {
-			fieldsValues["pseudo"] = append(make([]interface{}, 0), false)
-		}
-
-		if _, ok := fieldsValues["active"]; !ok {
-			fieldsValues["active"] = append(make([]interface{}, 0), true)
-		}
-
-		searchFilter.Filters.ContainsFilter.FieldsValues = fieldsValues
-	} else {
-		searchFilter = reporters_search.SearchFilter{
-			InFieldFilter: []string{"node_id"},
-			Filters: reporters.FieldsFilters{
-				ContainsFilter: reporters.ContainsFilter{
-					FieldsValues: map[string][]interface{}{"pseudo": {false}, "active": {true}},
-				},
+	searchFilter := reporters_search.SearchFilter{
+		InFieldFilter: []string{"node_id"},
+		Filters: reporters.FieldsFilters{
+			ContainsFilter: reporters.ContainsFilter{
+				FieldsValues: map[string][]interface{}{"pseudo": {false}, "active": {true}},
 			},
-		}
+		},
 	}
 
 	extSearchFilter := reporters_search.SearchFilter{}
@@ -168,6 +150,75 @@ func runScheduledTasks(ctx context.Context, messagePayload map[string]interface{
 	return nil
 }
 
+func runCustomScheduledTasks(ctx context.Context, messagePayload map[string]interface{}) error {
+	var payload model.ScheduleTaskPayload
+	val := messagePayload["payload"].(map[string]interface{})
+	payloadRaw, err := json.Marshal(val)
+	if err != nil {
+		log.Error().Msgf("Failed to marshal the payload, error:%v", err)
+		return err
+	}
+
+	err = json.Unmarshal(payloadRaw, &payload)
+	if err != nil {
+		return err
+	}
+
+	nodeIds := payload.NodeIds
+	scanFilter := payload.Filters
+	scheduleJobId := int64(messagePayload["id"].(float64))
+
+	if len(nodeIds) == 0 {
+		log.Info().Msgf("No nodes found for CustomScheduledTasks, jobid:%d, description:%s",
+			scheduleJobId, messagePayload["description"])
+		return nil
+	}
+
+	scanTrigger := model.ScanTriggerCommon{NodeIds: nodeIds, Filters: scanFilter}
+
+	action := utils.Neo4jScanType(messagePayload["action"].(string))
+
+	switch action {
+	case utils.NEO4J_VULNERABILITY_SCAN:
+		binArgs := make(map[string]string, 0)
+		if payload.ScanConfigLanguages != nil && len(payload.ScanConfigLanguages) > 0 {
+			languages := []string{}
+			for _, language := range payload.ScanConfigLanguages {
+				languages = append(languages, language.Language)
+			}
+			binArgs["scan_type"] = strings.Join(languages, ",")
+		}
+
+		actionBuilder := handler.StartScanActionBuilder(ctx, ctl.StartVulnerabilityScan, binArgs)
+		_, _, err := handler.StartMultiScan(ctx, false, utils.NEO4J_VULNERABILITY_SCAN, scanTrigger, actionBuilder)
+		if err != nil {
+			return err
+		}
+	case utils.NEO4J_SECRET_SCAN:
+		actionBuilder := handler.StartScanActionBuilder(ctx, ctl.StartSecretScan, nil)
+		_, _, err := handler.StartMultiScan(ctx, false, utils.NEO4J_SECRET_SCAN, scanTrigger, actionBuilder)
+		if err != nil {
+			return err
+		}
+	case utils.NEO4J_MALWARE_SCAN:
+		actionBuilder := handler.StartScanActionBuilder(ctx, ctl.StartMalwareScan, nil)
+		_, _, err := handler.StartMultiScan(ctx, false, utils.NEO4J_MALWARE_SCAN, scanTrigger, actionBuilder)
+		if err != nil {
+			return err
+		}
+	case utils.NEO4J_COMPLIANCE_SCAN, utils.NEO4J_CLOUD_COMPLIANCE_SCAN:
+		if payload.BenchmarkTypes == nil || len(payload.BenchmarkTypes) == 0 {
+			log.Warn().Msgf("Invalid benchmarkType for compliance scan, job id: %d", scheduleJobId)
+			return nil
+		}
+		_, _, err := handler.StartMultiCloudComplianceScan(ctx, nodeIds, payload.BenchmarkTypes)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func saveJobStatus(ctx context.Context, scheduleId int64, jobStatus string) error {
 	pgClient, err := directory.PostgresClient(ctx)
 	if err != nil {

deepfence_worker/cronscheduler/scheduler.go

@@ -112,7 +112,7 @@ func (s *Scheduler) RemoveJobs(ctx context.Context) error {
 }
 
 func (s *Scheduler) updateScheduledJobs() {
-	ticker := time.NewTicker(15 * time.Minute)
+	ticker := time.NewTicker(1 * time.Minute)
 	defer ticker.Stop()
 
 	for range ticker.C {
@@ -158,12 +158,7 @@ func (s *Scheduler) addScheduledJobs(ctx context.Context) error {
 			newJobHashToId[jobHash] = scheduledJobs.jobHashToId[jobHash]
 			continue
 		}
-		var payload map[string]string
-		err = json.Unmarshal(schedule.Payload, &payload)
-		if err != nil {
-			log.Error().Msg("addScheduledJobs payload: " + err.Error())
-			continue
-		}
+		payload := schedule.Payload
 		jobId, err := s.cron.AddFunc(schedule.CronExpr, s.enqueueScheduledTask(namespace, schedule, payload))
 		if err != nil {
 			return err
@@ -172,7 +167,8 @@ func (s *Scheduler) addScheduledJobs(ctx context.Context) error {
 		newJobHashToId[jobHash] = jobId
 	}
 	for _, oldJobHash := range scheduledJobs.jobHashes {
-		if !sdkUtils.InSlice(oldJobHash, scheduledJobs.jobHashes) {
+		if !sdkUtils.InSlice(oldJobHash, newHashes) {
+			log.Info().Msgf("Removing job from cron: %s", oldJobHash)
 			s.cron.Remove(scheduledJobs.jobHashToId[oldJobHash])
 		}
 	}
@@ -322,7 +318,8 @@ func (s *Scheduler) Run() {
 	s.cron.Run()
 }
 
-func (s *Scheduler) enqueueScheduledTask(namespace directory.NamespaceID, schedule postgresqlDb.Scheduler, payload map[string]string) func() {
+func (s *Scheduler) enqueueScheduledTask(namespace directory.NamespaceID,
+	schedule postgresqlDb.Scheduler, payload json.RawMessage) func() {
 	log.Info().Msgf("Registering task: %s, %s for namespace %s", schedule.Description, schedule.CronExpr, namespace)
 	return func() {
 		log.Info().Msgf("Enqueuing task: %s, %s for namespace %s",
@@ -332,6 +329,7 @@ func (s *Scheduler) enqueueScheduledTask(namespace directory.NamespaceID, schedu
 			"id":          schedule.ID,
 			"payload":     payload,
 			"description": schedule.Description,
+			"is_system":   schedule.IsSystem,
 		}
 		messageJson, _ := json.Marshal(message)
 		worker, err := directory.Worker(directory.NewContextWithNameSpace(namespace))