[lint] CI def + fixes in deepfence_utils + worker/server adjustments

deepfence · Nov 28, 2023 · 14cdf10 · 14cdf10
1 parent 14c181e
commit 14cdf10
Show file tree

Hide file tree

Showing 89 changed files with 726 additions and 705 deletions.
diff --git a/.github/workflows/golang-linter.yaml b/.github/workflows/golang-linter.yaml
@@ -15,7 +15,6 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         with:
-          # token: ${{ secrets.CI_PAT }}
           submodules: recursive
           fetch-depth: "0"
       - uses: actions/setup-go@v4
@@ -28,3 +27,22 @@ jobs:
           version: v1.55
           only-new-issues: true
           working-directory: deepfence_bootstrapper
+
+  lint-utils:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: recursive
+          fetch-depth: "0"
+      - uses: actions/setup-go@v4
+        with:
+          go-version: '1.21'
+          cache: false
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v3
+        with:
+          version: v1.55
+          only-new-issues: true
+          working-directory: deepfence_utils
+          args: --timeout=30m
diff --git a/deepfence_agent/plugins/YaraHunter b/deepfence_agent/plugins/YaraHunter
diff --git a/deepfence_bootstrapper/controls/controls.go b/deepfence_bootstrapper/controls/controls.go
@@ -73,8 +73,8 @@ func SetAgentControls() {
 				linuxScannerUtil.Config{
 					ComplianceCheckTypes:      strings.Split(req.BinArgs["benchmark_types"], ","),
 					ScanID:                    req.BinArgs["scan_id"],
-					NodeID:                    req.NodeId,
-					NodeName:                  req.NodeId,
+					NodeID:                    req.NodeID,
+					NodeName:                  req.NodeID,
 					ComplianceResultsFilePath: fmt.Sprintf("/var/log/fenced/compliance/%s.log", req.BinArgs["scan_id"]),
 					ComplianceStatusFilePath:  "/var/log/fenced/compliance-scan-logs/status.log",
 				})
@@ -115,7 +115,7 @@ func SetAgentControls() {
 			log.Info().Msg("Start & download Agent Plugin")
 			router.SetUpgrade()
 			defer router.UnsetUpgrade()
-			err = supervisor.UpgradeProcessFromURL(req.PluginName, req.BinUrl)
+			err = supervisor.UpgradeProcessFromURL(req.PluginName, req.BinURL)
 			if err != nil {
 				return err
 			}

diff --git a/deepfence_bootstrapper/controls/diagnostic_logs.go b/deepfence_bootstrapper/controls/diagnostic_logs.go
@@ -23,22 +23,22 @@ func SendAgentDiagnosticLogs(req ctl.SendAgentDiagnosticLogsRequest, pathsToZip
 	fileName := "/tmp/" + req.FileName
 	err = utils.RecursiveZip(pathsToZip, excludePathPrefixes, fileName)
 	if err != nil {
-		_ = publishDiagnosticLogsStatus(ctx, httpsClient.Client(), req.NodeId, utils.SCAN_STATUS_FAILED, err.Error())
+		_ = publishDiagnosticLogsStatus(ctx, httpsClient.Client(), req.NodeID, utils.ScanStatusFailed, err.Error())
 		return err
 	}
 	defer os.RemoveAll(fileName)
 
 	resp, statusCode, err := utils.UploadFile(req.UploadURL, fileName)
 	if err != nil {
-		_ = publishDiagnosticLogsStatus(ctx, httpsClient.Client(), req.NodeId, utils.SCAN_STATUS_FAILED, err.Error())
+		_ = publishDiagnosticLogsStatus(ctx, httpsClient.Client(), req.NodeID, utils.ScanStatusFailed, err.Error())
 		return err
 	}
 	if statusCode != http.StatusOK {
-		_ = publishDiagnosticLogsStatus(ctx, httpsClient.Client(), req.NodeId, utils.SCAN_STATUS_FAILED, string(resp))
+		_ = publishDiagnosticLogsStatus(ctx, httpsClient.Client(), req.NodeID, utils.ScanStatusFailed, string(resp))
 		return errors.New(string(resp))
 	}
 
-	return publishDiagnosticLogsStatus(ctx, httpsClient.Client(), req.NodeId, utils.SCAN_STATUS_SUCCESS, "")
+	return publishDiagnosticLogsStatus(ctx, httpsClient.Client(), req.NodeID, utils.ScanStatusSuccess, "")
 }
 
 func publishDiagnosticLogsStatus(ctx context.Context, httpsClient *client.APIClient, nodeID string, status string, message string) error {

diff --git a/deepfence_bootstrapper/controls/kubernetes.go b/deepfence_bootstrapper/controls/kubernetes.go
@@ -14,8 +14,8 @@ func StartComplianceScan(req ctl.StartComplianceScanRequest) error {
 		k8sscannerutil.Config{
 			ComplianceCheckType:       k8sscannerutil.NsaCisaCheckType,
 			ScanId:                    req.BinArgs["scan_id"],
-			NodeId:                    req.NodeId,
-			NodeName:                  req.NodeId,
+			NodeId:                    req.NodeID,
+			NodeName:                  req.NodeID,
 			ComplianceResultsFilePath: fmt.Sprintf("/var/log/fenced/compliance/%s.log", req.BinArgs["scan_id"]),
 			ComplianceStatusFilePath:  "/var/log/fenced/compliance-scan-logs/status.log",
 		})

diff --git a/deepfence_bootstrapper/router/upgrade.go b/deepfence_bootstrapper/router/upgrade.go
@@ -21,8 +21,8 @@ const (
 )
 
 func StartAgentUpgrade(req ctl.StartAgentUpgradeRequest) error {
-	log.Info().Msgf("Fetching %v", req.HomeDirectoryUrl)
-	err := downloadFile(binariesFile, req.HomeDirectoryUrl)
+	log.Info().Msgf("Fetching %v", req.HomeDirectoryURL)
+	err := downloadFile(binariesFile, req.HomeDirectoryURL)
 	if err != nil {
 		return err
 	}

diff --git a/deepfence_server/controls/agent.go b/deepfence_server/controls/agent.go
@@ -92,7 +92,7 @@ func GetPendingAgentScans(ctx context.Context, nodeId string, availableWorkload
 
 	r, err := tx.Run(`
 		MATCH (s) -[:SCHEDULED]-> (n:Node{node_id:$id})
-		WHERE s.status = '`+utils.SCAN_STATUS_INPROGRESS+`'
+		WHERE s.status = '`+utils.ScanStatusInProgress+`'
 		AND s.retries < 3
 		SET s.retries = s.retries + 1
 		WITH s
@@ -145,7 +145,7 @@ func hasAgentDiagnosticLogRequests(client neo4j.Driver, nodeId string, nodeType
 
 	r, err := tx.Run(`MATCH (s:AgentDiagnosticLogs) -[:SCHEDULEDLOGS]-> (n{node_id:$id})
 		WHERE (n:`+controls.ResourceTypeToNeo4j(nodeType)+`)
-		AND s.status = '`+utils.SCAN_STATUS_STARTING+`'
+		AND s.status = '`+utils.ScanStatusStarting+`'
 		AND s.retries < 3
 		WITH s LIMIT $max_work
 		WITH s
@@ -186,10 +186,10 @@ func ExtractAgentDiagnosticLogRequests(ctx context.Context, nodeId string, nodeT
 
 	r, err := tx.Run(`MATCH (s:AgentDiagnosticLogs) -[:SCHEDULEDLOGS]-> (n{node_id:$id})
 		WHERE (n:`+controls.ResourceTypeToNeo4j(nodeType)+`)
-		AND s.status = '`+utils.SCAN_STATUS_STARTING+`'
+		AND s.status = '`+utils.ScanStatusStarting+`'
 		AND s.retries < 3
 		WITH s LIMIT $max_work
-		SET s.status = '`+utils.SCAN_STATUS_INPROGRESS+`'
+		SET s.status = '`+utils.ScanStatusInProgress+`'
 		WITH s
 		RETURN s.trigger_action`,
 		map[string]interface{}{"id": nodeId, "max_work": max_work})
@@ -234,7 +234,7 @@ func hasPendingAgentScans(client neo4j.Driver, nodeId string, max_work int) (boo
 	defer tx.Close()
 
 	r, err := tx.Run(`MATCH (s) -[:SCHEDULED]-> (n:Node{node_id:$id})
-		WHERE s.status = '`+utils.SCAN_STATUS_STARTING+`'
+		WHERE s.status = '`+utils.ScanStatusStarting+`'
 		AND s.retries < 3
 		WITH s LIMIT $max_work
 		RETURN s.trigger_action`,
@@ -275,10 +275,10 @@ func ExtractStartingAgentScans(ctx context.Context, nodeId string,
 	defer tx.Close()
 
 	r, err := tx.Run(`MATCH (s) -[:SCHEDULED]-> (n:Node{node_id:$id})
-		WHERE s.status = '`+utils.SCAN_STATUS_STARTING+`'
+		WHERE s.status = '`+utils.ScanStatusStarting+`'
 		AND s.retries < 3
 		WITH s ORDER BY s.is_priority DESC, s.updated_at ASC LIMIT $max_work
-		SET s.status = '`+utils.SCAN_STATUS_INPROGRESS+`', s.updated_at = TIMESTAMP()
+		SET s.status = '`+utils.ScanStatusInProgress+`', s.updated_at = TIMESTAMP()
 		WITH s
 		RETURN s.trigger_action`,
 		map[string]interface{}{"id": nodeId, "max_work": max_work})
@@ -338,9 +338,9 @@ func ExtractStoppingAgentScans(ctx context.Context, nodeId string,
 	defer tx.Close()
 
 	r, err := tx.Run(`MATCH (s) -[:SCHEDULED]-> (n:Node{node_id:$id})
-		WHERE s.status = '`+utils.SCAN_STATUS_CANCEL_PENDING+`'
+		WHERE s.status = '`+utils.ScanStatusCancelPending+`'
 		WITH s LIMIT $max_work
-        SET s.status = '`+utils.SCAN_STATUS_CANCELLING+`', s.updated_at = TIMESTAMP()
+        SET s.status = '`+utils.ScanStatusCancelling+`', s.updated_at = TIMESTAMP()
 		WITH s
 		RETURN s.trigger_action`,
 		map[string]interface{}{"id": nodeId, "max_work": max_work})
@@ -401,7 +401,7 @@ func hasPendingAgentUpgrade(client neo4j.Driver, nodeId string, max_work int) (b
 	defer tx.Close()
 
 	r, err := tx.Run(`MATCH (s:AgentVersion) -[r:SCHEDULED]-> (n:Node{node_id:$id})
-		WHERE r.status = '`+utils.SCAN_STATUS_STARTING+`'
+		WHERE r.status = '`+utils.ScanStatusStarting+`'
 		AND r.retries < 3
 		WITH r LIMIT $max_work
 		RETURN r.trigger_action`,
@@ -440,10 +440,10 @@ func ExtractPendingAgentUpgrade(ctx context.Context, nodeId string, max_work int
 	defer tx.Close()
 
 	r, err := tx.Run(`MATCH (s:AgentVersion) -[r:SCHEDULED]-> (n:Node{node_id:$id})
-		WHERE r.status = '`+utils.SCAN_STATUS_STARTING+`'
+		WHERE r.status = '`+utils.ScanStatusStarting+`'
 		AND r.retries < 3
 		WITH r LIMIT $max_work
-		SET r.status = '`+utils.SCAN_STATUS_INPROGRESS+`'
+		SET r.status = '`+utils.ScanStatusInProgress+`'
 		WITH r
 		RETURN r.trigger_action`,
 		map[string]interface{}{"id": nodeId, "max_work": max_work})
@@ -516,5 +516,4 @@ func CheckNodeExist(ctx context.Context, nodeId string) error {
 	}
 
 	return nil
-
 }
diff --git a/deepfence_server/controls/kubernetes_cluster.go b/deepfence_server/controls/kubernetes_cluster.go
@@ -34,7 +34,8 @@ func GetKubernetesClusterActions(ctx context.Context, nodeId string, workNumToEx
 	}
 
 	diagnosticLogActions, scan_err := ExtractAgentDiagnosticLogRequests(ctx, nodeId, controls.KubernetesCluster, workNumToExtract)
-	workNumToExtract -= len(diagnosticLogActions)
+
+	workNumToExtract -= len(diagnosticLogActions) //nolint:ineffassign
 	if scan_err == nil {
 		actions = append(actions, diagnosticLogActions...)
 	}
@@ -66,10 +67,10 @@ func ExtractStartingKubernetesClusterScans(ctx context.Context, nodeId string, m
 	defer tx.Close()
 
 	r, err := tx.Run(`MATCH (s) -[:SCHEDULED]-> (n:KubernetesCluster{node_id:$id})
-		WHERE s.status = '`+utils.SCAN_STATUS_STARTING+`'
+		WHERE s.status = '`+utils.ScanStatusStarting+`'
 		AND s.retries < 3
 		WITH s LIMIT $max_work
-		SET s.status = '`+utils.SCAN_STATUS_INPROGRESS+`'
+		SET s.status = '`+utils.ScanStatusInProgress+`'
 		WITH s
 		RETURN s.trigger_action`,
 		map[string]interface{}{"id": nodeId, "max_work": max_work})
@@ -123,10 +124,10 @@ func ExtractPendingKubernetesClusterUpgrade(ctx context.Context, nodeId string,
 	defer tx.Close()
 
 	r, err := tx.Run(`MATCH (s:AgentVersion) -[r:SCHEDULED]-> (n:KubernetesCluster{node_id:$id})
-		WHERE r.status = '`+utils.SCAN_STATUS_STARTING+`'
+		WHERE r.status = '`+utils.ScanStatusStarting+`'
 		AND r.retries < 3
 		WITH r LIMIT $max_work
-		SET r.status = '`+utils.SCAN_STATUS_INPROGRESS+`'
+		SET r.status = '`+utils.ScanStatusInProgress+`'
 		WITH r
 		RETURN r.trigger_action`,
 		map[string]interface{}{"id": nodeId, "max_work": max_work})

diff --git a/deepfence_server/controls/version.go b/deepfence_server/controls/version.go
@@ -28,7 +28,7 @@ func PrepareAgentUpgradeAction(ctx context.Context, version string) (ctl.Action,
 	}
 
 	internal_req := ctl.StartAgentUpgradeRequest{
-		HomeDirectoryUrl: url,
+		HomeDirectoryURL: url,
 		Version:          version,
 	}
 
@@ -72,7 +72,7 @@ func ScheduleAgentUpgrade(ctx context.Context, version string, nodeIds []string,
 		map[string]interface{}{
 			"version":  version,
 			"node_ids": nodeIds,
-			"status":   utils.SCAN_STATUS_STARTING,
+			"status":   utils.ScanStatusStarting,
 			"action":   string(action_str),
 		})
 
@@ -343,7 +343,7 @@ func ScheduleAgentPluginEnable(ctx context.Context, version, plugin_name string,
 		map[string]interface{}{
 			"version":  version,
 			"node_ids": nodeIds,
-			"status":   utils.SCAN_STATUS_STARTING,
+			"status":   utils.ScanStatusStarting,
 			"action":   string(action_str),
 		})
 
@@ -385,7 +385,7 @@ func ScheduleAgentPluginDisable(ctx context.Context, plugin_name string, nodeIds
 	_, err = tx.Run(query,
 		map[string]interface{}{
 			"node_ids": nodeIds,
-			"status":   utils.SCAN_STATUS_STARTING,
+			"status":   utils.ScanStatusStarting,
 			"action":   string(action_str),
 		})
 

diff --git a/deepfence_server/diagnosis/agent-diagnosis/agent_diagnosis.go b/deepfence_server/diagnosis/agent-diagnosis/agent_diagnosis.go
@@ -43,8 +43,8 @@ func verifyNodeIds(ctx context.Context, nodeIdentifiers []diagnosis.NodeIdentifi
 		WHERE NOT a.status = $complete AND NOT a.status = $failed
 		RETURN n.node_id,a.status`,
 		map[string]interface{}{"node_ids": nodeIds,
-			"complete": utils.SCAN_STATUS_SUCCESS,
-			"failed":   utils.SCAN_STATUS_FAILED})
+			"complete": utils.ScanStatusSuccess,
+			"failed":   utils.ScanStatusFailed})
 	if err != nil {
 		return inProgressNodeIds, err
 	}
@@ -111,7 +111,7 @@ func GenerateAgentDiagnosticLogs(ctx context.Context, nodeIdentifiers []diagnosi
 
 	actionBuilder := func(nodeIdentifier diagnosis.NodeIdentifier, uploadUrl string, fileName string, tail string) (ctl.Action, error) {
 		req := ctl.SendAgentDiagnosticLogsRequest{
-			NodeId:    nodeIdentifier.NodeId,
+			NodeID:    nodeIdentifier.NodeId,
 			NodeType:  ctl.StringToResourceType(nodeIdentifier.NodeType),
 			UploadURL: uploadUrl,
 			Tail:      tail,
@@ -168,7 +168,7 @@ func GenerateAgentDiagnosticLogs(ctx context.Context, nodeIdentifiers []diagnosi
 		MERGE (m:%s{node_id:$node_id})
 		MERGE (n)-[:SCHEDULEDLOGS]->(m)`, controls.ResourceTypeToNeo4j(controls.StringToResourceType(nodeIdentifier.NodeType))),
 			map[string]interface{}{
-				"status":          utils.SCAN_STATUS_STARTING,
+				"status":          utils.ScanStatusStarting,
 				"node_id":         nodeIdentifier.NodeId,
 				"action":          string(b),
 				"minio_file_name": fileName,

diff --git a/deepfence_server/diagnosis/cloudscanner-diagnosis/cloudscanner_diagnosis.go b/deepfence_server/diagnosis/cloudscanner-diagnosis/cloudscanner_diagnosis.go
@@ -45,8 +45,8 @@ func getInProgressCloudScannerNodeIds(ctx context.Context, nodeIdentifiers []dia
 		RETURN n.node_id,a.status`,
 		map[string]interface{}{
 			"node_ids": nodeIds,
-			"complete": utils.SCAN_STATUS_SUCCESS,
-			"failed":   utils.SCAN_STATUS_FAILED})
+			"complete": utils.ScanStatusSuccess,
+			"failed":   utils.ScanStatusFailed})
 	if err != nil {
 		return inProgressNodeIds, err
 	}
@@ -114,7 +114,7 @@ func GenerateCloudScannerDiagnosticLogs(ctx context.Context, nodeIdentifiers []d
 
 	actionBuilder := func(nodeIdentifier diagnosis.NodeIdentifier, uploadUrl string, fileName string, tail string) (ctl.Action, error) {
 		req := ctl.SendAgentDiagnosticLogsRequest{
-			NodeId:    nodeIdentifier.NodeId,
+			NodeID:    nodeIdentifier.NodeId,
 			NodeType:  ctl.StringToResourceType(nodeIdentifier.NodeType),
 			UploadURL: uploadUrl,
 			Tail:      tail,
@@ -167,7 +167,7 @@ func GenerateCloudScannerDiagnosticLogs(ctx context.Context, nodeIdentifiers []d
 		MERGE (m:%s{node_id:$node_id})
 		MERGE (n)-[:SCHEDULEDLOGS]->(m)`, controls.ResourceTypeToNeo4j(controls.StringToResourceType(nodeIdentifier.NodeType))),
 			map[string]interface{}{
-				"status":          utils.SCAN_STATUS_STARTING,
+				"status":          utils.ScanStatusStarting,
 				"node_id":         nodeIdentifier.NodeId,
 				"action":          string(b),
 				"minio_file_name": fileName,
@@ -197,7 +197,7 @@ func GetQueuedCloudScannerDiagnosticLogs(ctx context.Context, nodeIDs []string)
 		WHERE n.status = $status and n.node_id in $node_ids
 		RETURN n.trigger_action
 		ORDER BY n.updated_at ASC LIMIT 1`,
-		map[string]interface{}{"status": utils.SCAN_STATUS_STARTING, "node_ids": nodeIDs})
+		map[string]interface{}{"status": utils.ScanStatusStarting, "node_ids": nodeIDs})
 
 	if err != nil {
 		return ctl.Action{}, err

diff --git a/deepfence_server/diagnosis/common.go b/deepfence_server/diagnosis/common.go
@@ -176,7 +176,7 @@ func getAgentDiagnosticLogs(ctx context.Context, mc directory.FileManager, pathP
 		}
 		updatedAtTime := time.UnixMilli(updatedAt.(int64))
 		nodeIdToName[nodeId.(string)] = nodeName.(string)
-		if message.(string) == "" && status.(string) != utils.SCAN_STATUS_SUCCESS {
+		if message.(string) == "" && status.(string) != utils.ScanStatusSuccess {
 			message = status.(string)
 		}
 
@@ -255,7 +255,7 @@ func getCloudScannerDiagnosticLogs(ctx context.Context, mc directory.FileManager
 		}
 		updatedAtTime := time.UnixMilli(updatedAt.(int64))
 		nodeIdToName[nodeId.(string)] = nodeName.(string)
-		if message.(string) == "" && status.(string) != utils.SCAN_STATUS_SUCCESS {
+		if message.(string) == "" && status.(string) != utils.ScanStatusSuccess {
 			message = status.(string)
 		}
 

diff --git a/deepfence_server/handler/agent_controls.go b/deepfence_server/handler/agent_controls.go
@@ -143,7 +143,7 @@ func (h *Handler) ScheduleAgentPluginsEnable(w http.ResponseWriter, r *http.Requ
 	}
 
 	internal_req := ctl.EnableAgentPluginRequest{
-		BinUrl:     url,
+		BinURL:     url,
 		Version:    agentUp.Version,
 		PluginName: agentUp.PluginName,
 	}

diff --git a/deepfence_server/handler/agent_upload.go b/deepfence_server/handler/agent_upload.go
@@ -191,7 +191,7 @@ func ScheduleAutoUpgradeForPatchChanges(ctx context.Context, latest map[string]s
 		AND v.node_id <> row.latest
 		MERGE (vnew) -[:SCHEDULED{status: $status, retries: 0, trigger_action: row.action, updated_at: TIMESTAMP()}]-> (n)`,
 		map[string]interface{}{
-			"status": utils.SCAN_STATUS_STARTING,
+			"status": utils.ScanStatusStarting,
 			"batch":  tags_to_ingest}); err != nil {
 		return err
 	}

diff --git a/deepfence_server/handler/audit_log.go b/deepfence_server/handler/audit_log.go
@@ -19,10 +19,10 @@ import (
 )
 
 const (
-	EVENT_COMPLIANCE_SCAN           = string(utils.NEO4J_COMPLIANCE_SCAN)
-	EVENT_VULNERABILITY_SCAN        = string(utils.NEO4J_VULNERABILITY_SCAN)
-	EVENT_SECRET_SCAN               = string(utils.NEO4J_SECRET_SCAN)
-	EVENT_MALWARE_SCAN              = string(utils.NEO4J_MALWARE_SCAN)
+	EVENT_COMPLIANCE_SCAN           = string(utils.NEO4JComplianceScan)
+	EVENT_VULNERABILITY_SCAN        = string(utils.NEO4JVulnerabilityScan)
+	EVENT_SECRET_SCAN               = string(utils.NEO4JSecretScan)
+	EVENT_MALWARE_SCAN              = string(utils.NEO4JMalwareScan)
 	EVENT_INTEGRATION               = "integration"
 	EVENT_GENERATIVE_AI_INTEGRATION = "generative-ai-integration"
 	EVENT_AUTH                      = "auth"
@@ -152,7 +152,7 @@ func (h *Handler) AddAuditLog(namespace string, params postgresql_db.CreateAudit
 	}
 
 	h.IngestChan <- &kgo.Record{
-		Topic: utils.AUDIT_LOGS,
+		Topic: utils.AuditLogs,
 		Value: data,
 		Headers: []kgo.RecordHeader{
 			{Key: "namespace", Value: []byte(namespace)},