Change list/results query requests to post #728

Add output to deepfence_ctl
Change stdout to stderr in utils/logs

deepfence_ctl/cmd/auth.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/spf13/cobra"
 
+	"github.com/deepfence/ThreatMapper/deepfence_ctl/output"
 	oahttp "github.com/deepfence/ThreatMapper/deepfence_utils/http"
 	"github.com/deepfence/ThreatMapper/deepfence_utils/log"
 )
@@ -56,7 +57,7 @@ var authCmd = &cobra.Command{
 			log.Fatal().Msgf("Failed to authenticate %v\n", err)
 		}
 
-		log.Info().Msgf("Successful login")
+		output.Out(map[string]string{"login": "successful"})
 	},
 }
 

deepfence_ctl/cmd/scan.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/spf13/cobra"
 
+	"github.com/deepfence/ThreatMapper/deepfence_ctl/output"
 	"github.com/deepfence/ThreatMapper/deepfence_server_client"
 	ctl "github.com/deepfence/ThreatMapper/deepfence_utils/controls"
 	oahttp "github.com/deepfence/ThreatMapper/deepfence_utils/http"
@@ -74,7 +75,7 @@ var scanStartSubCmd = &cobra.Command{
 		if err != nil {
 			log.Fatal().Msgf("Fail to execute: %v", err)
 		}
-		log.Info().Msgf("Scan Id: %s", res.ScanId)
+		output.Out(res)
 	},
 }
 
@@ -109,7 +110,7 @@ var scanStatusSubCmd = &cobra.Command{
 		if err != nil {
 			log.Fatal().Msgf("Fail to execute: %v", err)
 		}
-		log.Info().Msgf("Scan Id: %s, Status: %s", scan_id, res.GetStatus())
+		output.Out(res)
 	},
 }
 
@@ -135,10 +136,12 @@ var scanListSubCmd = &cobra.Command{
 		switch scan_type {
 		case "secret":
 			req := https_client.Client().SecretScanApi.ListSecretScan(context.Background())
-			req = req.NodeId(node_id)
-			req = req.Window(deepfence_server_client.ModelFetchWindow{
-				Offset: 0,
-				Size:   20,
+			req = req.ModelScanListReq(deepfence_server_client.ModelScanListReq{
+				NodeId: node_id,
+				Window: deepfence_server_client.ModelFetchWindow{
+					Offset: 0,
+					Size:   20,
+				},
 			})
 			res, _, err = https_client.Client().SecretScanApi.ListSecretScanExecute(req)
 		default:
@@ -148,7 +151,7 @@ var scanListSubCmd = &cobra.Command{
 		if err != nil {
 			log.Fatal().Msgf("Fail to execute: %v", err)
 		}
-		log.Info().Msgf("%v", node_id, res.ScansInfo)
+		output.Out(res)
 	},
 }
 
@@ -174,10 +177,12 @@ var scanResultsSubCmd = &cobra.Command{
 		switch scan_type {
 		case "secret":
 			req := https_client.Client().SecretScanApi.ResultsSecretScan(context.Background())
-			req = req.ScanId(scan_id)
-			req = req.Window(deepfence_server_client.ModelFetchWindow{
-				Offset: 0,
-				Size:   20,
+			req = req.ModelScanResultsReq(deepfence_server_client.ModelScanResultsReq{
+				ScanId: scan_id,
+				Window: deepfence_server_client.ModelFetchWindow{
+					Offset: 0,
+					Size:   20,
+				},
 			})
 			res, _, err = https_client.Client().SecretScanApi.ResultsSecretScanExecute(req)
 		default:
@@ -187,7 +192,7 @@ var scanResultsSubCmd = &cobra.Command{
 		if err != nil {
 			log.Fatal().Msgf("Fail to execute: %v", err)
 		}
-		log.Info().Msgf("%v", res)
+		output.Out(res)
 	},
 }
 

deepfence_ctl/output/common.go

@@ -0,0 +1,30 @@
+package output
+
+import (
+	"os"
+
+	"github.com/deepfence/ThreatMapper/deepfence_utils/log"
+)
+
+var format string
+
+func Out[T any](t T) {
+	var err error
+	switch format {
+	case "json":
+		err = out_json(t)
+	default:
+		log.Error().Msgf("Output format %s not supported", format)
+	}
+	if err != nil {
+		log.Error().Msgf("Could not marshal %v into %v format: %v", t, format, err)
+	}
+}
+
+func init() {
+	format = os.Getenv("DEEPFENCE_CTL_OUT_FORMAT")
+	if format == "" {
+		format = "json"
+		log.Warn().Msgf("Using default output format %s", format)
+	}
+}

deepfence_ctl/output/json.go

@@ -0,0 +1,15 @@
+package output
+
+import (
+	"encoding/json"
+	"fmt"
+)
+
+func out_json[T any](t T) error {
+	b, err := json.Marshal(t)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("%s\n", string(b))
+	return nil
+}

deepfence_server/apiDocs/operation.go

@@ -141,30 +141,30 @@ func (d *OpenApiDocs) AddScansOperations() {
 		http.StatusOK, []string{tagMalwareScan}, bearerToken, new(model.ScanStatusReq), new(model.ScanStatusResp))
 
 	// List scans
-	d.AddOperation("listVulnerabilityScans", http.MethodGet, "/deepfence/scan/list/vulnerability",
+	d.AddOperation("listVulnerabilityScans", http.MethodPost, "/deepfence/scan/list/vulnerability",
 		"Get Vulnerability Scans List", "Get Vulnerability Scan list on agent or registry",
 		http.StatusOK, []string{tagVulnerability}, bearerToken, new(model.ScanListReq), new(model.ScanListResp))
-	d.AddOperation("listSecretScan", http.MethodGet, "/deepfence/scan/list/secret",
+	d.AddOperation("listSecretScan", http.MethodPost, "/deepfence/scan/list/secret",
 		"Get Secret Scans List", "Get Secret Scans list on agent or registry",
 		http.StatusOK, []string{tagSecretScan}, bearerToken, new(model.ScanListReq), new(model.ScanListResp))
-	d.AddOperation("listComplianceScan", http.MethodGet, "/deepfence/scan/list/compliance",
+	d.AddOperation("listComplianceScan", http.MethodPost, "/deepfence/scan/list/compliance",
 		"Get Compliance Scans List", "Get Compliance Scans list on agent or registry",
 		http.StatusOK, []string{tagCompliance}, bearerToken, new(model.ScanListReq), new(model.ScanListResp))
-	d.AddOperation("listMalwareScan", http.MethodGet, "/deepfence/scan/list/malware",
+	d.AddOperation("listMalwareScan", http.MethodPost, "/deepfence/scan/list/malware",
 		"Get Malware Scans List", "Get Malware Scans list on agent or registry",
 		http.StatusOK, []string{tagMalwareScan}, bearerToken, new(model.ScanListReq), new(model.ScanListResp))
 
 	// Scans' Results
-	d.AddOperation("resultsVulnerabilityScans", http.MethodGet, "/deepfence/scan/results/vulnerability",
+	d.AddOperation("resultsVulnerabilityScans", http.MethodPost, "/deepfence/scan/results/vulnerability",
 		"Get Vulnerability Scans Results", "Get Vulnerability Scan results on agent or registry",
 		http.StatusOK, []string{tagVulnerability}, bearerToken, new(model.ScanResultsReq), new(model.ScanResultsResp))
-	d.AddOperation("resultsSecretScan", http.MethodGet, "/deepfence/scan/results/secret",
+	d.AddOperation("resultsSecretScan", http.MethodPost, "/deepfence/scan/results/secret",
 		"Get Secret Scans Results", "Get Secret Scans results on agent or registry",
 		http.StatusOK, []string{tagSecretScan}, bearerToken, new(model.ScanResultsReq), new(model.ScanResultsResp))
-	d.AddOperation("resultsComplianceScan", http.MethodGet, "/deepfence/scan/results/compliance",
+	d.AddOperation("resultsComplianceScan", http.MethodPost, "/deepfence/scan/results/compliance",
 		"Get Compliance Scans Results", "Get Compliance Scans results on agent or registry",
 		http.StatusOK, []string{tagCompliance}, bearerToken, new(model.ScanResultsReq), new(model.ScanResultsResp))
-	d.AddOperation("resultsMalwareScan", http.MethodGet, "/deepfence/scan/results/malware",
+	d.AddOperation("resultsMalwareScan", http.MethodPost, "/deepfence/scan/results/malware",
 		"Get Malware Scans Results", "Get Malware Scans results on agent or registry",
 		http.StatusOK, []string{tagMalwareScan}, bearerToken, new(model.ScanResultsReq), new(model.ScanResultsResp))
 }

deepfence_server/handler/scan_reports.go

@@ -405,7 +405,7 @@ func (h *Handler) ListMalwareScansHandler(w http.ResponseWriter, r *http.Request
 func listScansHandler(w http.ResponseWriter, r *http.Request, scan_type utils.Neo4jScanType) {
 	defer r.Body.Close()
 	var req model.ScanListReq
-	err := httpext.DecodeQueryParams(r, &req)
+	err := httpext.DecodeJSON(r, httpext.NoQueryParams, MaxPostRequestSize, &req)
 	if err != nil {
 		log.Error().Msgf("%v", err)
 		httpext.JSON(w, http.StatusBadRequest, model.Response{Success: false})
@@ -441,7 +441,7 @@ func (h *Handler) ListMalwareScanResultsHandler(w http.ResponseWriter, r *http.R
 func listScanResultsHandler(w http.ResponseWriter, r *http.Request, scan_type utils.Neo4jScanType) {
 	defer r.Body.Close()
 	var req model.ScanResultsReq
-	err := httpext.DecodeQueryParams(r, &req)
+	err := httpext.DecodeJSON(r, httpext.NoQueryParams, MaxPostRequestSize, &req)
 	if err != nil {
 		log.Error().Msgf("%v", err)
 		httpext.JSON(w, http.StatusBadRequest, model.Response{Success: false})

deepfence_server/model/scans.go

@@ -15,7 +15,7 @@ type ScanStatus string
 type ScanInfo struct {
 	ScanId    string `json:"scan_id" required:"true"`
 	Status    string `json:"status" required:"true"`
-	UpdatedAt int64  `json:"updated_at" required:"true"`
+	UpdatedAt int64  `json:"updated_at" required:"true" format:"int64"`
 }
 
 const (
@@ -37,17 +37,17 @@ type ScanStatusResp struct {
 }
 
 type ScanListReq struct {
-	NodeId string      `query:"node_id" form:"node_id" required:"true"`
-	Window FetchWindow `query:"window"  form:"window" required:"true"`
+	NodeId string      `json:"node_id" required:"true"`
+	Window FetchWindow `json:"window"  required:"true"`
 }
 
 type ScanListResp struct {
 	ScansInfo []ScanInfo `json:"scans_info" required:"true"`
 }
 
 type ScanResultsReq struct {
-	ScanId string      `query:"scan_id" form:"scan_id" required:"true"`
-	Window FetchWindow `query:"window" form:"window" required:"true"`
+	ScanId string      `json:"scan_id" required:"true"`
+	Window FetchWindow `json:"window"  required:"true"`
 }
 
 type ScanResultsResp struct {

deepfence_server/router/router.go

@@ -168,16 +168,16 @@ func SetupRoutes(r *chi.Mux, serverPort string, jwtSecret []byte, serveOpenapiDo
 				r.Get("/malware", dfHandler.AuthHandler(ResourceScan, PermissionStop, dfHandler.StatusMalwareScanHandler))
 			})
 			r.Route("/scan/list", func(r chi.Router) {
-				r.Get("/vulnerability", dfHandler.AuthHandler(ResourceScan, PermissionStop, dfHandler.ListVulnerabilityScansHandler))
-				r.Get("/secret", dfHandler.AuthHandler(ResourceScan, PermissionStop, dfHandler.ListSecretScansHandler))
-				r.Get("/compliance", dfHandler.AuthHandler(ResourceScan, PermissionStop, dfHandler.ListComplianceScansHandler))
-				r.Get("/malware", dfHandler.AuthHandler(ResourceScan, PermissionStop, dfHandler.ListMalwareScansHandler))
+				r.Post("/vulnerability", dfHandler.AuthHandler(ResourceScan, PermissionStop, dfHandler.ListVulnerabilityScansHandler))
+				r.Post("/secret", dfHandler.AuthHandler(ResourceScan, PermissionStop, dfHandler.ListSecretScansHandler))
+				r.Post("/compliance", dfHandler.AuthHandler(ResourceScan, PermissionStop, dfHandler.ListComplianceScansHandler))
+				r.Post("/malware", dfHandler.AuthHandler(ResourceScan, PermissionStop, dfHandler.ListMalwareScansHandler))
 			})
 			r.Route("/scan/results", func(r chi.Router) {
-				r.Get("/vulnerability", dfHandler.AuthHandler(ResourceScan, PermissionStop, dfHandler.ListVulnerabilityScanResultsHandler))
-				r.Get("/secret", dfHandler.AuthHandler(ResourceScan, PermissionStop, dfHandler.ListSecretScanResultsHandler))
-				r.Get("/compliance", dfHandler.AuthHandler(ResourceScan, PermissionStop, dfHandler.ListComplianceScanResultsHandler))
-				r.Get("/malware", dfHandler.AuthHandler(ResourceScan, PermissionStop, dfHandler.ListMalwareScanResultsHandler))
+				r.Post("/vulnerability", dfHandler.AuthHandler(ResourceScan, PermissionStop, dfHandler.ListVulnerabilityScanResultsHandler))
+				r.Post("/secret", dfHandler.AuthHandler(ResourceScan, PermissionStop, dfHandler.ListSecretScanResultsHandler))
+				r.Post("/compliance", dfHandler.AuthHandler(ResourceScan, PermissionStop, dfHandler.ListComplianceScanResultsHandler))
+				r.Post("/malware", dfHandler.AuthHandler(ResourceScan, PermissionStop, dfHandler.ListMalwareScanResultsHandler))
 			})
 
 			openApiDocs.AddDiagnosisOperations()

deepfence_server_client/.openapi-generator/FILES

@@ -45,7 +45,9 @@ docs/ModelRawReport.md
 docs/ModelResponse.md
 docs/ModelResponseAccessToken.md
 docs/ModelScanInfo.md
+docs/ModelScanListReq.md
 docs/ModelScanListResp.md
+docs/ModelScanResultsReq.md
 docs/ModelScanResultsResp.md
 docs/ModelScanStatusResp.md
 docs/ModelScanTriggerReq.md
@@ -88,7 +90,9 @@ model_model_raw_report.go
 model_model_response.go
 model_model_response_access_token.go
 model_model_scan_info.go
+model_model_scan_list_req.go
 model_model_scan_list_resp.go
+model_model_scan_results_req.go
 model_model_scan_results_resp.go
 model_model_scan_status_resp.go
 model_model_scan_trigger_req.go

deepfence_server_client/README.md

@@ -86,8 +86,8 @@ Class | Method | HTTP request | Description
 *CloudComplianceApi* | [**IngestCloudCompliances**](docs/CloudComplianceApi.md#ingestcloudcompliances) | **Post** /deepfence/ingest/cloud-compliance | Ingest Cloud Compliances
 *CloudResourcesApi* | [**IngestCloudResources**](docs/CloudResourcesApi.md#ingestcloudresources) | **Post** /deepfence/ingest/cloud-resources | Ingest Cloud resources
 *ComplianceApi* | [**IngestCompliances**](docs/ComplianceApi.md#ingestcompliances) | **Post** /deepfence/ingest/compliance | Ingest Compliances
-*ComplianceApi* | [**ListComplianceScan**](docs/ComplianceApi.md#listcompliancescan) | **Get** /deepfence/scan/list/compliance | Get Compliance Scans List
-*ComplianceApi* | [**ResultsComplianceScan**](docs/ComplianceApi.md#resultscompliancescan) | **Get** /deepfence/scan/results/compliance | Get Compliance Scans Results
+*ComplianceApi* | [**ListComplianceScan**](docs/ComplianceApi.md#listcompliancescan) | **Post** /deepfence/scan/list/compliance | Get Compliance Scans List
+*ComplianceApi* | [**ResultsComplianceScan**](docs/ComplianceApi.md#resultscompliancescan) | **Post** /deepfence/scan/results/compliance | Get Compliance Scans Results
 *ComplianceApi* | [**StartComplianceScan**](docs/ComplianceApi.md#startcompliancescan) | **Post** /deepfence/scan/start/compliance | Start Compliance Scan
 *ComplianceApi* | [**StatusComplianceScan**](docs/ComplianceApi.md#statuscompliancescan) | **Get** /deepfence/scan/status/compliance | Get Compliance Scan Status
 *ComplianceApi* | [**StopComplianceScan**](docs/ComplianceApi.md#stopcompliancescan) | **Post** /deepfence/scan/stop/compliance | Stop Compliance Scan
@@ -98,15 +98,15 @@ Class | Method | HTTP request | Description
 *DiagnosisApi* | [**DiagnosticNotification**](docs/DiagnosisApi.md#diagnosticnotification) | **Get** /deepfence/diagnosis/notification | Get Diagnostic Notification
 *DiagnosisApi* | [**GenerateAgentDiagnosticLogs**](docs/DiagnosisApi.md#generateagentdiagnosticlogs) | **Post** /deepfence/diagnosis/agent-logs | Generate Agent Diagnostic Logs
 *DiagnosisApi* | [**GenerateConsoleDiagnosticLogs**](docs/DiagnosisApi.md#generateconsolediagnosticlogs) | **Post** /deepfence/diagnosis/console-logs | Generate Console Diagnostic Logs
-*MalwareScanApi* | [**ListMalwareScan**](docs/MalwareScanApi.md#listmalwarescan) | **Get** /deepfence/scan/list/malware | Get Malware Scans List
-*MalwareScanApi* | [**ResultsMalwareScan**](docs/MalwareScanApi.md#resultsmalwarescan) | **Get** /deepfence/scan/results/malware | Get Malware Scans Results
+*MalwareScanApi* | [**ListMalwareScan**](docs/MalwareScanApi.md#listmalwarescan) | **Post** /deepfence/scan/list/malware | Get Malware Scans List
+*MalwareScanApi* | [**ResultsMalwareScan**](docs/MalwareScanApi.md#resultsmalwarescan) | **Post** /deepfence/scan/results/malware | Get Malware Scans Results
 *MalwareScanApi* | [**StartMalwareScan**](docs/MalwareScanApi.md#startmalwarescan) | **Post** /deepfence/scan/start/malware | Start Malware Scan
 *MalwareScanApi* | [**StatusMalwareScan**](docs/MalwareScanApi.md#statusmalwarescan) | **Get** /deepfence/scan/status/malware | Get Malware Scan Status
 *MalwareScanApi* | [**StopMalwareScan**](docs/MalwareScanApi.md#stopmalwarescan) | **Post** /deepfence/scan/stop/malware | Stop Malware Scan
 *SecretScanApi* | [**IngestSecretScanStatus**](docs/SecretScanApi.md#ingestsecretscanstatus) | **Post** /deepfence/ingest/secret-scan-logs | Ingest Secrets Scan Status
 *SecretScanApi* | [**IngestSecrets**](docs/SecretScanApi.md#ingestsecrets) | **Post** /deepfence/ingest/secrets | Ingest Secrets
-*SecretScanApi* | [**ListSecretScan**](docs/SecretScanApi.md#listsecretscan) | **Get** /deepfence/scan/list/secret | Get Secret Scans List
-*SecretScanApi* | [**ResultsSecretScan**](docs/SecretScanApi.md#resultssecretscan) | **Get** /deepfence/scan/results/secret | Get Secret Scans Results
+*SecretScanApi* | [**ListSecretScan**](docs/SecretScanApi.md#listsecretscan) | **Post** /deepfence/scan/list/secret | Get Secret Scans List
+*SecretScanApi* | [**ResultsSecretScan**](docs/SecretScanApi.md#resultssecretscan) | **Post** /deepfence/scan/results/secret | Get Secret Scans Results
 *SecretScanApi* | [**StartSecretScan**](docs/SecretScanApi.md#startsecretscan) | **Post** /deepfence/scan/start/secret | Start Secret Scan
 *SecretScanApi* | [**StatusSecretScan**](docs/SecretScanApi.md#statussecretscan) | **Get** /deepfence/scan/status/secret | Get Secret Scan Status
 *SecretScanApi* | [**StopSecretScan**](docs/SecretScanApi.md#stopsecretscan) | **Post** /deepfence/scan/stop/secret | Stop Secret Scan
@@ -119,8 +119,8 @@ Class | Method | HTTP request | Description
 *UserApi* | [**RegisterUser**](docs/UserApi.md#registeruser) | **Post** /deepfence/user/register | Register User
 *UserApi* | [**UpdateCurrentUser**](docs/UserApi.md#updatecurrentuser) | **Put** /deepfence/user | Update Current User
 *VulnerabilityApi* | [**IngestVulnerabilities**](docs/VulnerabilityApi.md#ingestvulnerabilities) | **Post** /deepfence/ingest/vulnerabilities | Ingest Vulnerabilities
-*VulnerabilityApi* | [**ListVulnerabilityScans**](docs/VulnerabilityApi.md#listvulnerabilityscans) | **Get** /deepfence/scan/list/vulnerability | Get Vulnerability Scans List
-*VulnerabilityApi* | [**ResultsVulnerabilityScans**](docs/VulnerabilityApi.md#resultsvulnerabilityscans) | **Get** /deepfence/scan/results/vulnerability | Get Vulnerability Scans Results
+*VulnerabilityApi* | [**ListVulnerabilityScans**](docs/VulnerabilityApi.md#listvulnerabilityscans) | **Post** /deepfence/scan/list/vulnerability | Get Vulnerability Scans List
+*VulnerabilityApi* | [**ResultsVulnerabilityScans**](docs/VulnerabilityApi.md#resultsvulnerabilityscans) | **Post** /deepfence/scan/results/vulnerability | Get Vulnerability Scans Results
 *VulnerabilityApi* | [**StartVulnerabilityScan**](docs/VulnerabilityApi.md#startvulnerabilityscan) | **Post** /deepfence/scan/start/vulnerability | Start Vulnerability Scan
 *VulnerabilityApi* | [**StatusVulnerabilityScan**](docs/VulnerabilityApi.md#statusvulnerabilityscan) | **Get** /deepfence/scan/status/vulnerability | Get Vulnerability Scan Status
 *VulnerabilityApi* | [**StopVulnerabilityScan**](docs/VulnerabilityApi.md#stopvulnerabilityscan) | **Post** /deepfence/scan/stop/vulnerability | Stop Vulnerability Scan
@@ -150,7 +150,9 @@ Class | Method | HTTP request | Description
  - [ModelResponse](docs/ModelResponse.md)
  - [ModelResponseAccessToken](docs/ModelResponseAccessToken.md)
  - [ModelScanInfo](docs/ModelScanInfo.md)
+ - [ModelScanListReq](docs/ModelScanListReq.md)
  - [ModelScanListResp](docs/ModelScanListResp.md)
+ - [ModelScanResultsReq](docs/ModelScanResultsReq.md)
  - [ModelScanResultsResp](docs/ModelScanResultsResp.md)
  - [ModelScanStatusResp](docs/ModelScanStatusResp.md)
  - [ModelScanTriggerReq](docs/ModelScanTriggerReq.md)