Revert "report error when results exceed 100k nodes" (#2131)

Revert "report error when results exceed 100k nodes (#2108)"

This reverts commit 960b717.

deepfence_server/handler/export_reports.go

@@ -107,7 +107,7 @@ func (h *Handler) BulkDeleteReports(w http.ResponseWriter, r *http.Request) {
 		var report model.ExportReport
 		utils.FromMap(dbNode.Props, &report)
 
-		if report.StoragePath != "" {
+		if report.Status != utils.ScanStatusFailed {
 			err = mc.DeleteFile(r.Context(), report.StoragePath, false, minio.RemoveObjectOptions{ForceDelete: true})
 			if err != nil {
 				log.Error().Err(err).Msgf("Failed to delete in file server for report id: %s",
@@ -118,11 +118,15 @@ func (h *Handler) BulkDeleteReports(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	if len(deletedRecs) != len(req.ReportIDs) {
+	if len(deletedRecs) == 0 {
+		log.Error().Msgf("Failed to delete any reports")
+		h.respondError(fmt.Errorf("failed to delete any reports"), w)
+		return
+	} else if len(deletedRecs) != len(req.ReportIDs) {
 		log.Warn().Msgf("Not able to delete all the requested reports")
 	}
 
-	vars["uids"] = req.ReportIDs
+	vars["uids"] = deletedRecs
 	deleteQuery := `MATCH (n:Report) WHERE n.report_id in $uids DELETE n`
 	_, err = tx.Run(ctx, deleteQuery, vars)
 	if err != nil {

deepfence_server/model/reports.go

@@ -33,7 +33,6 @@ type ExportReport struct {
 	URL           string `json:"url"`
 	FileName      string `json:"-"`
 	Status        string `json:"status"`
-	StatusMessage string `json:"status_message"`
 	StoragePath   string `json:"storage_path"`
 	FromTimestamp int64  `json:"from_timestamp"` // timestamp in milliseconds
 	ToTimestamp   int64  `json:"to_timestamp"`   // timestamp in milliseconds

deepfence_utils/utils/constants.go

@@ -70,9 +70,7 @@ const (
 )
 
 const (
-	ReportRetentionTime     = 24 * time.Hour
-	ReportGenerationTimeout = 30 * time.Minute
-	ReportRecordsMax        = 100000
+	ReportRetentionTime = 24 * time.Hour
 )
 
 const (

deepfence_utils/utils/structs.go

@@ -122,11 +122,3 @@ func (r ReportFilters) String() string {
 		return string(b)
 	}
 }
-
-func (ar AdvancedReportFilters) String() string {
-	if b, err := json.Marshal(ar); err != nil {
-		return ""
-	} else {
-		return string(b)
-	}
-}

deepfence_worker/Dockerfile

@@ -4,6 +4,7 @@ ARG IMAGE_REPOSITORY=deepfenceio
 FROM $IMAGE_REPOSITORY/deepfence_package_scanner_ce:$DF_IMG_TAG AS packagescanner
 FROM $IMAGE_REPOSITORY/deepfence_secret_scanner_ce:$DF_IMG_TAG AS secretscanner
 FROM $IMAGE_REPOSITORY/deepfence_malware_scanner_ce:$DF_IMG_TAG AS yarahunter
+FROM surnet/alpine-wkhtmltopdf:3.17.0-0.12.6-full as wkhtmltopdf
 FROM $IMAGE_REPOSITORY/deepfence_builder_ce:$DF_IMG_TAG AS builder-yara
 
 FROM alpine:3.18 AS final
@@ -29,12 +30,7 @@ ENV DEEPFENCE_KAFKA_TOPIC_PARTITIONS=3 \
     DEEPFENCE_MODE=worker \
     LD_LIBRARY_PATH=/usr/local/yara/lib
 
-# ENV GRYPE_DB_UPDATE_URL="http://${DEEPFENCE_FILE_SERVER_HOST}:${DEEPFENCE_FILE_SERVER_PORT}/database/database/vulnerability/listing.json"
-
-# RUN apk add --no-cache --update bash curl \
-#     && apk upgrade \
-#     && curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin v0.55.0
-
+COPY --from=wkhtmltopdf /bin/wkhtmltopdf /bin/libwkhtmltox.so /bin/
 
 COPY --from=packagescanner /usr/local/bin/syft /usr/local/bin/syft
 COPY --from=packagescanner /usr/local/bin/grype /usr/local/bin/grype

deepfence_worker/cronjobs/reports.go

@@ -20,7 +20,6 @@ func CleanUpReports(ctx context.Context, task *asynq.Task) error {
 	log := log.WithCtx(ctx)
 
 	log.Info().Msg("Start reports cleanup")
-	defer log.Info().Msg("Complete reports cleanup")
 
 	mc, err := directory.FileServerClient(ctx)
 	if err != nil {
@@ -40,16 +39,6 @@ func CleanUpReports(ctx context.Context, task *asynq.Task) error {
 	}
 	defer session.Close(ctx)
 
-	// mark reports which are inprogress for long time as failed
-	if err = failInprogressReports(ctx, session); err != nil {
-		log.Error().Err(err).Msg("failed to mark inprogress report")
-	}
-
-	// delete the reports which are in failed state
-	if err = deleteFailedReports(ctx, session); err != nil {
-		log.Error().Err(err).Msg("failed to delete failed report")
-	}
-
 	hoursAgo := time.Now().Add(time.Duration(-utils.ReportRetentionTime))
 
 	cleanup := func(pathPrefix string) {
@@ -69,7 +58,12 @@ func CleanUpReports(ctx context.Context, task *asynq.Task) error {
 
 	cleanup(minioReportsPrefix)
 
-	return nil
+	// delete the reports which are in failed state
+	err = deleteFailedReports(ctx, session)
+
+	log.Info().Msg("Complete reports cleanup")
+
+	return err
 }
 
 func deleteReport(ctx context.Context, session neo4j.SessionWithContext, path string) error {
@@ -101,9 +95,6 @@ func deleteFailedReports(ctx context.Context, session neo4j.SessionWithContext)
 
 	log := log.WithCtx(ctx)
 
-	log.Info().Msg("Start reports cleanup delete failed reports")
-	defer log.Info().Msg("Complete reports cleanup delete failed reports")
-
 	ctx, span := telemetry.NewSpan(ctx, "cronjobs", "delete-failed-reports")
 	defer span.End()
 
@@ -128,37 +119,3 @@ func deleteFailedReports(ctx context.Context, session neo4j.SessionWithContext)
 
 	return tx.Commit(ctx)
 }
-
-func failInprogressReports(ctx context.Context, session neo4j.SessionWithContext) error {
-
-	log := log.WithCtx(ctx)
-
-	log.Info().Msg("Start reports cleanup fail long inprogress")
-	defer log.Info().Msg("Complete reports cleanup fail long inprogress")
-
-	ctx, span := telemetry.NewSpan(ctx, "cronjobs", "fail-inprogress-reports")
-	defer span.End()
-
-	tx, err := session.BeginTransaction(ctx, neo4j.WithTxTimeout(10*time.Second))
-	if err != nil {
-		log.Error().Msg(err.Error())
-		return err
-	}
-	defer tx.Close(ctx)
-
-	duration := utils.ReportGenerationTimeout.Milliseconds()
-
-	query := `MATCH (n:Report{status:$inprogress}) WHERE TIMESTAMP()-n.updated_at > $duration SET n.status=$error`
-	vars := map[string]interface{}{
-		"duration":   duration,
-		"inprogress": utils.ScanStatusInProgress,
-		"error":      utils.ScanStatusFailed,
-	}
-	_, err = tx.Run(ctx, query, vars)
-	if err != nil {
-		log.Error().Msg(err.Error())
-		return err
-	}
-
-	return tx.Commit(ctx)
-}

deepfence_worker/cronscheduler/scheduler.go

@@ -296,7 +296,7 @@ func (s *Scheduler) addCronJobs(ctx context.Context) error {
 	}
 	jobIDs = append(jobIDs, jobID)
 
-	jobID, err = s.cron.AddFunc("@every 30m",
+	jobID, err = s.cron.AddFunc("@every 60m",
 		s.enqueueTask(namespace, utils.ReportCleanUpTask, true, utils.DefaultTaskOpts()...))
 	if err != nil {
 		return err

deepfence_worker/go.mod

@@ -1,8 +1,8 @@
 module github.com/deepfence/ThreatMapper/deepfence_worker
 
-go 1.21.1
+go 1.21.0
 
-toolchain go1.21.9
+toolchain go1.21.1
 
 replace github.com/deepfence/golang_deepfence_sdk/client => ../golang_deepfence_sdk/client/
 
@@ -26,6 +26,8 @@ replace github.com/deepfence/agent-plugins-grpc => ../deepfence_agent/plugins/ag
 require github.com/anchore/syft v0.94.0
 
 require (
+	github.com/Masterminds/sprig/v3 v3.2.3
+	github.com/SebastiaanKlippert/go-wkhtmltopdf v1.9.2
 	github.com/aws/aws-sdk-go v1.49.16
 	github.com/cenkalti/backoff/v3 v3.2.2
 	github.com/deepfence/SecretScanner v0.0.0-00010101000000-000000000000
@@ -38,7 +40,6 @@ require (
 	github.com/go-chi/chi/v5 v5.0.8
 	github.com/hibiken/asynq v0.24.1
 	github.com/jellydator/ttlcache/v3 v3.2.0
-	github.com/johnfercher/maroto/v2 v2.0.0-beta.17
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/minio/minio-go/v7 v7.0.66
 	github.com/neo4j/neo4j-go-driver/v5 v5.17.0
@@ -67,7 +68,6 @@ require (
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Masterminds/semver v1.5.0 // indirect
 	github.com/Masterminds/semver/v3 v3.2.0 // indirect
-	github.com/Masterminds/sprig/v3 v3.2.3 // indirect
 	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/Microsoft/hcsshim v0.12.0-rc.0 // indirect
 	github.com/PagerDuty/go-pagerduty v1.7.0 // indirect
@@ -85,7 +85,6 @@ require (
 	github.com/becheran/wildmatch-go v1.0.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bmatcuk/doublestar/v4 v4.6.0 // indirect
-	github.com/boombuler/barcode v1.0.1 // indirect
 	github.com/casbin/casbin/v2 v2.81.0 // indirect
 	github.com/casbin/govaluate v1.1.0 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
@@ -114,7 +113,6 @@ require (
 	github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
-	github.com/f-amaral/go-async v0.3.0 // indirect
 	github.com/facebookincubator/nvdtools v0.1.5 // indirect
 	github.com/fatih/color v1.16.0 // indirect
 	github.com/fatih/structs v1.1.0 // indirect
@@ -155,19 +153,15 @@ require (
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-retryablehttp v0.7.5 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
-	github.com/hhrutter/lzw v1.0.0 // indirect
-	github.com/hhrutter/tiff v1.0.1 // indirect
 	github.com/hillu/go-yara/v4 v4.3.2 // indirect
 	github.com/huandu/xstrings v1.3.3 // indirect
 	github.com/iancoleman/strcase v0.3.0 // indirect
 	github.com/imdario/mergo v0.3.16 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jinzhu/copier v0.4.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
-	github.com/johnfercher/go-tree v1.0.5 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/jung-kurt/gofpdf v1.16.2 // indirect
 	github.com/k-sone/critbitgo v1.4.0 // indirect
 	github.com/klauspost/compress v1.17.4 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.6 // indirect
@@ -211,7 +205,6 @@ require (
 	github.com/opencontainers/runtime-spec v1.1.1-0.20230823135140-4fec88fd00a4 // indirect
 	github.com/opencontainers/selinux v1.11.0 // indirect
 	github.com/pborman/indent v1.2.1 // indirect
-	github.com/pdfcpu/pdfcpu v0.6.0 // indirect
 	github.com/pelletier/go-toml v1.9.5 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.8 // indirect
 	github.com/pierrec/lz4/v4 v4.1.19 // indirect
@@ -223,7 +216,7 @@ require (
 	github.com/redis/go-redis/v9 v9.4.0 // indirect
 	github.com/richardlehane/mscfb v1.0.4 // indirect
 	github.com/richardlehane/msoleps v1.0.3 // indirect
-	github.com/rivo/uniseg v0.4.4 // indirect
+	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/rs/xid v1.5.0 // indirect
 	github.com/samber/lo v1.39.0 // indirect
 	github.com/samber/mo v1.11.0 // indirect
@@ -265,11 +258,10 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/crypto v0.17.0 // indirect
 	golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 // indirect
-	golang.org/x/image v0.15.0 // indirect
 	golang.org/x/net v0.19.0 // indirect
 	golang.org/x/oauth2 v0.13.0 // indirect
 	golang.org/x/sync v0.5.0 // indirect
-	golang.org/x/sys v0.16.0 // indirect
+	golang.org/x/sys v0.15.0 // indirect
 	golang.org/x/term v0.15.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.3.0 // indirect