diff --git a/deepfence_server/handler/export_reports.go b/deepfence_server/handler/export_reports.go index 1232c64388..ce42f826ad 100644 --- a/deepfence_server/handler/export_reports.go +++ b/deepfence_server/handler/export_reports.go @@ -492,6 +492,7 @@ func (h *Handler) GenerateReport(w http.ResponseWriter, r *http.Request) { ToTimestamp: toTimestamp, Filters: req.Filters, Options: req.Options, + ZippedReport: req.ZippedReport, } // scan id can only be sent while downloading individual scans diff --git a/deepfence_server/model/reports.go b/deepfence_server/model/reports.go index 46b636059a..51fc98af5a 100644 --- a/deepfence_server/model/reports.go +++ b/deepfence_server/model/reports.go @@ -10,6 +10,7 @@ type GenerateReportReq struct { ToTimestamp int64 `json:"to_timestamp"` // timestamp in milliseconds Filters utils.ReportFilters `json:"filters"` Options utils.ReportOptions `json:"options" validate:"omitempty"` + ZippedReport bool `json:"zipped_report"` } type GenerateReportResp struct { diff --git a/deepfence_utils/utils/constants.go b/deepfence_utils/utils/constants.go index f109c5f875..d9fb5cc5fc 100644 --- a/deepfence_utils/utils/constants.go +++ b/deepfence_utils/utils/constants.go @@ -252,6 +252,7 @@ const ( ReportXLSX ReportType = "xlsx" ReportPDF ReportType = "pdf" ReportSBOM ReportType = "sbom" + ReportZIP ReportType = "zip" ) // mask_global : This is to mask gobally. (same as previous mask_across_hosts_and_images flag) diff --git a/deepfence_utils/utils/structs.go b/deepfence_utils/utils/structs.go index 6f7f9e2842..9fbf930bf6 100644 --- a/deepfence_utils/utils/structs.go +++ b/deepfence_utils/utils/structs.go @@ -83,6 +83,7 @@ type ReportParams struct { ToTimestamp time.Time `json:"to_timestamp"` Filters ReportFilters `json:"filters"` Options ReportOptions `json:"options,omitempty"` + ZippedReport bool `json:"zipped_report"` } type ReportOptions struct { diff --git a/deepfence_utils/utils/utils.go b/deepfence_utils/utils/utils.go index 0990e75d68..030dd56c53 100644 --- a/deepfence_utils/utils/utils.go +++ b/deepfence_utils/utils/utils.go @@ -45,6 +45,8 @@ var ( SBOMFormatReplacer = strings.NewReplacer("@", "_", ".", "_") + NodeNameReplacer = strings.NewReplacer("/", "_", " ", "") + matchFirstCap = regexp.MustCompile("(.)([A-Z][a-z]+)") matchAllCap = regexp.MustCompile("([a-z0-9])([A-Z])") once1, once2 sync.Once @@ -782,3 +784,41 @@ func ComputeChecksumForFile(filePath string) (string, error) { cs := fmt.Sprintf("%x", h.Sum(nil)) return cs, nil } + +func ZipDir(sourceDir string, baseZipPath string, outputZip string) error { + archive, err := os.Create(outputZip) + if err != nil { + return err + } + defer archive.Close() + + zw := zip.NewWriter(archive) + defer zw.Close() + + return filepath.Walk(sourceDir, + func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + + f, err := zw.Create(filepath.Join(baseZipPath, info.Name())) + if err != nil { + return err + } + + _, err = io.Copy(f, file) + if err != nil { + return err + } + + return nil + }) +} diff --git a/deepfence_worker/tasks/reports/pdf.go b/deepfence_worker/tasks/reports/pdf.go index 062af7a949..6d22dde0d9 100644 --- a/deepfence_worker/tasks/reports/pdf.go +++ b/deepfence_worker/tasks/reports/pdf.go @@ -3,7 +3,6 @@ package reports import ( "context" _ "embed" - "os" "strconv" "time" @@ -189,7 +188,7 @@ func generatePDF(ctx context.Context, params utils.ReportParams) (string, error) log := log.WithCtx(ctx) var ( - document core.Document + document string err error ) @@ -213,19 +212,5 @@ func generatePDF(ctx context.Context, params utils.ReportParams) (string, error) return "", err } - // create a temp file to hold pdf report - temp, err := os.CreateTemp("", "report-*-"+reportFileName(params)) - if err != nil { - return "", err - } - defer temp.Close() - - if _, err := temp.Write(document.GetBytes()); err != nil { - return "", err - } - - log.Info().Msgf("report id %s pdf generation metrics %s", - params.ReportID, document.GetReport()) - - return temp.Name(), nil + return document, nil } diff --git a/deepfence_worker/tasks/reports/pdf_cloud_compliance.go b/deepfence_worker/tasks/reports/pdf_cloud_compliance.go index 5fa04a6e12..0560a4bccb 100644 --- a/deepfence_worker/tasks/reports/pdf_cloud_compliance.go +++ b/deepfence_worker/tasks/reports/pdf_cloud_compliance.go @@ -78,14 +78,14 @@ func getCloudComplianceSummaryPage(data map[string]map[string]int32) core.Page { return summaryPage } -func cloudcompliancePDF(ctx context.Context, params utils.ReportParams) (core.Document, error) { +func cloudcompliancePDF(ctx context.Context, params utils.ReportParams) (string, error) { log := log.WithCtx(ctx) data, err := getCloudComplianceData(ctx, params) if err != nil { log.Error().Err(err).Msg("failed to get compliance info") - return nil, err + return "", err } log.Info().Msgf("report id %s has %d records", @@ -173,5 +173,13 @@ func cloudcompliancePDF(ctx context.Context, params utils.ReportParams) (core.Do m.AddPages(summaryPage) m.AddPages(resultPages...) - return m.Generate() + doc, err := m.Generate() + if err != nil { + return "", err + } + + log.Info().Msgf("report id %s pdf generation metrics %s", + params.ReportID, doc.GetReport()) + + return writeReportToFile("", "report-*-"+reportFileName(params), doc.GetBytes()) } diff --git a/deepfence_worker/tasks/reports/pdf_compliance.go b/deepfence_worker/tasks/reports/pdf_compliance.go index 64ff06392b..bf16747373 100644 --- a/deepfence_worker/tasks/reports/pdf_compliance.go +++ b/deepfence_worker/tasks/reports/pdf_compliance.go @@ -80,14 +80,14 @@ func getComplianceSummaryPage(data map[string]map[string]int32) core.Page { return summaryPage } -func compliancePDF(ctx context.Context, params utils.ReportParams) (core.Document, error) { +func compliancePDF(ctx context.Context, params utils.ReportParams) (string, error) { log := log.WithCtx(ctx) data, err := getComplianceData(ctx, params) if err != nil { log.Error().Err(err).Msg("failed to get compliance info") - return nil, err + return "", err } log.Info().Msgf("report id %s has %d records", @@ -224,5 +224,13 @@ func compliancePDF(ctx context.Context, params utils.ReportParams) (core.Documen m.AddPages(summaryPage) m.AddPages(resultPages...) - return m.Generate() + doc, err := m.Generate() + if err != nil { + return "", err + } + + log.Info().Msgf("report id %s pdf generation metrics %s", + params.ReportID, doc.GetReport()) + + return writeReportToFile("", "report-*-"+reportFileName(params), doc.GetBytes()) } diff --git a/deepfence_worker/tasks/reports/pdf_malware.go b/deepfence_worker/tasks/reports/pdf_malware.go index a59704d536..b65e5594d9 100644 --- a/deepfence_worker/tasks/reports/pdf_malware.go +++ b/deepfence_worker/tasks/reports/pdf_malware.go @@ -19,14 +19,14 @@ import ( "github.com/johnfercher/maroto/v2/pkg/props" ) -func malwarePDF(ctx context.Context, params utils.ReportParams) (core.Document, error) { +func malwarePDF(ctx context.Context, params utils.ReportParams) (string, error) { log := log.WithCtx(ctx) data, err := getMalwareData(ctx, params) if err != nil { log.Error().Err(err).Msg("failed to get malware info") - return nil, err + return "", err } log.Info().Msgf("report id %s has %d records", @@ -118,5 +118,13 @@ func malwarePDF(ctx context.Context, params utils.ReportParams) (core.Document, m.AddPages(summaryPage) m.AddPages(resultPages...) - return m.Generate() + doc, err := m.Generate() + if err != nil { + return "", err + } + + log.Info().Msgf("report id %s pdf generation metrics %s", + params.ReportID, doc.GetReport()) + + return writeReportToFile("", "report-*-"+reportFileName(params), doc.GetBytes()) } diff --git a/deepfence_worker/tasks/reports/pdf_secret.go b/deepfence_worker/tasks/reports/pdf_secret.go index 20a0d2e13b..4976f72612 100644 --- a/deepfence_worker/tasks/reports/pdf_secret.go +++ b/deepfence_worker/tasks/reports/pdf_secret.go @@ -19,14 +19,14 @@ import ( "github.com/johnfercher/maroto/v2/pkg/props" ) -func secretPDF(ctx context.Context, params utils.ReportParams) (core.Document, error) { +func secretPDF(ctx context.Context, params utils.ReportParams) (string, error) { log := log.WithCtx(ctx) data, err := getSecretData(ctx, params) if err != nil { log.Error().Err(err).Msg("failed to get secrets info") - return nil, err + return "", err } log.Info().Msgf("report id %s has %d records", @@ -114,5 +114,13 @@ func secretPDF(ctx context.Context, params utils.ReportParams) (core.Document, e m.AddPages(summaryPage) m.AddPages(resultPages...) - return m.Generate() + doc, err := m.Generate() + if err != nil { + return "", err + } + + log.Info().Msgf("report id %s pdf generation metrics %s", + params.ReportID, doc.GetReport()) + + return writeReportToFile("", "report-*-"+reportFileName(params), doc.GetBytes()) } diff --git a/deepfence_worker/tasks/reports/pdf_vulnerability.go b/deepfence_worker/tasks/reports/pdf_vulnerability.go index fe4b2b58ac..9099e5fd81 100644 --- a/deepfence_worker/tasks/reports/pdf_vulnerability.go +++ b/deepfence_worker/tasks/reports/pdf_vulnerability.go @@ -3,11 +3,16 @@ package reports import ( "context" "fmt" + "os" + "path/filepath" "strconv" "strings" + "time" + "github.com/deepfence/ThreatMapper/deepfence_server/model" "github.com/deepfence/ThreatMapper/deepfence_utils/log" "github.com/deepfence/ThreatMapper/deepfence_utils/utils" + sdkUtils "github.com/deepfence/ThreatMapper/deepfence_utils/utils" "github.com/johnfercher/maroto/v2/pkg/components/page" "github.com/johnfercher/maroto/v2/pkg/components/row" "github.com/johnfercher/maroto/v2/pkg/components/text" @@ -19,23 +24,48 @@ import ( "github.com/johnfercher/maroto/v2/pkg/props" ) -func vulnerabilityPDF(ctx context.Context, params utils.ReportParams) (core.Document, error) { +var ( + vulnCellStyle = &props.Cell{ + BackgroundColor: &props.Color{Red: 255, Green: 255, Blue: 255}, + BorderType: border.Full, + BorderColor: &props.Color{Red: 0, Green: 0, Blue: 0}, + BorderThickness: 0.1, + } + + vulnResultHeaderProps = props.Text{ + Size: 10, + Left: 1, + Top: 1, + Align: align.Center, + Style: fontstyle.Bold, + Color: &props.Color{Red: 0, Green: 0, Blue: 200}, + } +) + +func vulnerabilityPDF(ctx context.Context, params utils.ReportParams) (string, error) { log := log.WithCtx(ctx) data, err := getVulnerabilityData(ctx, params) if err != nil { log.Error().Err(err).Msg("failed to get vulnerabilities info") - return nil, err + return "", err } log.Info().Msgf("report id %s has %d records", params.ReportID, data.NodeWiseData.RecordCount) + if !params.ZippedReport { + return createSingleReport(data, params) + } + return createZippedReport(data, params) +} + +func createSingleReport(data *Info[model.Vulnerability], params utils.ReportParams) (string, error) { // get new instance of marato m := getMarato() - // applied filter page + // applied filter pageparams utils.ReportParams filtersPage := getFiltersPage( data.Title, data.ScanType, @@ -45,71 +75,20 @@ func vulnerabilityPDF(ctx context.Context, params utils.ReportParams) (core.Docu data.AppliedFilters.AdvancedReportFilters.String(), ) - // summary table + // summary tableparams utils.ReportParams summaryPage := getSummaryPage(&data.NodeWiseData.SeverityCount) - cellStyle := &props.Cell{ - BackgroundColor: &props.Color{Red: 255, Green: 255, Blue: 255}, - BorderType: border.Full, - BorderColor: &props.Color{Red: 0, Green: 0, Blue: 0}, - BorderThickness: 0.1, - } - - resultHeaderProps := props.Text{ - Size: 10, - Left: 1, - Top: 1, - Align: align.Center, - Style: fontstyle.Bold, - Color: &props.Color{Red: 0, Green: 0, Blue: 200}, - } - // page per scan resultPages := []core.Page{} for i, d := range data.NodeWiseData.ScanData { - // skip if there are no results if len(d.ScanResults) == 0 { continue } - + // add result pages p := page.New() - p.Add(text.NewRow(10, fmt.Sprintf("%s - Scan Details", i), resultHeaderProps)) - p.Add(row.New(6).Add( - text.NewCol(1, "No.", resultHeaderProps).WithStyle(cellStyle), - text.NewCol(2, "CVE ID", resultHeaderProps).WithStyle(cellStyle), - text.NewCol(3, "Package", resultHeaderProps).WithStyle(cellStyle), - text.NewCol(1, "Severity", resultHeaderProps).WithStyle(cellStyle), - text.NewCol(4, "Summary", resultHeaderProps).WithStyle(cellStyle), - text.NewCol(1, "Link", resultHeaderProps).WithStyle(cellStyle), - )) - - resultRows := []core.Row{} - for k, v := range d.ScanResults { - resultRows = append( - resultRows, - row.New(15).Add( - text.NewCol(1, strconv.Itoa(k+1), - props.Text{Size: 10, Top: 1, Align: align.Center}). - WithStyle(cellStyle), - text.NewCol(2, v.CveID, - props.Text{Size: 10, Top: 1, Align: align.Center, BreakLineStrategy: breakline.DashStrategy}).WithStyle(cellStyle), - text.NewCol(3, v.CveCausedByPackage, - props.Text{Size: 10, Left: 1, Top: 1, BreakLineStrategy: breakline.DashStrategy}). - WithStyle(cellStyle), - text.NewCol(1, v.GetCategory(), - props.Text{Size: 10, Top: 1, Align: align.Center, Style: fontstyle.Bold, Color: colors[v.GetCategory()]}). - WithStyle(cellStyle), - text.NewCol(4, truncateText(v.CveDescription, 80), - props.Text{Size: 10, Left: 1, Top: 1, BreakLineStrategy: breakline.EmptySpaceStrategy}). - WithStyle(cellStyle), - text.NewCol(1, "link", - props.Text{Size: 10, Top: 1, Align: align.Center, Hyperlink: &d.ScanResults[k].CveLink}). - WithStyle(cellStyle), - ), - ) - } - p.Add(resultRows...) + addVulnResultHeaders(p, i) + p.Add(getVulnerabilityResultRows(d)...) resultPages = append(resultPages, p) } @@ -118,5 +97,120 @@ func vulnerabilityPDF(ctx context.Context, params utils.ReportParams) (core.Docu m.AddPages(summaryPage) m.AddPages(resultPages...) - return m.Generate() + doc, err := m.Generate() + if err != nil { + return "", err + } + + log.Info().Msgf("report id %s pdf generation metrics %s", + params.ReportID, doc.GetReport()) + + return writeReportToFile("", tempReportFile(params), doc.GetBytes()) +} + +func createZippedReport(data *Info[model.Vulnerability], params utils.ReportParams) (string, error) { + + // tmp dir to save generated reports + tmpDir := filepath.Join( + os.TempDir(), + fmt.Sprintf("%d", time.Now().UnixMilli())+"-"+params.ReportID, + ) + defer os.RemoveAll(tmpDir) + + for i, d := range data.NodeWiseData.ScanData { + // get new instance of marato + m := getMarato() + + // applied filter pageparams utils.ReportParams + filtersPage := getFiltersPage( + data.Title, + data.ScanType, + data.AppliedFilters.NodeType, + fmt.Sprintf("%s - %s", data.StartTime, data.EndTime), + strings.Join(data.AppliedFilters.SeverityOrCheckType, ","), + data.AppliedFilters.AdvancedReportFilters.String(), + ) + + // summary tableparams utils.ReportParams + singleSummary := map[string]map[string]int32{ + i: data.NodeWiseData.SeverityCount[i], + } + summaryPage := getSummaryPage(&singleSummary) + + // skip if there are no results + if len(d.ScanResults) == 0 { + continue + } + + resultPage := page.New() + addVulnResultHeaders(resultPage, i) + resultPage.Add(getVulnerabilityResultRows(d)...) + + // add all pages + m.AddPages(filtersPage) + m.AddPages(summaryPage) + m.AddPages(resultPage) + + doc, err := m.Generate() + if err != nil { + return "", err + } + + outputFile := sdkUtils.NodeNameReplacer.Replace(i) + + fileExt(sdkUtils.ReportType(params.ReportType)) + + log.Info().Msgf("report id %s %s pdf generation metrics %s", + params.ReportID, outputFile, doc.GetReport()) + + writeReportToFile(tmpDir, outputFile, doc.GetBytes()) + } + + outputZip := reportFileName(params) + + if err := sdkUtils.ZipDir(tmpDir, "reports", outputZip); err != nil { + return "", err + } + + return outputZip, nil +} + +func addVulnResultHeaders(p core.Page, nodeName string) { + p.Add(text.NewRow(10, fmt.Sprintf("%s - Scan Details", nodeName), vulnResultHeaderProps)) + p.Add(row.New(6).Add( + text.NewCol(1, "No.", vulnResultHeaderProps).WithStyle(vulnCellStyle), + text.NewCol(2, "CVE ID", vulnResultHeaderProps).WithStyle(vulnCellStyle), + text.NewCol(3, "Package", vulnResultHeaderProps).WithStyle(vulnCellStyle), + text.NewCol(1, "Severity", vulnResultHeaderProps).WithStyle(vulnCellStyle), + text.NewCol(4, "Summary", vulnResultHeaderProps).WithStyle(vulnCellStyle), + text.NewCol(1, "Link", vulnResultHeaderProps).WithStyle(vulnCellStyle), + )) +} + +func getVulnerabilityResultRows(d ScanData[model.Vulnerability]) []core.Row { + resultRows := []core.Row{} + for k, v := range d.ScanResults { + resultRows = append( + resultRows, + row.New(15).Add( + text.NewCol(1, strconv.Itoa(k+1), + props.Text{Size: 10, Top: 1, Align: align.Center}). + WithStyle(vulnCellStyle), + text.NewCol(2, v.CveID, + props.Text{Size: 10, Top: 1, Align: align.Center, BreakLineStrategy: breakline.DashStrategy}).WithStyle(vulnCellStyle), + text.NewCol(3, v.CveCausedByPackage, + props.Text{Size: 10, Left: 1, Top: 1, BreakLineStrategy: breakline.DashStrategy}). + WithStyle(vulnCellStyle), + text.NewCol(1, v.GetCategory(), + props.Text{Size: 10, Top: 1, Align: align.Center, Style: fontstyle.Bold, Color: colors[v.GetCategory()]}). + WithStyle(vulnCellStyle), + text.NewCol(4, truncateText(v.CveDescription, 80), + props.Text{Size: 10, Left: 1, Top: 1, BreakLineStrategy: breakline.EmptySpaceStrategy}). + WithStyle(vulnCellStyle), + text.NewCol(1, "link", + props.Text{Size: 10, Top: 1, Align: align.Center, Hyperlink: &d.ScanResults[k].CveLink}). + WithStyle(vulnCellStyle), + ), + ) + } + return resultRows } diff --git a/deepfence_worker/tasks/reports/reports.go b/deepfence_worker/tasks/reports/reports.go index 5cdc88c497..e5fc4596f6 100644 --- a/deepfence_worker/tasks/reports/reports.go +++ b/deepfence_worker/tasks/reports/reports.go @@ -7,12 +7,14 @@ import ( "fmt" "os" "path" + "path/filepath" "strings" "time" "github.com/deepfence/ThreatMapper/deepfence_utils/directory" "github.com/deepfence/ThreatMapper/deepfence_utils/log" "github.com/deepfence/ThreatMapper/deepfence_utils/telemetry" + "github.com/deepfence/ThreatMapper/deepfence_utils/utils" sdkUtils "github.com/deepfence/ThreatMapper/deepfence_utils/utils" "github.com/hibiken/asynq" "github.com/minio/minio-go/v7" @@ -39,10 +41,43 @@ func reportFileName(params sdkUtils.ReportParams) string { if sdkUtils.ReportType(params.ReportType) == sdkUtils.ReportSBOM { return fmt.Sprintf("sbom_%s%s", params.ReportID, fileExt(sdkUtils.ReportSBOM)) } + list := []string{params.Filters.ScanType, params.Filters.NodeType, params.ReportID} + + if params.ZippedReport { + return strings.Join(list, "_") + ".zip" + } + return strings.Join(list, "_") + fileExt(sdkUtils.ReportType(params.ReportType)) } +func writeReportToFile(dir string, fileName string, data []byte) (string, error) { + + // make sure directory exists + os.MkdirAll(dir, os.ModePerm) + + out := filepath.Join(dir, fileName) + + log.Debug().Msgf("write report to path %s", out) + + err := os.WriteFile(out, data, os.ModePerm) + if err != nil { + return "", err + } + + return out, nil +} + +func tempReportFile(params utils.ReportParams) string { + return strings.Join( + []string{ + "report", + fmt.Sprintf("%d", time.Now().UnixMilli()), + reportFileName(params), + }, + "-") +} + func putOpts(reportType sdkUtils.ReportType) minio.PutObjectOptions { switch reportType { case sdkUtils.ReportXLSX: @@ -51,6 +86,8 @@ func putOpts(reportType sdkUtils.ReportType) minio.PutObjectOptions { return minio.PutObjectOptions{ContentType: "application/pdf"} case sdkUtils.ReportSBOM: return minio.PutObjectOptions{ContentType: "application/gzip"} + case sdkUtils.ReportZIP: + return minio.PutObjectOptions{ContentType: "application/zip"} } return minio.PutObjectOptions{} } @@ -107,7 +144,8 @@ func GenerateReport(ctx context.Context, task *asynq.Task) error { localReportPath, err := generateReport(ctx, params) if err != nil { log.Error().Err(err).Msgf("failed to generate report with params %+v", params) - updateReportState(ctx, session, params.ReportID, "", "", sdkUtils.ScanStatusFailed, err.Error()) + updateReportState(ctx, session, params.ReportID, + "", "", sdkUtils.ScanStatusFailed, err.Error()) return nil } log.Info().Msgf("report file path %s", localReportPath) @@ -123,14 +161,21 @@ func GenerateReport(ctx context.Context, task *asynq.Task) error { } reportName := path.Join("/report", reportFileName(params)) - res, err := mc.UploadLocalFile(ctx, reportName, - localReportPath, true, putOpts(sdkUtils.ReportType(params.ReportType))) + + putOptions := putOpts(sdkUtils.ReportType(params.ReportType)) + // specical case zip in not report type + if params.ZippedReport { + putOptions = putOpts(sdkUtils.ReportZIP) + } + + res, err := mc.UploadLocalFile(ctx, reportName, localReportPath, true, putOptions) if err != nil { log.Error().Err(err).Msg("failed to upload file to minio") return nil } - updateReportState(ctx, session, params.ReportID, reportName, res.Key, sdkUtils.ScanStatusSuccess, "") + updateReportState(ctx, session, params.ReportID, + reportName, res.Key, sdkUtils.ScanStatusSuccess, "") return nil }