Enhance configuration management and reporting features

- Added handling to run each features using flags
- Added handling for create ssl report for postgres
- Added handling New Labels in PII scanner
- Improved logic for custom time range for logparser
- custom config path support added
- json output format for output file for all features
- improved HTML structure for all reports
- Added support for JSON output in various report generation functions.
- Updated .gitignore to include new report formats and configuration files.
- Improved error handling and logging in database connection functions.
- Updated dependencies in go.mod for better compatibility.

These changes aim to improve the overall functionality and usability of the configuration management and reporting tools.

.gitignore

@@ -37,6 +37,7 @@ config.toml
 .vscode
 klouddbshield_report.html
 klouddbshield_report.txt
+klouddbshield_report.json
 kshield_pii_*.log
 
 cronreport/*
@@ -46,3 +47,6 @@ docker_testing/pglog/*
 output.csv
 detailed_output.csv
 all_checks.json
+
+postgresql.conf
+testoutput.log

.golangci.yml

@@ -6,7 +6,6 @@ run:
 issues:
   exclude-use-default: true
   exclude-case-sensitive: false
-  exclude-generated-strict: false
   exclude-dirs-use-default: true
   max-issues-per-linter: 50
   max-same-issues: 3

.goreleaser.yml

@@ -40,6 +40,9 @@ nfpms:
       - src: passwords/*
         dst: /etc/klouddbshield/passwords
         type: config
+      # Simple config file
+      - src: systemd.service
+        dst: /lib/systemd/system/ciscollector.service
       - src: python/*
         dst: /etc/klouddbshield/python
         type: config

cmd/ciscollector/compareconfig.go

@@ -0,0 +1,52 @@
+package main
+
+import (
+	"context"
+
+	"github.com/klouddb/klouddbshield/htmlreport"
+	"github.com/klouddb/klouddbshield/postgresconfig"
+)
+
+type compareConfigRunner struct {
+	baseServer        string
+	connectionStrings []string
+	htmlReportHelper  *htmlreport.HtmlReportHelper
+}
+
+func newCompareConfigRunner(baseServer string, connectionStrings []string, htmlReportHelper *htmlreport.HtmlReportHelper) *compareConfigRunner {
+	return &compareConfigRunner{
+		baseServer:        baseServer,
+		connectionStrings: connectionStrings,
+		htmlReportHelper:  htmlReportHelper,
+	}
+}
+
+func (c *compareConfigRunner) cronProcess(ctx context.Context) error {
+	return c.run(ctx)
+}
+
+func (c *compareConfigRunner) run(_ context.Context) error {
+
+	connectionStrings := c.connectionStrings
+	if c.baseServer != "" {
+		connectionStrings = append([]string{c.baseServer}, c.connectionStrings...)
+	}
+
+	result, err := postgresconfig.GetAllConfigValues(connectionStrings)
+	if err != nil {
+		return err
+	}
+
+	var one2oneComparison *postgresconfig.ConfigCompareOne2OneResult
+	if c.baseServer != "" {
+		one2oneComparison = postgresconfig.CompareAllServersWithBase(result)
+	}
+
+	configCompareResult := &postgresconfig.ConfigCompareResult{
+		One2OneComparison: one2oneComparison,
+	}
+
+	c.htmlReportHelper.RegisterCompareConfig(configCompareResult)
+
+	return nil
+}

cmd/ciscollector/configaudit.go

@@ -0,0 +1,45 @@
+package main
+
+import (
+	"context"
+
+	"github.com/klouddb/klouddbshield/htmlreport"
+	"github.com/klouddb/klouddbshield/pkg/postgresdb"
+	"github.com/klouddb/klouddbshield/postgres"
+	"github.com/klouddb/klouddbshield/postgres/configaudit"
+)
+
+type configAuditor struct {
+	postgresConfig   *postgresdb.Postgres
+	htmlReportHelper *htmlreport.HtmlReportHelper
+}
+
+func newConfigAuditor(postgresConfig *postgresdb.Postgres, htmlReportHelper *htmlreport.HtmlReportHelper) *configAuditor {
+	return &configAuditor{
+		postgresConfig:   postgresConfig,
+		htmlReportHelper: htmlReportHelper,
+	}
+}
+
+func (h *configAuditor) cronProcess(ctx context.Context) error {
+	return h.run(ctx)
+}
+
+func (h *configAuditor) run(ctx context.Context) error {
+	postgresStore, _, err := postgresdb.Open(*h.postgresConfig)
+	if err != nil {
+		return err
+	}
+	defer postgresStore.Close()
+
+	result, err := configaudit.AuditConfig(ctx, postgresStore)
+	if err != nil {
+		return err
+	}
+
+	h.htmlReportHelper.RegisterConfigAudit(result)
+
+	postgres.PrintConfigAuditSummary(result)
+
+	return nil
+}

cmd/ciscollector/crons.go

@@ -6,7 +6,6 @@ import (
 	"os"
 	"os/signal"
 	"path"
-	"strings"
 	"syscall"
 	"time"
 
@@ -34,11 +33,11 @@ func getProcessorsForCron(schedule string, commnd *config.Command, htmlHelperMap
 		for _, p := range commnd.Postgres {
 			htmlHelper := htmlHelperMap.Get(p.HtmlReportName())
 
-			out = append(out, newPostgresRunnerFromConfig(p, &strings.Builder{},
-				utils.NewDummyContainsAllSet[string](), htmlHelper))
-			out = append(out, newHBARunnerFromConfig(p, &strings.Builder{}, htmlHelper))
+			out = append(out, newPostgresRunnerFromConfig(p, map[string]interface{}{},
+				utils.NewDummyContainsAllSet[string](), htmlHelper, "json"))
+			out = append(out, newHBARunnerFromConfig(p, map[string]interface{}{}, htmlHelper, "json"))
 
-			out = append(out, newPwnedUserRunner(p, true, &strings.Builder{}, htmlHelper))
+			out = append(out, newPwnedUserRunner(p, true, map[string]interface{}{}, htmlHelper, "json"))
 		}
 
 		logPaser, err := getLogParserCron(schedule, commnd, htmlHelperMap)
@@ -56,8 +55,8 @@ func getProcessorsForCron(schedule string, commnd *config.Command, htmlHelperMap
 
 		out := make([]Runner, 0, len(commnd.Postgres))
 		for _, p := range commnd.Postgres {
-			out = append(out, newPostgresRunnerFromConfig(p, &strings.Builder{},
-				utils.NewDummyContainsAllSet[string](), htmlHelperMap.Get(p.HtmlReportName())))
+			out = append(out, newPostgresRunnerFromConfig(p, map[string]interface{}{},
+				utils.NewDummyContainsAllSet[string](), htmlHelperMap.Get(p.HtmlReportName()), "json"))
 		}
 
 		return out, nil
@@ -69,7 +68,7 @@ func getProcessorsForCron(schedule string, commnd *config.Command, htmlHelperMap
 
 		out := make([]Runner, 0, len(commnd.Postgres))
 		for _, p := range commnd.Postgres {
-			out = append(out, newHBARunnerFromConfig(p, &strings.Builder{}, htmlHelperMap.Get(p.HtmlReportName())))
+			out = append(out, newHBARunnerFromConfig(p, map[string]interface{}{}, htmlHelperMap.Get(p.HtmlReportName()), "json"))
 		}
 
 		return out, nil
@@ -82,21 +81,21 @@ func getProcessorsForCron(schedule string, commnd *config.Command, htmlHelperMap
 
 		out := make([]Runner, 0, len(commnd.Postgres))
 		for _, p := range commnd.Postgres {
-			out = append(out, newPwnedUserRunner(p, false, &strings.Builder{}, htmlHelperMap.Get(p.HtmlReportName())))
+			out = append(out, newPwnedUserRunner(p, false, map[string]interface{}{}, htmlHelperMap.Get(p.HtmlReportName()), "json"))
 		}
 		return out, nil
 
 	case cons.RootCMD_AWSRDS, cons.RootCMD_AWSAurora:
-		return []Runner{newRDSRunner(&strings.Builder{})}, nil
+		return []Runner{newRDSRunner("json")}, nil
 
 	case cons.RootCMD_MySQL:
 		out := make([]Runner, 0, len(commnd.Postgres))
 		for _, p := range commnd.MySQL {
-			out = append(out, newMySqlRunner(p, &strings.Builder{}, htmlHelperMap.Get(p.HtmlReportName())))
+			out = append(out, newMySqlRunner(p, map[string]interface{}{}, htmlHelperMap.Get(p.HtmlReportName()), "json"))
 		}
 		return out, nil
 
-	case cons.LogParserCMD_UniqueIPs, cons.LogParserCMD_InactiveUsr,
+	case cons.LogParserCMD_UniqueIPs, cons.LogParserCMD_InactiveUser,
 		cons.LogParserCMD_HBAUnusedLines, cons.LogParserCMD_PasswordLeakScanner:
 		return getLogParserCron(schedule, commnd, htmlHelperMap)
 
@@ -126,7 +125,7 @@ func getLogParserCron(schedule string, command *config.Command, htmlHelperMap ht
 		logParserConfig.Begin = startTime
 		logParserConfig.End = time.Now()
 
-		u := newLogParserRunnerFromConfig(p, logParserConfig, false, &strings.Builder{}, htmlHelperMap.Get(p.HtmlReportName()))
+		u := newLogParserRunnerFromConfig(p, logParserConfig, false, map[string]interface{}{}, htmlHelperMap.Get(p.HtmlReportName()), "json")
 		out = append(out, u)
 	}
 

cmd/ciscollector/hbarunner.go

@@ -15,15 +15,18 @@ import (
 
 type hbaRunner struct {
 	postgresConfig   *postgresdb.Postgres
-	builder          *strings.Builder
+	fileData         map[string]interface{}
 	htmlReportHelper *htmlreport.HtmlReportHelper
+	outputType       string
 }
 
-func newHBARunnerFromConfig(postgresConfig *postgresdb.Postgres, builder *strings.Builder, htmlReportHelper *htmlreport.HtmlReportHelper) *hbaRunner {
+func newHBARunnerFromConfig(postgresConfig *postgresdb.Postgres, fileData map[string]interface{},
+	htmlReportHelper *htmlreport.HtmlReportHelper, outputType string) *hbaRunner {
 	return &hbaRunner{
 		postgresConfig:   postgresConfig,
-		builder:          builder,
+		fileData:         fileData,
 		htmlReportHelper: htmlReportHelper,
+		outputType:       outputType,
 	}
 }
 
@@ -54,7 +57,11 @@ func (h *hbaRunner) run(ctx context.Context) ([]*model.HBAScannerResult, error)
 		}
 	}
 
-	h.builder.WriteString("\nHBA Report\n" + simpletextreport.PrintHBAReportInFile(listOfResults) + "\n")
+	if h.outputType == "json" {
+		h.fileData["HBA Report"] = listOfResults
+	} else {
+		h.fileData["HBA Report"] = simpletextreport.PrintHBAReportInFile(listOfResults)
+	}
 
 	return listOfResults, nil
 }

cmd/ciscollector/logparserrunner.go

@@ -5,7 +5,6 @@ import (
 	"database/sql"
 	"fmt"
 	"os"
-	"strings"
 
 	"github.com/jedib0t/go-pretty/text"
 	"github.com/klouddb/klouddbshield/htmlreport"
@@ -21,20 +20,22 @@ import (
 
 type logParserRunner struct {
 	postgresConfig   *postgresdb.Postgres
-	builder          *strings.Builder
+	fileData         map[string]interface{}
 	logParserCnf     *config.LogParser
 	isRunCmd         bool
 	htmlReportHelper *htmlreport.HtmlReportHelper
+	outputType       string
 }
 
 func newLogParserRunnerFromConfig(postgresConfig *postgresdb.Postgres, logParserCnf *config.LogParser, isRunCmd bool,
-	builder *strings.Builder, htmlReportHelper *htmlreport.HtmlReportHelper) *logParserRunner {
+	fileData map[string]interface{}, htmlReportHelper *htmlreport.HtmlReportHelper, outputType string) *logParserRunner {
 	return &logParserRunner{
 		postgresConfig:   postgresConfig,
-		builder:          builder,
+		fileData:         fileData,
 		logParserCnf:     logParserCnf,
 		isRunCmd:         isRunCmd,
 		htmlReportHelper: htmlReportHelper,
+		outputType:       outputType,
 	}
 }
 
@@ -52,7 +53,7 @@ func (l *logParserRunner) run(ctx context.Context) error {
 		}
 	}
 	updatePgSettings(ctx, store, l.logParserCnf.PgSettings)
-	return runLogParserWithMultipleParser(ctx, l.isRunCmd, l.logParserCnf, store, l.htmlReportHelper, l.builder)
+	return runLogParserWithMultipleParser(ctx, l.isRunCmd, l.logParserCnf, store, l.htmlReportHelper, l.fileData, l.outputType)
 }
 
 func updatePgSettings(ctx context.Context, store *sql.DB, pgSettings *model.PgSettings) {
@@ -68,20 +69,22 @@ func updatePgSettings(ctx context.Context, store *sql.DB, pgSettings *model.PgSe
 	pgSettings.LogConnections = ps.LogConnections
 }
 
-func runLogParserWithMultipleParser(ctx context.Context, runCmd bool, logParserCnf *config.LogParser, store *sql.DB, htmlReportHelper *htmlreport.HtmlReportHelper, builder *strings.Builder) error {
-	baseParser := parselog.GetDynamicBaseParser(logParserCnf.PgSettings.LogLinePrefix)
-	allParser, err := getAllParser(ctx, logParserCnf, store, baseParser)
+func runLogParserWithMultipleParser(ctx context.Context, runCmd bool, logParserCnf *config.LogParser,
+	store *sql.DB, htmlReportHelper *htmlreport.HtmlReportHelper, fileData map[string]interface{}, outputType string) error {
+
+	allParser, err := getAllParser(ctx, logParserCnf, store)
 	if err != nil {
 		return fmt.Errorf("Error while getting all parser: %v", err)
 	}
 
-	validatorFunc := parselog.GetBaseParserValidator(baseParser)
 	runnerFunctions := []runner.ParserFunc{}
 	for _, parser := range allParser {
 		runnerFunctions = append(runnerFunctions, parser.Feed)
 	}
 
-	fastRunnerResp, err := runner.RunFastParser(ctx, runCmd, logParserCnf, runnerFunctions, validatorFunc)
+	baseParser := parselog.GetDynamicBaseParser(logParserCnf.PgSettings.LogLinePrefix)
+
+	fastRunnerResp, err := runner.RunFastParser(ctx, runCmd, baseParser, logParserCnf, runnerFunctions)
 	if err != nil {
 		return fmt.Errorf("Error while running fast parser: %v", err)
 	}
@@ -103,24 +106,24 @@ func runLogParserWithMultipleParser(ctx context.Context, runCmd bool, logParserC
 	}
 
 	if runCmd {
-		logparser.PrintSummary(ctx, allParser, logParserCnf, fastRunnerResp, builder)
+		logparser.PrintSummary(ctx, allParser, logParserCnf, fastRunnerResp, fileData, outputType)
 	} else {
 		logparser.PrintFastRunnerReport(logParserCnf, fastRunnerResp)
-		logparser.PrintTerminalResultsForLogParser(ctx, allParser, logParserCnf.OutputType)
+		logparser.PrintTerminalResultsForLogParser(ctx, allParser, outputType)
 	}
 
 	htmlReportHelper.RenderLogparserResponse(ctx, store, allParser)
 	return nil
 }
 
-func getAllParser(ctx context.Context, logParserCnf *config.LogParser, store *sql.DB, baseParser parselog.BaseParser) ([]runner.Parser, error) {
+func getAllParser(ctx context.Context, logParserCnf *config.LogParser, store *sql.DB) ([]runner.Parser, error) {
 	allParser := []runner.Parser{}
 
 	for _, command := range logParserCnf.Commands {
 		switch command {
 		case cons.LogParserCMD_HBAUnusedLines:
 			unusedLinesHelper := logparser.NewUnusedHBALineHelper(store)
-			err := unusedLinesHelper.Init(ctx, logParserCnf, baseParser)
+			err := unusedLinesHelper.Init(ctx, logParserCnf)
 			if err != nil {
 				allParser = append(allParser, logparser.NewErrorHelper(command, "warning", err.Error()))
 			} else {
@@ -129,28 +132,36 @@ func getAllParser(ctx context.Context, logParserCnf *config.LogParser, store *sq
 
 		case cons.LogParserCMD_UniqueIPs:
 			uniqueIPs := logparser.NewUniqueIPHelper()
-			err := uniqueIPs.Init(ctx, logParserCnf, baseParser)
+			err := uniqueIPs.Init(ctx, logParserCnf)
 			if err != nil {
 				allParser = append(allParser, logparser.NewErrorHelper(command, "warning", err.Error()))
 			} else {
 				allParser = append(allParser, uniqueIPs)
 			}
-		case cons.LogParserCMD_InactiveUsr:
+		case cons.LogParserCMD_InactiveUser:
 			inactiveUser := logparser.NewInactiveUsersHelper(store)
-			err := inactiveUser.Init(ctx, logParserCnf, baseParser)
+			err := inactiveUser.Init(ctx, logParserCnf)
 			if err != nil {
 				allParser = append(allParser, logparser.NewErrorHelper(command, "warning", err.Error()))
 			} else {
 				allParser = append(allParser, inactiveUser)
 			}
 		case cons.LogParserCMD_PasswordLeakScanner:
 			passwordLeakScanner := logparser.NewPasswordLeakHelper()
-			err := passwordLeakScanner.Init(ctx, logParserCnf, baseParser)
+			err := passwordLeakScanner.Init(ctx, logParserCnf)
 			if err != nil {
 				allParser = append(allParser, logparser.NewErrorHelper(command, "warning", err.Error()))
 			} else {
 				allParser = append(allParser, passwordLeakScanner)
 			}
+		case cons.LogParserCMD_SqlInjectionScan:
+			sqlInjectionScan := logparser.NewSQLInjectionHelper()
+			err := sqlInjectionScan.Init(ctx, logParserCnf)
+			if err != nil {
+				allParser = append(allParser, logparser.NewErrorHelper(command, "warning", err.Error()))
+			} else {
+				allParser = append(allParser, sqlInjectionScan)
+			}
 		default:
 			return nil, fmt.Errorf("Invalid command: %s", command)
 		}