added hba unused line handling in logparser

.github/workflows/integration.yml

@@ -70,4 +70,4 @@ jobs:
           integrationtest setup --prefix "${{ matrix.log_prefix }}"
           sudo chmod -R 777 pglog/log0
           integrationtest test -p "${{ matrix.log_prefix }}" -f "pglog/log0/postgresql*.log"
-          docker-compose down -v
+          docker compose down -v

cmd/ciscollector/main.go

@@ -22,6 +22,7 @@ import (
 	"github.com/klouddb/klouddbshield/passwordmanager"
 	"github.com/klouddb/klouddbshield/pkg/config"
 	cons "github.com/klouddb/klouddbshield/pkg/const"
+	"github.com/klouddb/klouddbshield/pkg/hbarules"
 	"github.com/klouddb/klouddbshield/pkg/logger"
 	"github.com/klouddb/klouddbshield/pkg/mysqldb"
 	"github.com/klouddb/klouddbshield/pkg/parselog"
@@ -110,6 +111,8 @@ func main() {
 			runInactiveUSersLogParser(ctx, cnf, store)
 		// case cons.LogParserCMD_MismatchIPs:
 		// 	runMismatchIPsLogParser(ctx, cnf)
+		case cons.LogParserCMD_HBAUnusedLines:
+			runHBAUnusedLinesLogParser(ctx, cnf, store)
 		default:
 			fmt.Println("Invalid command for log parser")
 			os.Exit(1)
@@ -146,6 +149,62 @@ func updatePgSettings(ctx context.Context, store *sql.DB, pgSettings *model.PgSe
 	pgSettings.LogConnections = ps.LogConnections
 }
 
+func runHBAUnusedLinesLogParser(ctx context.Context, cnf *config.Config, store *sql.DB) {
+
+	// check if postgres setting contains required variable or connection logs
+	if !strings.Contains(cnf.LogParser.PgSettings.LogLinePrefix, "%h") && !strings.Contains(cnf.LogParser.PgSettings.LogLinePrefix, "%r") {
+		fmt.Println("Please set log_line_prefix to '%h' or '%r' or enable log_connections")
+		return
+	}
+
+	if !strings.Contains(cnf.LogParser.PgSettings.LogLinePrefix, "%u") || !strings.Contains(cnf.LogParser.PgSettings.LogLinePrefix, "%d") {
+		fmt.Printf("In logline prefix, please set '%s' and '%s'\n", "%u", "%d") // using printf to avoid the warning for %d in println
+		return
+	}
+
+	baseParser := parselog.GetDynamicBaseParser(cnf.LogParser.PgSettings.LogLinePrefix)
+
+	var hbaRules []model.HBAFIleRules
+
+	// if user is passing hba conf file manually then he or she are expecting that file to be scanned
+	if cnf.LogParser.HbaConfFile != "" {
+		var err error
+		hbaRules, err = hbarules.ScanHBAFile(ctx, store, cnf.LogParser.HbaConfFile)
+		if err != nil {
+			fmt.Println("Got error while scanning hba file:", err)
+			return
+		}
+	} else if store != nil {
+		var err error
+		hbaRules, err = utils.GetDatabaseAndHostForUSerFromHbaFileRules(ctx, store)
+		if err != nil {
+			fmt.Println("Got error while getting hba rules:", err)
+			return
+		}
+	} else {
+		fmt.Println("Please provide hba file or database connection")
+		return
+	}
+
+	hbaValidator, err := hbarules.ParseHBAFileRules(hbaRules)
+	if err != nil {
+		fmt.Println("Got error while parsing hba rules:", err)
+		return
+	}
+
+	hbaUnusedLineParser := parselog.NewHbaUnusedLines(cnf, baseParser, hbaValidator)
+	runner.RunFastParser(ctx, cnf, hbaUnusedLineParser.Feed, parselog.GetBaseParserValidator(baseParser))
+
+	if ctx.Err() != nil {
+		fmt.Println("file parsing is taking longer then expected, please check the file or errors in" + logger.GetLogFileName())
+		return
+	}
+
+	fmt.Println("")
+	fmt.Println("Unused lines found from given log file:", hbaValidator.GetUnusedLines())
+	fmt.Println("")
+}
+
 func runMismatchIPsLogParser(ctx context.Context, cnf *config.Config) {
 
 	// check if postgres setting contains required variable or connection logs

docker_testing/database

@@ -0,0 +1,2 @@
+newdbs
+teestingdbs

docker_testing/integrationtest/setup.go

@@ -53,7 +53,7 @@ func init() {
 }
 
 func startPostgres() {
-	cmd := exec.Command("docker-compose", "up", "--build", "-d", "postgres")
+	cmd := exec.Command("docker", "compose", "up", "--build", "-d", "postgres")
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
 	cmd.Stdin = os.Stdin
@@ -72,7 +72,7 @@ func startPostgres() {
 }
 
 func createUSers() {
-	cmd := exec.Command("docker-compose", "run", "--rm", "createuser")
+	cmd := exec.Command("docker", "compose", "run", "--rm", "createuser")
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
 	cmd.Stdin = os.Stdin
@@ -93,7 +93,7 @@ func createUSers() {
 func execPgbench(wg *sync.WaitGroup, pgUsers, ip string) {
 	defer wg.Done()
 	fmt.Println("executing pgbench command for users:", pgUsers, "and ip:", ip)
-	cmd := exec.Command("docker-compose", "run", "--rm", "pgbench")
+	cmd := exec.Command("docker", "compose", "run", "--rm", "pgbench")
 	cmd.Env = append(os.Environ(),
 		"PGUSERS="+pgUsers,
 		"IP="+ip,

docker_testing/integrationtest/testcase.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"bytes"
 	"fmt"
 	"os"
 	"os/exec"
@@ -20,6 +21,7 @@ func init() {
 			testInactiveUser(prefix, filename)
 			// testMissingIPs(prefix, filename)
 			testUniqueIPs(prefix, filename)
+			testUnusedHbaLines(prefix, filename)
 		},
 	}
 
@@ -152,3 +154,46 @@ func testMissingIPs(prefix, file string) {
 
 	fmt.Println("mismatch ip test is working fine for prefix:", prefix)
 }
+
+func testUnusedHbaLines(prefix, file string) {
+	cmd := exec.Command("ciscollector",
+		"-logparser", cons.LogParserCMD_HBAUnusedLines,
+		"-prefix", prefix,
+		"-file-path", file,
+		"-output-type", "json",
+		"-hba-file", "./pg_hba.conf",
+	)
+
+	// create io.Writer to store output and print it later
+	var buf bytes.Buffer
+
+	cmd.Stdout = &buf
+	cmd.Stderr = os.Stderr
+	cmd.Stdin = os.Stdin
+
+	err := cmd.Run()
+	if err != nil {
+		fmt.Println("Got error while parsing file:", err)
+		os.Exit(1)
+	}
+
+	out := buf.String()
+	if strings.Contains(out, "In logline prefix, please set '%u' and '%d'") || strings.Contains(out, "Please set log_line_prefix to '%h' or '%r' or enable log_connections") {
+		fmt.Println("skipping test for unused files as required details are not available in prefix:", prefix)
+		return
+	}
+
+	if !strings.Contains(out, "Successfully parsed all files") {
+		fmt.Println("Got error while parsing file:", out)
+		// fail the command
+		os.Exit(1)
+	}
+
+	if strings.Contains(out, `Unused lines found from given log file: [11 23 28]`) {
+		fmt.Println("unused lines test is working fine for prefix:", prefix)
+		return
+	}
+
+	fmt.Println("not getting valid unused lines:", out)
+	os.Exit(1)
+}

docker_testing/pg_hba.conf

@@ -0,0 +1,28 @@
+# Allow any user on the local system to connect to any database with
+# any database user name using Unix-domain sockets (the default for local
+# connections).
+#
+# TYPE  DATABASE        USER            ADDRESS                 METHOD
+local   all             all                                     trust
+
+# The same using local loopback TCP/IP connections.
+#
+# TYPE  DATABASE        USER            ADDRESS                 METHOD
+host    all             all             192.168.1.1/24            trust
+
+# The same as the previous line, but using a separate netmask column
+#
+# TYPE  DATABASE        USER            IP-ADDRESS      IP-MASK             METHOD
+host    all             user4           192.168.0.1       255.255.255.0     trust
+host    all             all             192.168.0.1       255.255.255.0     trust
+# host    all             all             all                    trust
+
+# The same over IPv6.
+#
+# TYPE  DATABASE        USER            ADDRESS                 METHOD
+host    all             all             ::1/128                 trust
+
+# The same using a host name (would typically cover both IPv4 and IPv6).
+#
+# TYPE  DATABASE        USER            ADDRESS                 METHOD
+host    all             all             localhost               trust

docker_testing/runner.sh

@@ -7,7 +7,7 @@
 ############################ widely used prefixes #################################
 # PREFIXES=('%t %h %u %m ' '%m (%h:%u) ' '%m from %h by %u ' '%m in %d by %u@%h ' '%t %h %u [%p] ' '%m (%h:%u:%p) ' '%m from %h by %u pid=%p ' '%m in %d by %u@%h pid=%p ' '%t %h %u db=%d %m ' '%m in %d by %u@%h db=%d')
 ####################################################################################
-PREFIXES=('%t [%p]: ')
+PREFIXES=('%m in %d by %u@%h ')
 
 FILE_SIZE=12MB # to modify file size
 
@@ -42,4 +42,4 @@ done
 
 
 # Set permissions for pglog directory
-sudo chmod 777 pglog/*
+sudo chmod 777 pglog/*

docker_testing/users

@@ -0,0 +1,5 @@
+user0
+user1
+user2
+user3
+user4

go.mod

@@ -3,6 +3,7 @@ module github.com/klouddb/klouddbshield
 go 1.18
 
 require (
+	github.com/DATA-DOG/go-sqlmock v1.5.0
 	github.com/go-sql-driver/mysql v1.7.0
 	github.com/hashicorp/go-version v1.6.0
 	github.com/jackc/pgx/v4 v4.18.3

go.sum

@@ -38,6 +38,8 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=
+github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
 github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
 github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef h1:46PFijGLmAjMPwCCCo7Jf0W6f9slllCkkv7vyc1yOSg=
 github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=

model/hba.go

@@ -0,0 +1,9 @@
+package model
+
+type HBAFIleRules struct {
+	LineNumber int    `json:"line_number"`
+	Database   string `json:"database"`
+	UserName   string `json:"user_name"`
+	Address    string `json:"address"`
+	NetMask    string `json:"netmask"`
+}

pkg/config/config.go

@@ -37,17 +37,20 @@ type LogParser struct {
 
 	IpFilePath string
 
+	HbaConfFile string
+
 	OutputType string
 }
 
-func NewLogParser(command, beginTime, endTime, prefix, logfile, ipfile string) (*LogParser, error) {
+func NewLogParser(command, beginTime, endTime, prefix, logfile, ipfile, hbaConfigFile string) (*LogParser, error) {
 	prefix = strings.TrimSpace(prefix)
 	logfile = strings.TrimSpace(logfile)
 	ipfile = strings.TrimSpace(ipfile)
 	beginTime = strings.TrimSpace(beginTime)
 	endTime = strings.TrimSpace(endTime)
+	hbaConfigFile = strings.TrimSpace(hbaConfigFile)
 
-	if command != cons.LogParserCMD_UniqueIPs && command != cons.LogParserCMD_InactiveUsr && command != cons.LogParserCMD_MismatchIPs {
+	if command != cons.LogParserCMD_UniqueIPs && command != cons.LogParserCMD_InactiveUsr && command != cons.LogParserCMD_MismatchIPs && command != cons.LogParserCMD_HBAUnusedLines {
 		return nil, fmt.Errorf("invalid command %s, please use unique_ip, mismatch_ips or inactive_users", command)
 	}
 
@@ -100,8 +103,9 @@ func NewLogParser(command, beginTime, endTime, prefix, logfile, ipfile string) (
 		Begin: begin,
 		End:   end,
 
-		LogFiles:   files,
-		IpFilePath: ipfile,
+		LogFiles:    files,
+		IpFilePath:  ipfile,
+		HbaConfFile: hbaConfigFile,
 	}, nil
 }
 
@@ -198,20 +202,29 @@ func NewConfig() (*Config, error) {
 	var logParser string
 	var logfile string
 	flag.StringVar(&logParser, "logparser", logParser, `To run Log Parser. Supported commands are:
-1. unique_ip: To get unique IPs from log file NOTE: --begin-time and --end-time are optional flags and --prefix and --file-path are required flags if you are using --logparser
+1. unique_ip: To get unique IPs from log file NOTE: --begin-time and --end-time are optional flags and --prefix and --file-path are required flags if you are using --logparser=unique_ip
 e.g 
 * ciscollector --logparser unique_ip --file-path /location/to/log/file.log --begin-time "2021-01-01 00:00:00" --end-time "2021-01-01 23:59:59" --prefix <logline prefix>
 * ciscollector --logparser unique_ip --file-path /location/to/log/file.log --prefix <logline prefix>
 * ciscollector --logparser unique_ip --file-path /location/to/log/*.log --begin-time "2021-01-01 00:00:00" --end-time "2021-01-01 23:59:59" --prefix <logline prefix>
 * ciscollector --logparser unique_ip --file-path /location/to/log/*.log --prefix <logline prefix>
 
-2. inactive_users: To get inactive users from log file	NOTE: --begin-time and --end-time are optional flags and --prefix and --file-path are required flags if you are using --logparser
+2. inactive_users: To get inactive users from log file	NOTE: --begin-time and --end-time are optional flags and --prefix and --file-path are required flags if you are using --logparser=inactive_users
 e.g
 * ciscollector --logparser inactive_users --file-path /location/to/log/file.log --begin-time "2021-01-01 00:00:00" --end-time "2021-01-01 23:59:59" --prefix <logline prefix>
 * ciscollector --logparser inactive_users --file-path /location/to/log/file.log --prefix <logline prefix>
 * ciscollector --logparser inactive_users --file-path /location/to/log/*.log --begin-time "2021-01-01 00:00:00" --end-time "2021-01-01 23:59:59" --prefix <logline prefix>
 * ciscollector --logparser inactive_users --file-path /location/to/log/*.log --prefix <logline prefix>
-	`)
+
+3. unused_lines: To get unused lines from pg_hba.conf file by comparing that with log file 
+NOTE: --begin-time and --end-time are optional flags and --prefix, --file-path and --hba-file are required flags if you are using --logparser=unused_lines
+e.g
+* ciscollector --logparser unused_lines --file-path /location/to/log/file.log --begin-time "2021-01-01 00:00:00" --end-time "2021-01-01 23:59:59" --prefix <logline prefix> --hba-file /location/to/pg_hba.conf
+* ciscollector --logparser unused_lines --file-path /location/to/log/file.log --prefix <logline prefix> --hba-file /location/to/pg_hba.conf
+* ciscollector --logparser unused_lines --file-path /location/to/log/*.log --begin-time "2021-01-01 00:00:00" --end-time "2021-01-01 23:59:59" --prefix <logline prefix> --hba-file /location/to/pg_hba.conf
+* ciscollector --logparser unused_lines --file-path /location/to/log/*.log --prefix <logline prefix> --hba-file /location/to/pg_hba.conf
+
+`)
 	flag.StringVar(&logfile, "file-path", "", "File path e.g /location/to/log/file.log. required for all commands in log parser")
 
 	var beginTime, endTime string
@@ -222,7 +235,9 @@ e.g
 	var prefix string
 	flag.StringVar(&prefix, "prefix", "", "Log line prefix for offline parsing. required for all commands in log parser")
 	var ipFilePath string
-	flag.StringVar(&ipFilePath, "ip-file-path", "", "File path for ip list. requered for mismatch_ips command in log parser")
+	// flag.StringVar(&ipFilePath, "ip-file-path", "", "File path for ip list. requered for mismatch_ips command in log parser") // TODO removed because we are not using missing_ip command
+	var hbaConfigFile string
+	flag.StringVar(&hbaConfigFile, "hba-file", "", "file path for pg_hba.conf. for unused_lines command in log parser")
 	var outputType string
 	flag.StringVar(&outputType, "output-type", "", "Output type for log parser. supported types are json, csv, table")
 
@@ -240,7 +255,7 @@ e.g
 	var logParserConf *LogParser
 	if logParser != "" {
 		var err error
-		logParserConf, err = NewLogParser(logParser, beginTime, endTime, prefix, logfile, ipFilePath)
+		logParserConf, err = NewLogParser(logParser, beginTime, endTime, prefix, logfile, ipFilePath, hbaConfigFile)
 		if err != nil {
 			fmt.Println("Invalid input for logparser:", err)
 			flag.Usage()
@@ -502,7 +517,12 @@ func getLogParserInputs() *LogParser {
 		ipfile = reader.Read("Enter IP File Path: ")
 	}
 
-	l, err := NewLogParser(command, beginTime, endTime, prefix, logfile, ipfile)
+	var hbaConfigFile string
+	if command == cons.LogParserCMD_HBAUnusedLines {
+		hbaConfigFile = reader.Read("Enter pg_hba.conf File Path: ")
+	}
+
+	l, err := NewLogParser(command, beginTime, endTime, prefix, logfile, ipfile, hbaConfigFile)
 	if err != nil {
 		fmt.Println("Invalid input for logparser:", err)
 		os.Exit(1)

pkg/const/const.go

@@ -26,20 +26,23 @@ Enter your choice to execute(1/2/3/4/5/6):`
 	MSG_LogPaserChoise = `
 1. Inactive Users
 2. Unique IPs
+3. HBA Unused Lines
 
-Enter your choice to execute(1/2/3):`
+Enter your choice to execute(1/2/3/4):`
 )
 
 const (
-	LogParserCMD_MismatchIPs = "mismatch_ips"
-	LogParserCMD_UniqueIPs   = "unique_ip"
-	LogParserCMD_InactiveUsr = "inactive_users"
+	LogParserCMD_MismatchIPs    = "mismatch_ips"
+	LogParserCMD_UniqueIPs      = "unique_ip"
+	LogParserCMD_InactiveUsr    = "inactive_users"
+	LogParserCMD_HBAUnusedLines = "unused_lines"
 )
 
 var (
 	LogParserChoiseMapping = map[int]string{
 		// 1: LogParserCMD_MismatchIPs,
 		1: LogParserCMD_InactiveUsr,
 		2: LogParserCMD_UniqueIPs,
+		3: LogParserCMD_HBAUnusedLines,
 	}
 )