Skip to content

Commit

Permalink
new(pkg,cmd): added new stats subcmd.
Browse files Browse the repository at this point in the history
Signed-off-by: Federico Di Pierro <nierro92@gmail.com>
  • Loading branch information
FedeDP committed Aug 29, 2023
1 parent 47078a6 commit 9f78978
Show file tree
Hide file tree
Showing 14 changed files with 533 additions and 83 deletions.
4 changes: 3 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ Right now, the tool implements:
* configs generation (comprehensive of automatic generation from kernel-crawler output)
* configs cleanup
* configs validation
* configs stats

This is enough to port [`update-dbg` image](https://github.com/falcosecurity/test-infra/tree/master/images/update-dbg) to make use of this tool instead of the currently used bash scripts.
First benchmarks showed a tremendous perf improvement: old update-dbg scripts took around 50m on my laptop for a single driverversion. The new tool takes ~10s.
Expand All @@ -29,11 +30,12 @@ Available Commands:
completion Generate the autocompletion script for the specified shell
generate Generate new dbg configs
help Help about any command
stats Fetch stats about configs
validate Validate dbg configs
Flags:
-a, --architecture string architecture to run against. (default "x86_64")
--driver-version strings driver versions to generate configs against.
--driver-version strings driver versions to run against.
--dry-run enable dry-run mode.
-h, --help help for dbg-go
-l, --log-level string set log verbosity. (default "INFO")
Expand Down
4 changes: 3 additions & 1 deletion cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"fmt"
"github.com/fededp/dbg-go/cmd/cleanup"
"github.com/fededp/dbg-go/cmd/generate"
"github.com/fededp/dbg-go/cmd/stats"
"github.com/fededp/dbg-go/cmd/validate"
"github.com/fededp/dbg-go/pkg/root"
"github.com/fededp/dbg-go/pkg/utils"
Expand Down Expand Up @@ -73,7 +74,7 @@ func init() {
flags.StringP("log-level", "l", logger.LevelInfo.String(), "set log verbosity.")
flags.String("repo-root", cwd, "test-infra repository root path.")
flags.StringP("architecture", "a", utils.FromDebArch(runtime.GOARCH), "architecture to run against.")
flags.StringSlice("driver-version", nil, "driver versions to generate configs against.")
flags.StringSlice("driver-version", nil, "driver versions to run against.")
flags.String("target-kernelrelease", "",
`target kernel release to work against. By default tool will work on any kernel release. Can be a regex.`)
flags.String("target-kernelversion", "",
Expand All @@ -97,6 +98,7 @@ func init() {
rootCmd.AddCommand(generate.Cmd)
rootCmd.AddCommand(cleanup.Cmd)
rootCmd.AddCommand(validate.Cmd)
rootCmd.AddCommand(stats.Cmd)
}

func initLogger(subcmd string) error {
Expand Down
19 changes: 19 additions & 0 deletions cmd/stats/stats.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
package stats

import (
"github.com/fededp/dbg-go/pkg/root"
"github.com/fededp/dbg-go/pkg/stats"
"github.com/spf13/cobra"
)

var (
Cmd = &cobra.Command{
Use: "stats",
Short: "Fetch stats about configs",
RunE: execute,
}
)

func execute(c *cobra.Command, args []string) error {
return stats.Run(stats.Options{Options: root.LoadRootOptions()})
}
3 changes: 3 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ go 1.21

require (
github.com/falcosecurity/driverkit v0.14.1-0.20230828134718-835307efe091
github.com/olekukonko/tablewriter v0.0.4
github.com/ompluscator/dynamic-struct v1.4.0
github.com/spf13/cobra v1.7.0
github.com/spf13/viper v1.16.0
Expand All @@ -23,12 +24,14 @@ require (
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
github.com/magiconair/properties v1.8.7 // indirect
github.com/mattn/go-isatty v0.0.16 // indirect
github.com/mattn/go-runewidth v0.0.13 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.0-rc2 // indirect
github.com/pelletier/go-toml/v2 v2.0.8 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
github.com/rivo/uniseg v0.2.0 // indirect
github.com/sirupsen/logrus v1.9.0 // indirect
github.com/spf13/afero v1.9.5 // indirect
github.com/spf13/cast v1.5.1 // indirect
Expand Down
7 changes: 7 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -158,10 +158,15 @@ github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3v
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mattn/go-sqlite3 v1.14.12 h1:TJ1bhYJPV44phC+IMu1u2K/i5RriLTPe+yc68XDJ1Z0=
github.com/mattn/go-sqlite3 v1.14.12/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/olekukonko/tablewriter v0.0.4 h1:vHD/YYe1Wolo78koG299f7V/VAS08c6IpCLn+Ejf/w8=
github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA=
github.com/ompluscator/dynamic-struct v1.4.0 h1:I/Si9LZtItSwiTMe7vosEuIu2TKdOvWbE3R/lokpN4Q=
github.com/ompluscator/dynamic-struct v1.4.0/go.mod h1:ADQ1+6Ox1D+ntuNwTHyl1NvpAqY2lBXPSPbcO4CJdeA=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
Expand All @@ -178,6 +183,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk=
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
Expand Down
29 changes: 4 additions & 25 deletions pkg/cleanup/cleanup.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,8 @@ package cleanup
import (
"fmt"
"github.com/fededp/dbg-go/pkg/root"
"github.com/fededp/dbg-go/pkg/validate"
logger "log/slog"
"os"
"path/filepath"
)

func Run(opts Options) error {
Expand Down Expand Up @@ -41,27 +39,8 @@ func cleanupFolder(opts Options, driverVersion string) error {
}

func cleanupMatchingConfigs(opts Options, driverVersion string) error {
configNameGlob := validate.ConfGlobFromDistro(opts.Target)

configPath := fmt.Sprintf(root.ConfigPathFmt,
opts.RepoRoot,
driverVersion,
opts.Architecture,
configNameGlob)

files, err := filepath.Glob(configPath)
if err != nil {
return err
}
for _, f := range files {
logger.Info("removing file", "path", f)
if opts.DryRun {
logger.Info("skipping because of dry-run.")
continue
}
if err = os.Remove(f); err != nil {
return err
}
}
return nil
opts.DriverVersion = []string{driverVersion} // locally overwrite driverVersions to only match current driverVersion
return root.LoopConfigsFiltered(opts.Options, "removing file", func(driverVersion, configPath string) error {
return os.Remove(configPath)
})
}
3 changes: 2 additions & 1 deletion pkg/cleanup/cleanup_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ func TestCleanupFiltered(t *testing.T) {
// then, for each parsed logged line, check if it contains one of the requested string by the test.
// Count all "containing" lines; they must match total lines logged (that have a "config:" key).
type MessageJSON struct {
Path string `json:"path,omitempty"`
Path string `json:"config,omitempty"`
}
var messageJSON MessageJSON
scanner := bufio.NewScanner(&buf)
Expand All @@ -219,6 +219,7 @@ func TestCleanupFiltered(t *testing.T) {
if found != lines {
t.Errorf("wrong number of printed lines; expected %d, found %d", lines, found)
}
buf.Reset()
})
}

Expand Down
25 changes: 25 additions & 0 deletions pkg/root/types.go
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
package root

import (
"fmt"
"github.com/spf13/viper"
logger "log/slog"
"strings"
)

type Target struct {
Expand All @@ -15,6 +17,29 @@ func (t Target) IsSet() bool {
return t.Distro != "" && t.KernelRelease != "" && t.KernelVersion != ""
}

func (t Target) toGlob() string {
// Empty filters fallback at ".*" since we are using a regex match below
if t.Distro == "" {
t.Distro = "*"
} else {
dkDistro, found := SupportedDistros[t.Distro]
if found {
// Filenames use driverkit lowercase target, instead of the kernel-crawler naming.
t.Distro = dkDistro
} else {
// Perhaps a regex? ToLower and pray
t.Distro = strings.ToLower(t.Distro)
}
}
if t.KernelRelease == "" {
t.KernelRelease = "*"
}
if t.KernelVersion == "" {
t.KernelVersion = "*"
}
return fmt.Sprintf("%s_%s_%s.yaml", t.Distro, t.KernelRelease, t.KernelVersion)
}

type Options struct {
DryRun bool
RepoRoot string
Expand Down
36 changes: 36 additions & 0 deletions pkg/root/utils.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
package root

import (
"fmt"
logger "log/slog"
"path/filepath"
)

type ConfigLooper func(driverVersion, configPath string) error

func LoopConfigsFiltered(opts Options, message string, worker ConfigLooper) error {
configNameGlob := opts.Target.toGlob()
for _, driverVersion := range opts.DriverVersion {
configPath := fmt.Sprintf(ConfigPathFmt,
opts.RepoRoot,
driverVersion,
opts.Architecture,
configNameGlob)
configs, err := filepath.Glob(configPath)
if err != nil {
return err
}
for _, config := range configs {
logger.Info(message, "config", config)
if opts.DryRun {
logger.Info("skipping because of dry-run.")
return nil
}
err = worker(driverVersion, config)
if err != nil {
return err
}
}
}
return nil
}
84 changes: 84 additions & 0 deletions pkg/stats/stats.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
package stats

import (
"github.com/fededp/dbg-go/pkg/root"
"github.com/fededp/dbg-go/pkg/validate"
"github.com/olekukonko/tablewriter"
"gopkg.in/yaml.v3"
"log"
logger "log/slog"
"os"
"strconv"
)

func Run(opts Options) error {
logger.Info("fetching stats from existing config files")
driverStatsByVersion := make(map[string]driverStats)
totalDriverStats := driverStats{}
err := root.LoopConfigsFiltered(opts.Options, "computing stats", func(driverVersion, configPath string) error {
dStats := driverStatsByVersion[driverVersion]
err := getConfigStats(&dStats, configPath)
driverStatsByVersion[driverVersion] = dStats
return err
})
if err != nil {
return err
}

table := tablewriter.NewWriter(log.Default().Writer())
table.SetHeader([]string{"Version", "Modules", "Probes", "Headers", "KernelConfigData"})
table.SetBorders(tablewriter.Border{Left: true, Top: false, Right: true, Bottom: false})
table.SetCenterSeparator("|")

data := make([]string, 5)
for key, stat := range driverStatsByVersion {
data[0] = key
data[1] = strconv.FormatInt(stat.NumModules, 10)
data[2] = strconv.FormatInt(stat.NumProbes, 10)
data[3] = strconv.FormatInt(stat.NumHeaders, 10)
data[4] = strconv.FormatInt(stat.NumKernelConfigDatas, 10)
table.Append(data)

totalDriverStats.NumModules += stat.NumModules
totalDriverStats.NumProbes += stat.NumProbes
totalDriverStats.NumHeaders += stat.NumHeaders
totalDriverStats.NumKernelConfigDatas += stat.NumKernelConfigDatas
}
data[0] = "TOTALS"
data[1] = strconv.FormatInt(totalDriverStats.NumModules, 10)
data[2] = strconv.FormatInt(totalDriverStats.NumProbes, 10)
data[3] = strconv.FormatInt(totalDriverStats.NumHeaders, 10)
data[4] = strconv.FormatInt(totalDriverStats.NumKernelConfigDatas, 10)
table.Append(data)
table.Render() // Send output

return nil
}

func getConfigStats(dStats *driverStats, configPath string) error {
configData, err := os.ReadFile(configPath)
if err != nil {
return err
}
var driverkitYaml validate.DriverkitYaml
err = yaml.Unmarshal(configData, &driverkitYaml)
if err != nil {
return err
}

logger.Debug("fetching stats", "parsedConfig", driverkitYaml)

if driverkitYaml.Output.Probe != "" {
dStats.NumProbes++
}
if driverkitYaml.Output.Module != "" {
dStats.NumModules++
}
if len(driverkitYaml.KernelUrls) > 0 {
dStats.NumHeaders++
}
if driverkitYaml.KernelConfigData != "" {
dStats.NumKernelConfigDatas++
}
return nil
}
Loading

0 comments on commit 9f78978

Please sign in to comment.