Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: read and write YAML config files #56

Merged
merged 5 commits into from
Jun 11, 2024
Merged

feat: read and write YAML config files #56

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
4990417
Move some options into Global
kentquirk Jun 6, 2024
bc5ac1d
Add ability to read and write YAML config files
kentquirk Jun 10, 2024
8923138
Doc fixes
kentquirk Jun 10, 2024
064f7d2
Update deps
kentquirk Jun 10, 2024
09ce2b6
Fix readme
kentquirk Jun 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,18 @@ Ramp up and down are handled only by increasing or decreasing the number of goro

To mix different kinds of traces, or send traces to multiple datasets, use multiple loadgen processes.

## Configuration File

A YAML configuration file can be used by specifying `--config=filename`.
The format of the YAML file reflects the configuration parameters.
See the file [sample_config.yaml](https://github.com/honeycombio/loadgen/blob/main/sample_config.yaml) for an example.

For an easy way to convert an existing command line to a YAML file, use `--writecfg=outputfile`.
This will write the YAML equivalent of the complete configuration (except for the API key) to the specified location.

Fields, which are specified on the command line as `key=value` (without any `-` characters) can be specified in YAML
by adding key-value pairs under the `fields` key.

## Generators

After the list of options, loadgen permits a list of fields in the form of name=constant or name=/gen.
Expand Down
51 changes: 26 additions & 25 deletions fielder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,14 @@ var nouns = []string{
"watch", "wheel", "whip", "whistle", "window", "wing", "wire", "worm",
}

var constpat = regexp.MustCompile(`^([a-zA-Z0-9_.]+)=([^/].*)$`)
var genpat = regexp.MustCompile(`^((?:[0-9]+\.)?[a-zA-Z0-9_]+)=/([ibfsu][awxrgqt]?)([0-9.-]+)?(,[0-9.-]+)?$`)
var keypat = regexp.MustCompile(`^([0-9]+)\.(.*$)`)
// constfield is a field that *doesn't* start with slash
var constfield = regexp.MustCompile(`^([^/].*)$`)

// genfield is used to parse generator fields by matching valid commands and numeric arguments
var genfield = regexp.MustCompile(`^/([ibfsu][awxrgqt]?)([0-9.-]+)?(,[0-9.-]+)?$`)

// keysplitter separates fields that look like number.name (ex: 1.myfield)
var keysplitter = regexp.MustCompile(`^([0-9]+)\.(.*$)`)

type Rng struct {
rng *rand.Rand
Expand Down Expand Up @@ -166,47 +171,43 @@ func getWordList(rng Rng, cardinality int, source []string) []string {

// parseUserFields expects a list of fields in the form of name=constant or name=/gen.
// See README.md for more information.
func parseUserFields(rng Rng, userfields []string) (map[string]func() any, error) {
func parseUserFields(rng Rng, userfields map[string]string) (map[string]func() any, error) {
// groups 1 2 3 4
fields := make(map[string]func() any)
for _, field := range userfields {
for name, value := range userfields {
// see if it's a constant
matches := constpat.FindStringSubmatch(field)
if matches != nil {
name := matches[1]
value := matches[2]
if constfield.MatchString(value) {
fields[name] = getConst(value)
continue
}

// see if it's a generator
matches = genpat.FindStringSubmatch(field)
matches := genfield.FindStringSubmatch(value)
if matches == nil {
return nil, fmt.Errorf("unparseable user field %s", field)
return nil, fmt.Errorf("unparseable user field %s=%s", name, value)
}
var err error
name := matches[1]
gentype := matches[2]
p1 := matches[3]
p2 := matches[4]
gentype := matches[1]
p1 := matches[2]
p2 := matches[3]
switch gentype {
case "i", "ir", "ig":
fields[name], err = getIntGen(rng, gentype, p1, p2)
if err != nil {
return nil, fmt.Errorf("invalid int in user field %s: %w", field, err)
return nil, fmt.Errorf("invalid int in user field %s=%s: %w", name, value, err)
}
case "f", "fr", "fg":
fields[name], err = getFloatGen(rng, gentype, p1, p2)
if err != nil {
return nil, fmt.Errorf("invalid float in user field %s: %w", field, err)
return nil, fmt.Errorf("invalid float in user field %s=%s: %w", name, value, err)
}
case "b":
n := 50.0
var err error
if p1 != "" {
n, err = strconv.ParseFloat(p1, 64)
if err != nil || n < 0 || n > 100 {
return nil, fmt.Errorf("invalid bool option in %s", field)
return nil, fmt.Errorf("invalid bool option in %s=%s", name, value)
}
}
fields[name] = func() any { return rng.BoolWithProb(n) }
Expand All @@ -215,7 +216,7 @@ func parseUserFields(rng Rng, userfields []string) (map[string]func() any, error
if p1 != "" {
n, err = strconv.Atoi(p1)
if err != nil {
return nil, fmt.Errorf("invalid string option in %s", field)
return nil, fmt.Errorf("invalid string option in %s=%s", name, value)
}
}
switch gentype {
Expand All @@ -236,7 +237,7 @@ func parseUserFields(rng Rng, userfields []string) (map[string]func() any, error
// Generate a URL-like string with a random path and possibly a query string
fields[name], err = getURLGen(rng, gentype, p1, p2)
if err != nil {
return nil, fmt.Errorf("invalid float in user field %s: %w", field, err)
return nil, fmt.Errorf("invalid float in user field %s=%s: %w", name, value, err)
}
case "st":
// Generate a semi-plausible mix of status codes; percentage of 400s and 500s can be controlled by the extra args
Expand All @@ -246,13 +247,13 @@ func parseUserFields(rng Rng, userfields []string) (map[string]func() any, error
if p1 != "" {
fours, err = strconv.ParseFloat(p1, 64)
if err != nil {
return nil, fmt.Errorf("invalid float in user field %s: %w", field, err)
return nil, fmt.Errorf("invalid float in user field %s=%s: %w", name, value, err)
}
}
if p2 != "" {
fives, err = strconv.ParseFloat(p2[1:], 64)
if err != nil {
return nil, fmt.Errorf("invalid float in user field %s: %w", field, err)
return nil, fmt.Errorf("invalid float in user field %s=%s: %w", name, value, err)
}
}
twos = 100 - fours - fives
Expand All @@ -268,7 +269,7 @@ func parseUserFields(rng Rng, userfields []string) (map[string]func() any, error
}

default:
return nil, fmt.Errorf("invalid generator type %s in field %s", gentype, field)
return nil, fmt.Errorf("invalid generator type %s in field %s=%s", gentype, name, value)
}
}
return fields, nil
Expand Down Expand Up @@ -411,7 +412,7 @@ type Fielder struct {
// combining an adjective and a noun and are consistent for a given fielder.
// The field values are randomly generated.
// Fielder also includes the process_id.
func NewFielder(seed string, userFields []string, nextras, nservices int) (*Fielder, error) {
func NewFielder(seed string, userFields map[string]string, nextras, nservices int) (*Fielder, error) {
rng := NewRng(seed)
gens := rng.getValueGenerators()
fields, err := parseUserFields(rng, userFields)
Expand Down Expand Up @@ -440,7 +441,7 @@ func (f *Fielder) GetServiceName(n int) string {
// indicate that the field should be included at a specific
// level in the trace, where 0 is the root.
func (f *Fielder) atLevel(name string, level int) (string, bool) {
matches := keypat.FindStringSubmatch(name)
matches := keysplitter.FindStringSubmatch(name)
if len(matches) == 0 {
return name, true
}
Expand Down
6 changes: 3 additions & 3 deletions generator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import (
// taking opts.Duration to do so. Its TPS method returns the number of traces
// per second it is currently generating.
type Generator interface {
Generate(opts Options, wg *sync.WaitGroup, stop chan struct{}, counter chan int64)
Generate(opts *Options, wg *sync.WaitGroup, stop chan struct{}, counter chan int64)
TPS() float64
}

Expand All @@ -39,7 +39,7 @@ type TraceGenerator struct {
// make sure it implements Generator
var _ Generator = (*TraceGenerator)(nil)

func NewTraceGenerator(tsender Sender, getFielder func() *Fielder, log Logger, opts Options) *TraceGenerator {
func NewTraceGenerator(tsender Sender, getFielder func() *Fielder, log Logger, opts *Options) *TraceGenerator {
chans := make([]chan struct{}, 0)
return &TraceGenerator{
depth: opts.Format.Depth,
Expand Down Expand Up @@ -146,7 +146,7 @@ func (s *TraceGenerator) generator(wg *sync.WaitGroup, counter chan int64) {
}
}

func (s *TraceGenerator) Generate(opts Options, wg *sync.WaitGroup, stop chan struct{}, counter chan int64) {
func (s *TraceGenerator) Generate(opts *Options, wg *sync.WaitGroup, stop chan struct{}, counter chan int64) {
defer wg.Done()
ngenerators := float64(opts.Quantity.TPS) / s.TPS()
uSgeneratorInterval := float64(opts.Quantity.RampTime.Microseconds()) / ngenerators
Expand Down
75 changes: 38 additions & 37 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,57 +7,58 @@ toolchain go1.21.1
require (
github.com/dgryski/go-wyhash v0.0.0-20191203203029-c4841ae36371
github.com/goware/urlx v0.3.2
github.com/honeycombio/beeline-go v1.15.0
github.com/honeycombio/otel-config-go v1.14.0
github.com/honeycombio/beeline-go v1.16.0
github.com/honeycombio/otel-config-go v1.15.0
github.com/jessevdk/go-flags v1.5.0
go.opentelemetry.io/otel v1.24.0
go.opentelemetry.io/otel/trace v1.24.0
github.com/jonboulle/clockwork v0.4.0
go.opentelemetry.io/otel v1.27.0
go.opentelemetry.io/otel/trace v1.27.0
gopkg.in/yaml.v3 v3.0.1
pgregory.net/rand v1.0.2
)

require (
github.com/PuerkitoBio/purell v1.2.1 // indirect
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect
github.com/facebookgo/limitgroup v0.0.0-20150612190941-6abd8d71ec01 // indirect
github.com/facebookgo/muster v0.0.0-20150708232844-fd3d7953fd52 // indirect
github.com/go-logr/logr v1.4.1 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-ole/go-ole v1.3.0 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 // indirect
github.com/honeycombio/libhoney-go v1.22.0 // indirect
github.com/klauspost/compress v1.17.4 // indirect
github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed // indirect
github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect
github.com/sethvargo/go-envconfig v1.0.0 // indirect
github.com/shirou/gopsutil/v3 v3.23.12 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
github.com/honeycombio/libhoney-go v1.23.0 // indirect
github.com/klauspost/compress v1.17.8 // indirect
github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae // indirect
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
github.com/sethvargo/go-envconfig v1.0.3 // indirect
github.com/shirou/gopsutil/v3 v3.24.5 // indirect
github.com/shoenig/go-m1cpu v0.1.6 // indirect
github.com/tklauser/go-sysconf v0.3.13 // indirect
github.com/tklauser/numcpus v0.7.0 // indirect
github.com/tklauser/go-sysconf v0.3.14 // indirect
github.com/tklauser/numcpus v0.8.0 // indirect
github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
github.com/yusufpapurcu/wmi v1.2.3 // indirect
go.opentelemetry.io/contrib/instrumentation/host v0.46.1 // indirect
go.opentelemetry.io/contrib/instrumentation/runtime v0.46.1 // indirect
go.opentelemetry.io/contrib/propagators/b3 v1.21.1 // indirect
go.opentelemetry.io/contrib/propagators/ot v1.21.1 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.45.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.45.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.22.0 // indirect
go.opentelemetry.io/otel/metric v1.24.0 // indirect
go.opentelemetry.io/otel/sdk v1.22.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.22.0 // indirect
go.opentelemetry.io/proto/otlp v1.1.0 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
go.opentelemetry.io/contrib/instrumentation/host v0.52.0 // indirect
go.opentelemetry.io/contrib/instrumentation/runtime v0.52.0 // indirect
go.opentelemetry.io/contrib/propagators/b3 v1.27.0 // indirect
go.opentelemetry.io/contrib/propagators/ot v1.27.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.27.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.27.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 // indirect
go.opentelemetry.io/otel/metric v1.27.0 // indirect
go.opentelemetry.io/otel/sdk v1.27.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.27.0 // indirect
go.opentelemetry.io/proto/otlp v1.2.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/net v0.20.0 // indirect
golang.org/x/sys v0.16.0 // indirect
golang.org/x/text v0.14.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240116215550-a9fa1716bcac // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac // indirect
google.golang.org/grpc v1.61.0 // indirect
google.golang.org/protobuf v1.33.0 // indirect
golang.org/x/net v0.26.0 // indirect
golang.org/x/sys v0.21.0 // indirect
golang.org/x/text v0.16.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240610135401-a8a62080eff3 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240610135401-a8a62080eff3 // indirect
google.golang.org/grpc v1.64.0 // indirect
google.golang.org/protobuf v1.34.1 // indirect
gopkg.in/alexcesaro/statsd.v2 v2.0.0 // indirect
)
Loading
Loading