Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

cmd/geth: actually enable metrics when passed via toml #30781

Closed
wants to merge 4 commits into from
+106 −83
Closed

cmd/geth: actually enable metrics when passed via toml #30781

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
83b532c
cmd/geth: actually enable metrics when passed via toml
lightclient Nov 21, 2024
f6971a7
cmd/utils: unindent
holiman Nov 22, 2024
1635da8
metrics: peek into toml to decide enabling
lightclient Nov 22, 2024
4abf2e5
metrics: actually read flag value
lightclient Nov 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 4 additions & 5 deletions cmd/geth/chaincmd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -282,14 +282,13 @@ func importChain(ctx *cli.Context) error {
if ctx.Args().Len() < 1 {
utils.Fatalf("This command requires an argument.")
}
stack, cfg := makeConfigNode(ctx)
defer stack.Close()

// Start metrics export if enabled
utils.SetupMetrics(ctx)
// Start system runtime metrics collection
utils.SetupMetrics(&cfg.Metrics)
go metrics.CollectProcessMetrics(3 * time.Second)

stack, _ := makeConfigNode(ctx)
defer stack.Close()

chain, db := utils.MakeChain(ctx, stack, false)
defer db.Close()

Expand Down
7 changes: 6 additions & 1 deletion cmd/geth/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"runtime"
"slices"
"strings"
"time"
"unicode"

"github.com/ethereum/go-ethereum/accounts"
Expand Down Expand Up @@ -150,7 +151,7 @@ func loadBaseConfig(ctx *cli.Context) gethConfig {
// Load config file.
if file := ctx.String(configFileFlag.Name); file != "" {
if err := loadConfig(file, &cfg); err != nil {
utils.Fatalf("%v", err)
utils.Fatalf("failed to load config: %v", err)
}
}

Expand Down Expand Up @@ -192,6 +193,10 @@ func makeFullNode(ctx *cli.Context) *node.Node {
cfg.Eth.OverrideVerkle = &v
}

// Start metrics export if enabled
utils.SetupMetrics(&cfg.Metrics)
go metrics.CollectProcessMetrics(3 * time.Second)

backend, eth := utils.RegisterEthService(stack, &cfg.Eth)

// Create gauge with geth system and build information
Expand Down
7 changes: 0 additions & 7 deletions cmd/geth/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@ import (
"github.com/ethereum/go-ethereum/internal/debug"
"github.com/ethereum/go-ethereum/internal/flags"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/node"
"go.uber.org/automaxprocs/maxprocs"

Expand Down Expand Up @@ -325,12 +324,6 @@ func prepare(ctx *cli.Context) {
ctx.Set(utils.CacheFlag.Name, strconv.Itoa(4096))
}
}

// Start metrics export if enabled
utils.SetupMetrics(ctx)

// Start system runtime metrics collection
go metrics.CollectProcessMetrics(3 * time.Second)
}

// geth is the main entry point into the system if no special subcommand is run.
Expand Down
94 changes: 45 additions & 49 deletions cmd/utils/flags.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1969,64 +1969,60 @@ func RegisterFullSyncTester(stack *node.Node, eth *eth.Ethereum, target common.H
log.Info("Registered full-sync tester", "hash", target)
}

func SetupMetrics(ctx *cli.Context) {
if metrics.Enabled {
log.Info("Enabling metrics collection")

var (
enableExport = ctx.Bool(MetricsEnableInfluxDBFlag.Name)
enableExportV2 = ctx.Bool(MetricsEnableInfluxDBV2Flag.Name)
)

if enableExport || enableExportV2 {
CheckExclusive(ctx, MetricsEnableInfluxDBFlag, MetricsEnableInfluxDBV2Flag)

v1FlagIsSet := ctx.IsSet(MetricsInfluxDBUsernameFlag.Name) ||
ctx.IsSet(MetricsInfluxDBPasswordFlag.Name)

v2FlagIsSet := ctx.IsSet(MetricsInfluxDBTokenFlag.Name) ||
ctx.IsSet(MetricsInfluxDBOrganizationFlag.Name) ||
ctx.IsSet(MetricsInfluxDBBucketFlag.Name)
func SetupMetrics(cfg *metrics.Config) {
if !cfg.Enabled {
return
}
log.Info("Enabling metrics collection")
var (
enableExport = cfg.EnableInfluxDB
enableExportV2 = cfg.EnableInfluxDBV2
)
if enableExport && enableExportV2 {
Fatalf("Flags %v can't be used at the same time", strings.Join([]string{MetricsEnableInfluxDBFlag.Name, MetricsEnableInfluxDBV2Flag.Name}, ", "))
}
if enableExport || enableExportV2 {
v1FlagIsSet := cfg.InfluxDBUsername != "" || cfg.InfluxDBPassword != ""
v2FlagIsSet := cfg.InfluxDBToken != "" || cfg.InfluxDBOrganization != "" || cfg.InfluxDBBucket != ""

if enableExport && v2FlagIsSet {
Fatalf("Flags --influxdb.metrics.organization, --influxdb.metrics.token, --influxdb.metrics.bucket are only available for influxdb-v2")
} else if enableExportV2 && v1FlagIsSet {
Fatalf("Flags --influxdb.metrics.username, --influxdb.metrics.password are only available for influxdb-v1")
}
if enableExport && v2FlagIsSet {
Fatalf("Flags --influxdb.metrics.organization, --influxdb.metrics.token, --influxdb.metrics.bucket are only available for influxdb-v2")
} else if enableExportV2 && v1FlagIsSet {
Fatalf("Flags --influxdb.metrics.username, --influxdb.metrics.password are only available for influxdb-v1")
}
}

var (
endpoint = ctx.String(MetricsInfluxDBEndpointFlag.Name)
database = ctx.String(MetricsInfluxDBDatabaseFlag.Name)
username = ctx.String(MetricsInfluxDBUsernameFlag.Name)
password = ctx.String(MetricsInfluxDBPasswordFlag.Name)

token = ctx.String(MetricsInfluxDBTokenFlag.Name)
bucket = ctx.String(MetricsInfluxDBBucketFlag.Name)
organization = ctx.String(MetricsInfluxDBOrganizationFlag.Name)
)
var (
endpoint = cfg.InfluxDBEndpoint
database = cfg.InfluxDBDatabase
username = cfg.InfluxDBUsername
password = cfg.InfluxDBPassword

token = cfg.InfluxDBToken
bucket = cfg.InfluxDBBucket
organization = cfg.InfluxDBOrganization
)

if enableExport {
tagsMap := SplitTagsFlag(ctx.String(MetricsInfluxDBTagsFlag.Name))
if enableExport {
tagsMap := SplitTagsFlag(cfg.InfluxDBTags)

log.Info("Enabling metrics export to InfluxDB")
log.Info("Enabling metrics export to InfluxDB")

go influxdb.InfluxDBWithTags(metrics.DefaultRegistry, 10*time.Second, endpoint, database, username, password, "geth.", tagsMap)
} else if enableExportV2 {
tagsMap := SplitTagsFlag(ctx.String(MetricsInfluxDBTagsFlag.Name))
go influxdb.InfluxDBWithTags(metrics.DefaultRegistry, 10*time.Second, endpoint, database, username, password, "geth.", tagsMap)
} else if enableExportV2 {
tagsMap := SplitTagsFlag(cfg.InfluxDBTags)

log.Info("Enabling metrics export to InfluxDB (v2)")
log.Info("Enabling metrics export to InfluxDB (v2)")

go influxdb.InfluxDBV2WithTags(metrics.DefaultRegistry, 10*time.Second, endpoint, token, bucket, organization, "geth.", tagsMap)
}
go influxdb.InfluxDBV2WithTags(metrics.DefaultRegistry, 10*time.Second, endpoint, token, bucket, organization, "geth.", tagsMap)
}

if ctx.IsSet(MetricsHTTPFlag.Name) {
address := net.JoinHostPort(ctx.String(MetricsHTTPFlag.Name), fmt.Sprintf("%d", ctx.Int(MetricsPortFlag.Name)))
log.Info("Enabling stand-alone metrics HTTP endpoint", "address", address)
exp.Setup(address)
} else if ctx.IsSet(MetricsPortFlag.Name) {
log.Warn(fmt.Sprintf("--%s specified without --%s, metrics server will not start.", MetricsPortFlag.Name, MetricsHTTPFlag.Name))
}
if cfg.HTTP != "" {
address := net.JoinHostPort(cfg.HTTP, fmt.Sprintf("%d", cfg.Port))
log.Info("Enabling stand-alone metrics HTTP endpoint", "address", address)
exp.Setup(address)
} else if cfg.HTTP == "" && cfg.Port != 0 {
log.Warn(fmt.Sprintf("--%s specified without --%s, metrics server will not start.", MetricsPortFlag.Name, MetricsHTTPFlag.Name))
}
}

Expand Down
72 changes: 51 additions & 21 deletions metrics/metrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,16 @@
package metrics

import (
"flag"
"io"
"os"
"runtime/metrics"
"runtime/pprof"
"strconv"
"strings"
"syscall"
"time"

"github.com/ethereum/go-ethereum/log"
"github.com/naoina/toml"
)

// Enabled is checked by the constructor functions for all of the
Expand All @@ -24,34 +25,63 @@ import (
// for less cluttered pprof profiles.
var Enabled = false

// enablerFlags is the CLI flag names to use to enable metrics collections.
var enablerFlags = []string{"metrics"}

// enablerEnvVars is the env var names to use to enable metrics collections.
var enablerEnvVars = []string{"GETH_METRICS"}

// init enables or disables the metrics system. Since we need this to run before
// any other code gets to create meters and timers, we'll actually do an ugly hack
// and peek into the command line args for the metrics flag.
func init() {
for _, enabler := range enablerEnvVars {
if val, found := syscall.Getenv(enabler); found && !Enabled {
if enable, _ := strconv.ParseBool(val); enable { // ignore error, flag parser will choke on it later
log.Info("Enabling metrics collection")
lightclient marked this conversation as resolved.
Show resolved Hide resolved
Enabled = true
}
if val, found := syscall.Getenv("GETH_METRICS"); found && !Enabled {
if enable, _ := strconv.ParseBool(val); enable { // ignore error, flag parser will choke on it later
Enabled = true
}
}
for _, arg := range os.Args {
flag := strings.TrimLeft(arg, "-")

for _, enabler := range enablerFlags {
if !Enabled && flag == enabler {
log.Info("Enabling metrics collection")
Enabled = true
}
fs := flag.NewFlagSet("", flag.ContinueOnError)
fs.SetOutput(io.Discard)
fs.Bool("metrics", false, "")
fs.String("config", "", "")

// The flag package will quit parsing immediately if it encounters a flag that
// was not declared to it ahead of time. We could be fancy and try to look
// through the args to find the ones we care about, but then we'd need to
// handle the various ways that flags can be defined -- which was the point of
// using the flags package in the first place! So instead, let's chop off the
// first element in the args list each time a parse fails. This way we will
// eventually parse every arg and get the ones we care about.
Comment on lines +47 to +49
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looks flaky...?

Let's assume metrics is enabled in toml config. Won't these two behave differently:

  1. geth --metrics=false --conf=conf.toml will correctly set Enabled=false.
  2. geth --conf=conf.toml --metrics=false will incorrectly set Enabled=true.

?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So in this case, the order of precedence is always flags first, config second. However, the way init() is written in metrics/metrics.go, Enabled is always set to true in case either a flag or config specifies it should be. This is also how it was previously.

So for both 1) and 2) Enabled will be set to true, but we will not actually serve the metrics because that occurs later in the geth command start up. That is where the precedence comes in.

How would you prefer to deal with this? I could change the parsing in init() so that when the flag metrics is set, it always uses that value, regardless of config value. Otherwise could detect a mismatch (enabled in config, disabled explicitly by flag) during client startup and fail there instead.

Otherwise, we can leave as-is. Explicitly setting --metrics=false while actually passing a config with metrics enabled is pretty unlikely and we will not serve the metrics as expected. It will just cause geth to record the metrics internally.

I think my pref is to just fail when metrics are enabled in config, but explicitly disabled by the flag. It is simple to deal with:

diff --git a/cmd/geth/config.go b/cmd/geth/config.go
index 5f252ebf1..200b24109 100644
--- a/cmd/geth/config.go
+++ b/cmd/geth/config.go
@@ -289,6 +289,10 @@ func dumpConfig(ctx *cli.Context) error {

 func applyMetricConfig(ctx *cli.Context, cfg *gethConfig) {
        if ctx.IsSet(utils.MetricsEnabledFlag.Name) {
+               enabled := ctx.Bool(utils.MetricsEnabledFlag.Name)
+               if cfg.Metrics.Enabled && !enabled {
+                       utils.Fatalf("Flag --metrics mismatch with provided config.")
+               }
                cfg.Metrics.Enabled = ctx.Bool(utils.MetricsEnabledFlag.Name)
        }
        if ctx.IsSet(utils.MetricsEnabledExpensiveFlag.Name) {

What do you think?

for i := range os.Args[1:] {
if err := fs.Parse(os.Args[i+1:]); err == nil {
break
}
}

// Now visit the flags we defined which are present in the args and see if we
// should enable metrics.
fs.Visit(func(f *flag.Flag) {
g, ok := f.Value.(flag.Getter)
if !ok {
// Don't expect this to fail, but skip over just in case it does.
return
}
switch f.Name {
case "metrics":
Enabled = g.Get().(bool)
case "config":
data, err := os.ReadFile(g.Get().(string))
if err != nil {
return
}
var cfg map[string]map[string]any
if err := toml.Unmarshal(data, &cfg); err != nil {
return
}
// A config file is definitely being used and was parsed correct. Let's
// try to peek inside and see if metrics are listed as enabled.
if m, ok := cfg["Metrics"]; ok {
if v, ok := m["Enabled"].(bool); ok && v {
Enabled = true
}
}
}
})
}

var threadCreateProfile = pprof.Lookup("threadcreate")
Expand Down