[new feature] promtail: Add config reload endoint / signal to promtail #7247
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,9 +1,13 @@ | ||
| package promtail | ||
|
|
||
| import ( | ||
| "os" | ||
| "os/signal" | ||
| "sync" | ||
| "syscall" | ||
|
|
||
| "github.com/go-kit/log" | ||
| "github.com/go-kit/log/level" | ||
| "github.com/prometheus/client_golang/prometheus" | ||
|
|
||
| "github.com/grafana/loki/clients/pkg/logentry/stages" | ||
|
|
@@ -42,17 +46,24 @@ type Promtail struct { | |
| logger log.Logger | ||
| reg prometheus.Registerer | ||
|
|
||
| stopped bool | ||
| mtx sync.Mutex | ||
| configFile string | ||
| newConfig func() config.Config | ||
| metrics *client.Metrics | ||
| dryRun bool | ||
| } | ||
|
|
||
| // New makes a new Promtail. | ||
| func New(cfg config.Config, newConfig func() config.Config, metrics *client.Metrics, dryRun bool, opts ...Option) (*Promtail, error) { | ||
| // Initialize promtail with some defaults and allow the options to override | ||
| // them. | ||
|
|
||
| promtail := &Promtail{ | ||
| logger: util_log.Logger, | ||
| reg: prometheus.DefaultRegisterer, | ||
| metrics: metrics, | ||
| dryRun: dryRun, | ||
| } | ||
| for _, o := range opts { | ||
| // todo (callum) I don't understand why I needed to add this check | ||
|
|
@@ -61,37 +72,62 @@ func New(cfg config.Config, metrics *client.Metrics, dryRun bool, opts ...Option | |
| } | ||
| o(promtail) | ||
| } | ||
| err := promtail.reloadConfig(cfg) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| server, err := server.New(cfg.ServerConfig, promtail.logger, promtail.targetManagers, cfg.String()) | ||
| if err != nil { | ||
| return nil, err | ||
|
There was a problem hiding this comment. Please wrap these errors |
||
| } | ||
| promtail.server = server | ||
| promtail.newConfig = newConfig | ||
|
|
||
| return promtail, nil | ||
| } | ||
|
|
||
| func (p *Promtail) reloadConfig(cfg config.Config) error { | ||
|
There was a problem hiding this comment. why is this not a pointer to config.Config? It seems like we're mutating it within this function (ie. There was a problem hiding this comment. Thank you for such fast feedback,done. |
||
| level.Info(p.logger).Log("msg", "Loading configuration file") | ||
liguozhong marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| p.mtx.Lock() | ||
| defer p.mtx.Unlock() | ||
| if p.targetManagers != nil { | ||
| p.targetManagers.Stop() | ||
| } | ||
| if p.client != nil { | ||
| p.client.Stop() | ||
| } | ||
|
|
||
| cfg.Setup(p.logger) | ||
| if cfg.LimitsConfig.ReadlineRateEnabled { | ||
| stages.SetReadLineRateLimiter(cfg.LimitsConfig.ReadlineRate, cfg.LimitsConfig.ReadlineBurst, cfg.LimitsConfig.ReadlineRateDrop) | ||
| } | ||
| var err error | ||
| if p.dryRun { | ||
| p.client, err = client.NewLogger(p.metrics, cfg.Options.StreamLagLabels, p.logger, cfg.ClientConfigs...) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| cfg.PositionsConfig.ReadOnly = true | ||
| } else { | ||
| p.client, err = client.NewMulti(p.metrics, cfg.Options.StreamLagLabels, p.logger, cfg.LimitsConfig.MaxStreams, cfg.ClientConfigs...) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| } | ||
|
|
||
| tms, err := targets.NewTargetManagers(p, p.reg, p.logger, cfg.PositionsConfig, p.client, cfg.ScrapeConfig, &cfg.TargetConfig) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| p.targetManagers = tms | ||
|
|
||
| promServer := p.server | ||
| if promServer != nil { | ||
| promtailServer := promServer.(*server.PromtailServer) | ||
| promtailServer.ReloadTms(p.targetManagers) | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| // Run the promtail; will block until a signal is received. | ||
|
|
@@ -103,6 +139,7 @@ func (p *Promtail) Run() error { | |
| return nil | ||
| } | ||
| p.mtx.Unlock() // unlock before blocking | ||
| go p.watchConfig() | ||
| return p.server.Run() | ||
| } | ||
|
|
||
|
|
@@ -133,3 +170,33 @@ func (p *Promtail) Shutdown() { | |
| func (p *Promtail) ActiveTargets() map[string][]target.Target { | ||
| return p.targetManagers.ActiveTargets() | ||
| } | ||
|
|
||
| func (p *Promtail) watchConfig() { | ||
| // Reload handler. | ||
| // Make sure that sighup handler is registered with a redirect to the channel before the potentially | ||
| if p.newConfig == nil { | ||
| level.Warn(p.logger).Log("msg", "disable watchConfig") | ||
| return | ||
| } | ||
| level.Warn(p.logger).Log("msg", "enable watchConfig") | ||
|
There was a problem hiding this comment. These log messages should be more clear, please. |
||
| hup := make(chan os.Signal, 1) | ||
| signal.Notify(hup, syscall.SIGHUP) | ||
| promtailServer := p.server.(*server.PromtailServer) | ||
|
There was a problem hiding this comment. Again, please add a check if the type is not expected |
||
| for { | ||
| select { | ||
| case <-hup: | ||
|
There was a problem hiding this comment. sorry, not super familiar with these live reload patterns, can you explain the SIGHUP flow? There was a problem hiding this comment. I refer to the reload documentation of prometheus, we should also provide this reload way. https://prometheus.io/docs/prometheus/latest/configuration/configuration/ |
||
| cfg := p.newConfig() | ||
| if err := p.reloadConfig(cfg); err != nil { | ||
| level.Error(p.logger).Log("msg", "Error reloading config", "err", err) | ||
| } | ||
| case rc := <-promtailServer.Reload(): | ||
| cfg := p.newConfig() | ||
| if err := p.reloadConfig(cfg); err != nil { | ||
| level.Error(p.logger).Log("msg", "Error reloading config", "err", err) | ||
|
There was a problem hiding this comment. I would prefer if we centralised this logic rather than repeating it |
||
| rc <- err | ||
| } else { | ||
| rc <- nil | ||
| } | ||
| } | ||
| } | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
What is this for?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I have deleted it.
My mistake, this is a debug variable left over.