diff --git a/CHANGELOG.md b/CHANGELOG.md index 57f721d02bfc8..1eeb46a22417d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ ##### Enhancements +* [8786](https://github.com/grafana/loki/pull/8786) **DylanGuedes**: Ingester: add new /ingester/prepare_shutdown endpoint. * [8744](https://github.com/grafana/loki/pull/8744) **dannykopping**: Ruler: remote rule evaluation. * [8727](https://github.com/grafana/loki/pull/8727) **cstyan** **jeschkies**: Propagate per-request limit header to querier. * [8682](https://github.com/grafana/loki/pull/8682) **dannykopping**: Add fetched chunk size distribution metric `loki_chunk_fetcher_fetched_size_bytes`. diff --git a/docs/sources/api/_index.md b/docs/sources/api/_index.md index d57a47221f966..925c6e8216b7e 100644 --- a/docs/sources/api/_index.md +++ b/docs/sources/api/_index.md @@ -621,6 +621,17 @@ backing store. Mainly used for local testing. In microservices mode, the `/flush` endpoint is exposed by the ingester. +### Tell ingester to release all resources on next SIGTERM + +``` +POST /ingester/prepare_shutdown +``` + +`/ingester/prepare_shutdown` will prepare the ingester to release resources on the next SIGTERM signal, +where releasing resources means flushing data and unregistering from the ingester ring. +This endpoint supersedes any YAML configurations and isn't necessary if the ingester is already +configured to unregister from the ring or to flush on shutdown. + ## Flush in-memory chunks and shut down ``` @@ -1401,4 +1412,4 @@ $ curl -H "Content-Type: application/json" -XPOST -s "https://localhost:3100/api This is helpful for scaling down WAL-enabled ingesters where we want to ensure old WAL directories are not orphaned, but instead flushed to our chunk backend. -In microservices mode, the `/ingester/flush_shutdown` endpoint is exposed by the ingester. +In microservices mode, the `/ingester/flush_shutdown` endpoint is exposed by the ingester. \ No newline at end of file diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 91b47e14ee63b..8c5ea7dcef633 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -181,6 +181,7 @@ type Interface interface { // deprecated LegacyShutdownHandler(w http.ResponseWriter, r *http.Request) ShutdownHandler(w http.ResponseWriter, r *http.Request) + PrepareShutdown(w http.ResponseWriter, r *http.Request) } // Ingester builds chunks for incoming log streams. @@ -502,7 +503,8 @@ func (i *Ingester) running(ctx context.Context) error { return serviceError } -// Called after running exits, when Ingester transitions to Stopping state. +// stopping is called when Ingester transitions to Stopping state. +// // At this point, loop no longer runs, but flushers are still running. func (i *Ingester) stopping(_ error) error { i.stopIncomingRequests() @@ -523,8 +525,8 @@ func (i *Ingester) stopping(_ error) error { i.streamRateCalculator.Stop() - // In case the flag to terminate on shutdown is set we need to mark the - // ingester service as "failed", so Loki will shut down entirely. + // In case the flag to terminate on shutdown is set or this instance is marked to release its resources, + // we need to mark the ingester service as "failed", so Loki will shut down entirely. // The module manager logs the failure `modules.ErrStopProcess` in a special way. if i.terminateOnShutdown && errs.Err() == nil { return modules.ErrStopProcess @@ -568,6 +570,19 @@ func (i *Ingester) LegacyShutdownHandler(w http.ResponseWriter, r *http.Request) w.WriteHeader(http.StatusNoContent) } +// PrepareShutdown will handle the /ingester/prepare_shutdown endpoint. +// +// Internally, when triggered, this handler will configure the ingester service to release their resources whenever a SIGTERM is received. +// Releasing resources meaning flushing data, deleting tokens, and removing itself from the ring. +func (i *Ingester) PrepareShutdown(w http.ResponseWriter, r *http.Request) { + level.Info(util_log.Logger).Log("msg", "preparing full ingester shutdown, resources will be released on SIGTERM") + i.lifecycler.SetFlushOnShutdown(true) + i.lifecycler.SetUnregisterOnShutdown(true) + i.terminateOnShutdown = true + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) +} + // ShutdownHandler handles a graceful shutdown of the ingester service and // termination of the Loki process. func (i *Ingester) ShutdownHandler(w http.ResponseWriter, r *http.Request) { diff --git a/pkg/loki/loki.go b/pkg/loki/loki.go index 2c25a21b550d8..0aea511c7e80d 100644 --- a/pkg/loki/loki.go +++ b/pkg/loki/loki.go @@ -372,6 +372,8 @@ type Loki struct { deleteClientMetrics *deletion.DeleteRequestClientMetrics HTTPAuthMiddleware middleware.Interface + + ingesterRelease *atomic.Bool } // New makes a new Loki. diff --git a/pkg/loki/modules.go b/pkg/loki/modules.go index 8e6f6ac79edf9..d2ace128c90e5 100644 --- a/pkg/loki/modules.go +++ b/pkg/loki/modules.go @@ -471,6 +471,9 @@ func (t *Loki) initIngester() (_ services.Service, err error) { t.Server.HTTP.Methods("POST").Path("/ingester/flush_shutdown").Handler( httpMiddleware.Wrap(http.HandlerFunc(t.Ingester.LegacyShutdownHandler)), ) + t.Server.HTTP.Methods("POST").Path("/ingester/prepare_shutdown").Handler( + httpMiddleware.Wrap(http.HandlerFunc(t.Ingester.PrepareShutdown)), + ) t.Server.HTTP.Methods("POST").Path("/ingester/shutdown").Handler( httpMiddleware.Wrap(http.HandlerFunc(t.Ingester.ShutdownHandler)), )