Distributor: Make key configurable when logging failures (#9659)

**What this PR does / why we need it**:
Make appending `insight=true` key-value pair to log failures
configurable.

**Which issue(s) this PR fixes**:
N/A

docs/sources/configuration/_index.md

@@ -489,6 +489,11 @@ write_failures_logging:
   # Default: 1KB.
   # CLI flag: -distributor.write-failures-logging.rate
   [rate: <int> | default = 1KB]
+
+  # Experimental and subject to change. Whether a insight=true key should be
+  # logged or not. Default: false.
+  # CLI flag: -distributor.write-failures-logging.add-insights-label
+  [add_insights_label: <boolean> | default = false]
 ```
 
 ### querier

pkg/distributor/writefailures/cfg.go

@@ -8,10 +8,14 @@ import (
 
 type Cfg struct {
 	LogRate flagext.ByteSize `yaml:"rate" category:"experimental"`
+
+	AddInsightsLabel bool `yaml:"add_insights_label" category:"experimental"`
 }
 
 // RegisterFlags registers distributor-related flags.
 func (cfg *Cfg) RegisterFlagsWithPrefix(prefix string, fs *flag.FlagSet) {
 	_ = cfg.LogRate.Set("1KB")
 	fs.Var(&cfg.LogRate, prefix+".rate", "Experimental and subject to change. Log volume allowed (per second). Default: 1KB.")
+
+	fs.BoolVar(&cfg.AddInsightsLabel, prefix+".add-insights-label", false, "Experimental and subject to change. Whether a insight=true key should be logged or not. Default: false.")
 }

pkg/distributor/writefailures/manager.go

@@ -17,7 +17,10 @@ type Manager struct {
 }
 
 func NewManager(logger log.Logger, cfg Cfg, tenants *runtime.TenantConfigs) *Manager {
-	logger = log.With(logger, "path", "write", "insight", "true")
+	logger = log.With(logger, "path", "write")
+	if cfg.AddInsightsLabel {
+		logger = log.With(logger, "insight", "true")
+	}
 
 	strat := newStrategy(cfg.LogRate.Val(), float64(cfg.LogRate.Val()))