diff --git a/pkg/loki/common/common.go b/pkg/loki/common/common.go index 6f7fd1c768ed..e9d5a017c03a 100644 --- a/pkg/loki/common/common.go +++ b/pkg/loki/common/common.go @@ -6,6 +6,7 @@ import ( "github.com/grafana/dskit/flagext" "github.com/grafana/dskit/netutil" + "github.com/grafana/loki/pkg/storage/bucket" "github.com/grafana/loki/pkg/storage/chunk/client/alibaba" "github.com/grafana/loki/pkg/storage/chunk/client/aws" "github.com/grafana/loki/pkg/storage/chunk/client/azure" @@ -58,6 +59,8 @@ func (c *Config) RegisterFlags(f *flag.FlagSet) { c.Ring.RegisterFlagsWithPrefix("common.storage.", "collectors/", f) c.Ring.RegisterFlagsWithPrefix("common.storage.", "collectors/", throwaway) + f.BoolVar(&c.Storage.ThanosObjStore, "common.thanos.enable", false, "Enable the thanos.io/objstore to be the backend for object storage") + // instance related flags. c.InstanceInterfaceNames = netutil.PrivateNetworkInterfacesWithFallback([]string{"eth0", "en0"}, util_log.Logger) throwaway.StringVar(&c.InstanceAddr, "common.instance-addr", "", "Default advertised address to be used by Loki components.") @@ -78,6 +81,8 @@ type Storage struct { Hedging hedging.Config `yaml:"hedging"` COS ibmcloud.COSConfig `yaml:"cos"` CongestionControl congestion.Config `yaml:"congestion_control,omitempty"` + ThanosObjStore bool `yaml:"thanos_objstore"` + ObjStoreConf bucket.Config `yaml:"objstore_config"` } func (s *Storage) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { @@ -91,6 +96,8 @@ func (s *Storage) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { s.Hedging.RegisterFlagsWithPrefix(prefix, f) s.COS.RegisterFlagsWithPrefix(prefix, f) s.CongestionControl.RegisterFlagsWithPrefix(prefix, f) + + s.ObjStoreConf.RegisterFlagsWithPrefix(prefix+"thanos.", f) } type FilesystemConfig struct { diff --git a/pkg/loki/config_wrapper.go b/pkg/loki/config_wrapper.go index 41a87775a9ec..f236299d3b05 100644 --- a/pkg/loki/config_wrapper.go +++ b/pkg/loki/config_wrapper.go @@ -17,6 +17,7 @@ import ( "github.com/grafana/loki/pkg/util/cfg" lokiring "github.com/grafana/loki/pkg/util/ring" + "github.com/grafana/loki/pkg/ruler/rulestore" "github.com/grafana/loki/pkg/ruler/rulestore/local" loki_net "github.com/grafana/loki/pkg/util/net" ) @@ -564,6 +565,20 @@ func applyStorageConfig(cfg, defaults *ConfigWrapper) error { } } + if !reflect.DeepEqual(cfg.Common.Storage.ObjStoreConf, defaults.StorageConfig.ObjStoreConf) { + configsFound++ + + applyConfig = func(r *ConfigWrapper) { + r.Ruler.StoreConfig.ThanosObjStore = r.Common.Storage.ThanosObjStore + r.Ruler.StoreConfig.ObjStoreConf = rulestore.Config{ + Config: r.Common.Storage.ObjStoreConf, + } + r.StorageConfig.ThanosObjStore = r.Common.Storage.ThanosObjStore + r.StorageConfig.ObjStoreConf = r.Common.Storage.ObjStoreConf + r.StorageConfig.Hedging = r.Common.Storage.Hedging + } + } + if configsFound > 1 { return ErrTooManyStorageConfigs } diff --git a/pkg/storage/chunk/client/gcp/gcs_thanos_object_client.go b/pkg/storage/chunk/client/gcp/gcs_thanos_object_client.go index a2fe73050ce0..da72b865dd6f 100644 --- a/pkg/storage/chunk/client/gcp/gcs_thanos_object_client.go +++ b/pkg/storage/chunk/client/gcp/gcs_thanos_object_client.go @@ -23,12 +23,13 @@ type GCSThanosObjectClient struct { client objstore.Bucket } -func NewGCSThanosObjectClient(ctx context.Context, cfg bucket.Config, component string, logger log.Logger, hedgingCfg hedging.Config) (*GCSThanosObjectClient, error) { - return newGCSThanosObjectClient(ctx, cfg, component, logger) +func NewGCSThanosObjectClient(ctx context.Context, cfg bucket.Config, component string, logger log.Logger, hedgingCfg hedging.Config, reg prometheus.Registerer) (*GCSThanosObjectClient, error) { + //TODO(JoaoBraveCoding) Add Hedging client once we are able to configure HTTP on GCS provider + return newGCSThanosObjectClient(ctx, cfg, component, logger, reg) } -func newGCSThanosObjectClient(ctx context.Context, cfg bucket.Config, component string, logger log.Logger) (*GCSThanosObjectClient, error) { - bucket, err := bucket.NewClient(ctx, cfg, component, logger, prometheus.DefaultRegisterer) +func newGCSThanosObjectClient(ctx context.Context, cfg bucket.Config, component string, logger log.Logger, reg prometheus.Registerer) (*GCSThanosObjectClient, error) { + bucket, err := bucket.NewClient(ctx, cfg, component, logger, reg) if err != nil { return nil, err } diff --git a/pkg/storage/factory.go b/pkg/storage/factory.go index 7f736eff7d60..538ab9bc2c25 100644 --- a/pkg/storage/factory.go +++ b/pkg/storage/factory.go @@ -365,7 +365,8 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { cfg.Hedging.RegisterFlagsWithPrefix("store.", f) cfg.CongestionControl.RegisterFlagsWithPrefix("store.", f) - cfg.ObjStoreConf.RegisterFlags(f) + f.BoolVar(&cfg.ThanosObjStore, "thanos.enable", false, "Enable the thanos.io/objstore to be the backend for object storage") + cfg.ObjStoreConf.RegisterFlagsWithPrefix("thanos.", f) cfg.IndexQueriesCacheConfig.RegisterFlagsWithPrefix("store.index-cache-read.", "", f) f.DurationVar(&cfg.IndexCacheValidity, "store.index-cache-validity", 5*time.Minute, "Cache validity for active index entries. Should be no higher than -ingester.max-chunk-idle.") @@ -719,7 +720,7 @@ func internalNewObjectClient(name string, cfg Config, clientMetrics ClientMetric // Passing "gcs" as the component name as currently it's not // possible to get the component called this method // TODO(JoaoBraveCoding) update compoent when bigger refactor happens - return gcp.NewGCSThanosObjectClient(context.Background(), cfg.ObjStoreConf, "gcs", utilLog.Logger, cfg.Hedging) + return gcp.NewGCSThanosObjectClient(context.Background(), cfg.ObjStoreConf, "gcs", utilLog.Logger, cfg.Hedging, prometheus.WrapRegistererWithPrefix("loki_", prometheus.NewRegistry())) } return gcp.NewGCSObjectClient(context.Background(), gcsCfg, cfg.Hedging)