Skip to content

Commit

Permalink
Add run-interval flag
Browse files Browse the repository at this point in the history
Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com>
  • Loading branch information
kakkoyun committed Mar 12, 2020
1 parent dddec62 commit 5610717
Show file tree
Hide file tree
Showing 4 changed files with 23 additions and 8 deletions.
7 changes: 6 additions & 1 deletion cmd/thanos/compact.go
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,9 @@ func registerCompact(m map[string]setupFunc, app *kingpin.Application) {
wait := cmd.Flag("wait", "Do not exit after all compactions have been processed and wait for new work.").
Short('w').Bool()

runInterval := cmd.Flag("run-interval", "Repeat interval to run compactions. Only works when --wait flag specified.").
Default("5m").Duration()

generateMissingIndexCacheFiles := cmd.Flag("index.generate-missing-cache-file", "If enabled, on startup compactor runs an on-off job that scans all the blocks to find all blocks with missing index cache file. It generates those if needed and upload.").
Hidden().Default("false").Bool()

Expand Down Expand Up @@ -151,6 +154,7 @@ func registerCompact(m map[string]setupFunc, app *kingpin.Application) {
*compactionConcurrency,
*dedupReplicaLabels,
selectorRelabelConf,
*runInterval,
)
}
}
Expand All @@ -176,6 +180,7 @@ func runCompact(
concurrency int,
dedupReplicaLabels []string,
selectorRelabelConf *extflag.PathOrContent,
runInterval time.Duration,
) error {
halted := promauto.With(reg).NewGauge(prometheus.GaugeOpts{
Name: "thanos_compactor_halted",
Expand Down Expand Up @@ -362,7 +367,7 @@ func runCompact(
}

// --wait=true is specified.
return runutil.Repeat(5*time.Minute, ctx.Done(), func() error {
return runutil.Repeat(runInterval, ctx.Done(), func() error {
err := compactMainFn()
if err == nil {
iterations.Inc()
Expand Down
2 changes: 2 additions & 0 deletions docs/components/compact.md
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,8 @@ Flags:
samples of this resolution forever
-w, --wait Do not exit after all compactions have been
processed and wait for new work.
--run-interval=5m Repeat interval to run compactions. Only works
when --wait flag specified.
--downsampling.disable Disables downsampling. This is not recommended as
querying long time ranges without non-downsampled
data is not efficient and useful e.g it is not
Expand Down
1 change: 1 addition & 0 deletions test/e2e/compact_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,7 @@ func TestCompact(t *testing.T) {
Insecure: true,
},
},
nil, // relabel configgst
dedupFlags...,
)

Expand Down
21 changes: 14 additions & 7 deletions test/e2e/e2ethanos/services.go
Original file line number Diff line number Diff line change
Expand Up @@ -314,7 +314,7 @@ func NewStoreGW(sharedDir string, name string, bucketConfig client.BucketConfig,
), nil
}

func NewCompactor(sharedDir string, name string, bucketConfig client.BucketConfig, extArgs ...string) (*Service, error) {
func NewCompactor(sharedDir string, name string, bucketConfig client.BucketConfig, relabelConfig []relabel.Config, extArgs ...string) (*Service, error) {
dir := filepath.Join(sharedDir, "data", "compact", name)
container := filepath.Join(e2e.ContainerSharedDir, "data", "compact", name)

Expand All @@ -327,16 +327,23 @@ func NewCompactor(sharedDir string, name string, bucketConfig client.BucketConfi
return nil, errors.Wrapf(err, "generate compact config file: %v", bucketConfig)
}

relabelConfigBytes, err := yaml.Marshal(relabelConfig)
if err != nil {
return nil, errors.Wrapf(err, "generate compact relabel file: %v", relabelConfig)
}

return NewService(
fmt.Sprintf("compact-%s", name),
DefaultImage(),
e2e.NewCommand("compact", append(e2e.BuildArgs(map[string]string{
"--debug.name": fmt.Sprintf("compact-%s", name),
"--log.level": logLevel,
"--data-dir": container,
"--objstore.config": string(bktConfigBytes),
"--http-address": ":80",
"--wait": "",
"--debug.name": fmt.Sprintf("compact-%s", name),
"--log.level": logLevel,
"--data-dir": container,
"--objstore.config": string(bktConfigBytes),
"--http-address": ":80",
"--selector.relabel-config": string(relabelConfigBytes),
"--run-interval": "1s",
"--wait": "",
}), extArgs...)...),
e2e.NewReadinessProbe(80, "/-/ready", 200),
80,
Expand Down

0 comments on commit 5610717

Please sign in to comment.