From 02483faff501db1bc1b0f44f931d6ea8ae7ac9f9 Mon Sep 17 00:00:00 2001 From: Jarod Watkins Date: Wed, 16 Sep 2020 12:27:27 -0400 Subject: [PATCH 1/3] Store: If request ctx has an error we do not increment opsFailures counter Signed-off-by: Jarod Watkins --- pkg/objstore/objstore.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/objstore/objstore.go b/pkg/objstore/objstore.go index 21bebb9482..33912ce7a5 100644 --- a/pkg/objstore/objstore.go +++ b/pkg/objstore/objstore.go @@ -325,7 +325,7 @@ func (b *metricBucket) Attributes(ctx context.Context, name string) (ObjectAttri start := time.Now() attrs, err := b.bkt.Attributes(ctx, name) if err != nil { - if !b.isOpFailureExpected(err) { + if !b.isOpFailureExpected(err) && ctx.Err() == nil { b.opsFailures.WithLabelValues(op).Inc() } return attrs, err @@ -340,7 +340,7 @@ func (b *metricBucket) Get(ctx context.Context, name string) (io.ReadCloser, err rc, err := b.bkt.Get(ctx, name) if err != nil { - if !b.isOpFailureExpected(err) { + if !b.isOpFailureExpected(err) && ctx.Err() == nil { b.opsFailures.WithLabelValues(op).Inc() } return nil, err @@ -360,7 +360,7 @@ func (b *metricBucket) GetRange(ctx context.Context, name string, off, length in rc, err := b.bkt.GetRange(ctx, name, off, length) if err != nil { - if !b.isOpFailureExpected(err) { + if !b.isOpFailureExpected(err) && ctx.Err() == nil { b.opsFailures.WithLabelValues(op).Inc() } return nil, err @@ -381,7 +381,7 @@ func (b *metricBucket) Exists(ctx context.Context, name string) (bool, error) { start := time.Now() ok, err := b.bkt.Exists(ctx, name) if err != nil { - if !b.isOpFailureExpected(err) { + if !b.isOpFailureExpected(err) && ctx.Err() == nil { b.opsFailures.WithLabelValues(op).Inc() } return false, err @@ -396,7 +396,7 @@ func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) err start := time.Now() if err := b.bkt.Upload(ctx, name, r); err != nil { - if !b.isOpFailureExpected(err) { + if !b.isOpFailureExpected(err) && ctx.Err() == nil { b.opsFailures.WithLabelValues(op).Inc() } return err @@ -412,7 +412,7 @@ func (b *metricBucket) Delete(ctx context.Context, name string) error { start := time.Now() if err := b.bkt.Delete(ctx, name); err != nil { - if !b.isOpFailureExpected(err) { + if !b.isOpFailureExpected(err) && ctx.Err() == nil { b.opsFailures.WithLabelValues(op).Inc() } return err From d989d945941d6bfa9f964a483e7591bd288aaac7 Mon Sep 17 00:00:00 2001 From: Jarod Watkins Date: Fri, 18 Sep 2020 10:10:30 -0400 Subject: [PATCH 2/3] Using context.Canceled Signed-off-by: Jarod Watkins --- pkg/objstore/objstore.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/pkg/objstore/objstore.go b/pkg/objstore/objstore.go index 33912ce7a5..61db70b2f7 100644 --- a/pkg/objstore/objstore.go +++ b/pkg/objstore/objstore.go @@ -312,8 +312,10 @@ func (b *metricBucket) Iter(ctx context.Context, dir string, f func(name string) b.ops.WithLabelValues(op).Inc() err := b.bkt.Iter(ctx, dir, f) - if err != nil && !b.isOpFailureExpected(err) { - b.opsFailures.WithLabelValues(op).Inc() + if err != nil { + if !b.isOpFailureExpected(err) && ctx.Err() != context.Canceled { + b.opsFailures.WithLabelValues(op).Inc() + } } return err } @@ -325,7 +327,7 @@ func (b *metricBucket) Attributes(ctx context.Context, name string) (ObjectAttri start := time.Now() attrs, err := b.bkt.Attributes(ctx, name) if err != nil { - if !b.isOpFailureExpected(err) && ctx.Err() == nil { + if !b.isOpFailureExpected(err) && ctx.Err() != context.Canceled { b.opsFailures.WithLabelValues(op).Inc() } return attrs, err @@ -340,7 +342,7 @@ func (b *metricBucket) Get(ctx context.Context, name string) (io.ReadCloser, err rc, err := b.bkt.Get(ctx, name) if err != nil { - if !b.isOpFailureExpected(err) && ctx.Err() == nil { + if !b.isOpFailureExpected(err) && ctx.Err() != context.Canceled { b.opsFailures.WithLabelValues(op).Inc() } return nil, err @@ -360,7 +362,7 @@ func (b *metricBucket) GetRange(ctx context.Context, name string, off, length in rc, err := b.bkt.GetRange(ctx, name, off, length) if err != nil { - if !b.isOpFailureExpected(err) && ctx.Err() == nil { + if !b.isOpFailureExpected(err) && ctx.Err() != context.Canceled { b.opsFailures.WithLabelValues(op).Inc() } return nil, err @@ -381,7 +383,7 @@ func (b *metricBucket) Exists(ctx context.Context, name string) (bool, error) { start := time.Now() ok, err := b.bkt.Exists(ctx, name) if err != nil { - if !b.isOpFailureExpected(err) && ctx.Err() == nil { + if !b.isOpFailureExpected(err) && ctx.Err() != context.Canceled { b.opsFailures.WithLabelValues(op).Inc() } return false, err @@ -396,7 +398,7 @@ func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) err start := time.Now() if err := b.bkt.Upload(ctx, name, r); err != nil { - if !b.isOpFailureExpected(err) && ctx.Err() == nil { + if !b.isOpFailureExpected(err) && ctx.Err() != context.Canceled { b.opsFailures.WithLabelValues(op).Inc() } return err @@ -412,7 +414,7 @@ func (b *metricBucket) Delete(ctx context.Context, name string) error { start := time.Now() if err := b.bkt.Delete(ctx, name); err != nil { - if !b.isOpFailureExpected(err) && ctx.Err() == nil { + if !b.isOpFailureExpected(err) && ctx.Err() != context.Canceled { b.opsFailures.WithLabelValues(op).Inc() } return err From 7f7e248e487aa3286c5035439a07aa6faedfc1eb Mon Sep 17 00:00:00 2001 From: Jarod Watkins Date: Fri, 18 Sep 2020 10:35:27 -0400 Subject: [PATCH 3/3] Updating CHANGELOG Signed-off-by: Jarod Watkins --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a356c8dae1..977b920398 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#3136](https://github.com/thanos-io/thanos/pull/3136) Sidecar: Add metric `thanos_sidecar_reloader_config_apply_operations_total` and rename metric `thanos_sidecar_reloader_config_apply_errors_total` to `thanos_sidecar_reloader_config_apply_operations_failed_total`. - [#3154](https://github.com/thanos-io/thanos/pull/3154) Query: Add metric `thanos_query_gate_queries_max`. Remove metric `thanos_query_concurrent_selects_gate_queries_in_flight`. - [#3154](https://github.com/thanos-io/thanos/pull/3154) Store: Rename metric `thanos_bucket_store_queries_concurrent_max` to `thanos_bucket_store_series_gate_queries_max`. +- [#3179](https://github.com/thanos-io/thanos/pull/3179) Store: context.Canceled will not increase `thanos_objstore_bucket_operation_failures_total`. ## [v0.15.0](https://github.com/thanos-io/thanos/releases) - 2020.09.07