Skip to content

Commit

Permalink
[pkg/translator/prometheusremotewrite] Improve performance of convers…
Browse files Browse the repository at this point in the history
…ion to Prometheus remote write format (#24288)

**Description:** 

This PR improves the performance of the conversion from OpenTelemetry to
Prometheus remote write. It reduces latency and memory utilisation by up
to 70% in some cases, particularly those involving histograms, summaries
or series with many labels.

<details>
<summary>Benchmark results</summary>

```

goos: darwin
goarch: arm64
pkg: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite
                                                                                                                                           │  before.txt  │              after.txt              │
                                                                                                                                           │    sec/op    │    sec/op     vs base               │
FromMetrics/resource_attribute_count:_0/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_0-10          2.150m ± 1%   1.987m ±  1%   -7.57% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_5-10          4.061m ± 1%   3.980m ±  2%   -1.98% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_10-10         5.570m ± 4%   5.443m ±  1%   -2.28% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_0-10         16.29m ± 2%   13.04m ±  1%  -19.96% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_5-10         18.14m ± 1%   14.98m ±  1%  -17.41% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_10-10        19.51m ± 1%   16.36m ±  1%  -16.11% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_0-10         10.568m ± 1%   6.405m ±  1%  -39.39% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_5-10          14.27m ± 1%   10.25m ±  0%  -28.18% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_10-10         16.70m ± 1%   12.75m ±  1%  -23.66% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_0-10         71.82m ± 0%   21.41m ±  1%  -70.19% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_5-10         76.46m ± 1%   25.36m ±  1%  -66.84% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_10-10        78.24m ± 0%   28.07m ±  0%  -64.12% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_0-10      12.180m ± 1%   7.912m ±  1%  -35.04% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_5-10       17.38m ± 0%   13.26m ±  0%  -23.71% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_10-10      21.14m ± 1%   17.07m ±  1%  -19.25% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_0-10      86.62m ± 3%   33.35m ±  2%  -61.50% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_5-10      91.72m ± 1%   38.95m ±  1%  -57.53% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_10-10     95.29m ± 1%   43.16m ± 36%  -54.70% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_0-10             1.338µ ± 1%   1.205µ ±  1%   -9.94% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_5-10             1.334µ ± 2%   1.225µ ±  2%   -8.17% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_10-10            1.328µ ± 1%   1.218µ ±  8%   -8.25% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_0-10            1.326µ ± 0%   1.240µ ±  4%   -6.49% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_5-10            1.326µ ± 1%   1.205µ ±  1%   -9.13% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_10-10           1.327µ ± 0%   1.215µ ±  2%   -8.48% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_0-10          2.144m ± 1%   2.008m ±  1%   -6.37% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_5-10          4.094m ± 1%   3.969m ±  1%   -3.06% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_10-10         5.595m ± 1%   5.465m ±  1%   -2.33% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_0-10         16.34m ± 1%   13.02m ±  0%  -20.28% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_5-10         18.27m ± 2%   14.99m ±  0%  -17.95% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_10-10        19.49m ± 1%   16.38m ±  0%  -15.95% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_0-10         10.624m ± 2%   6.389m ±  0%  -39.86% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_5-10          14.28m ± 1%   10.26m ±  0%  -28.17% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_10-10         16.80m ± 2%   12.75m ±  0%  -24.11% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_0-10         72.25m ± 1%   21.52m ±  1%  -70.22% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_5-10         76.34m ± 1%   25.29m ±  1%  -66.87% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_10-10        78.38m ± 1%   28.11m ±  1%  -64.13% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_0-10      12.452m ± 1%   7.930m ±  1%  -36.32% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_5-10       17.72m ± 3%   13.29m ±  2%  -25.00% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_10-10      21.57m ± 2%   17.13m ±  2%  -20.57% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_0-10      87.41m ± 1%   33.19m ±  1%  -62.03% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_5-10      92.93m ± 1%   38.91m ±  1%  -58.13% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_10-10     95.96m ± 0%   42.89m ±  1%  -55.30% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_0-10            16.58µ ± 1%   13.71µ ±  1%  -17.35% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_5-10            17.06µ ± 4%   13.69µ ±  0%  -19.78% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_10-10           16.96µ ± 2%   13.71µ ±  0%  -19.18% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_0-10           16.62µ ± 1%   13.71µ ±  0%  -17.49% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_5-10           16.64µ ± 2%   13.72µ ±  6%  -17.51% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_10-10          16.55µ ± 1%   13.73µ ±  0%  -17.04% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_0-10         2.220m ± 0%   2.148m ±  0%   -3.23% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_5-10         4.236m ± 1%   4.119m ±  1%   -2.76% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_10-10        5.705m ± 0%   5.605m ±  0%   -1.74% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_0-10        16.49m ± 0%   13.20m ±  0%  -19.96% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_5-10        18.43m ± 0%   15.13m ±  0%  -17.88% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_10-10       19.72m ± 1%   16.52m ±  1%  -16.25% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_0-10        11.039m ± 1%   6.469m ±  1%  -41.40% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_5-10         14.71m ± 1%   10.33m ±  0%  -29.80% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_10-10        17.18m ± 1%   12.82m ±  1%  -25.37% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_0-10        73.12m ± 1%   21.63m ±  1%  -70.41% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_5-10        77.44m ± 2%   25.49m ±  1%  -67.08% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_10-10       79.62m ± 1%   28.22m ±  0%  -64.56% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_0-10     12.979m ± 1%   8.137m ±  1%  -37.31% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_5-10      18.09m ± 1%   13.53m ±  1%  -25.23% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_10-10     21.86m ± 1%   17.35m ±  0%  -20.64% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_0-10     87.53m ± 2%   33.70m ±  1%  -61.50% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_5-10     93.54m ± 0%   39.12m ±  0%  -58.18% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_10-10    97.08m ± 0%   43.04m ±  1%  -55.66% (p=0.002 n=6)
geomean                                                                                                                                       4.704m        3.076m        -34.61%

                                                                                                                                           │  before.txt   │              after.txt              │
                                                                                                                                           │     B/op      │     B/op      vs base               │
FromMetrics/resource_attribute_count:_0/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_0-10          1.797Mi ± 0%   1.583Mi ± 0%  -11.90% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_5-10          4.583Mi ± 0%   4.369Mi ± 0%   -4.67% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_10-10         7.436Mi ± 0%   7.221Mi ± 0%   -2.88% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_0-10         17.26Mi ± 0%   10.15Mi ± 0%  -41.21% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_5-10         20.08Mi ± 0%   12.96Mi ± 0%  -35.46% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_10-10        22.97Mi ± 0%   15.84Mi ± 0%  -31.02% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_0-10          9.241Mi ± 0%   6.583Mi ± 0%  -28.76% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_5-10          12.73Mi ± 0%   10.07Mi ± 0%  -20.90% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_10-10         16.32Mi ± 0%   13.66Mi ± 0%  -16.32% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_0-10         78.03Mi ± 0%   23.99Mi ± 0%  -69.26% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_5-10         81.57Mi ± 0%   27.49Mi ± 0%  -66.30% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_10-10        85.26Mi ± 0%   31.10Mi ± 0%  -63.52% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_0-10      10.821Mi ± 0%   7.946Mi ± 0%  -26.56% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_5-10       17.12Mi ± 0%   14.24Mi ± 0%  -16.85% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_10-10      23.61Mi ± 0%   20.71Mi ± 0%  -12.27% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_0-10      95.50Mi ± 0%   34.07Mi ± 0%  -64.33% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_5-10     102.08Mi ± 0%   40.46Mi ± 0%  -60.36% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_10-10    108.76Mi ± 0%   47.06Mi ± 1%  -56.73% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_0-10             1.922Ki ± 0%   1.586Ki ± 0%  -17.48% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_5-10             1.922Ki ± 0%   1.586Ki ± 0%  -17.48% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_10-10            1.922Ki ± 0%   1.586Ki ± 0%  -17.48% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_0-10            1.922Ki ± 0%   1.586Ki ± 0%  -17.48% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_5-10            1.922Ki ± 0%   1.586Ki ± 0%  -17.48% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_10-10           1.922Ki ± 0%   1.586Ki ± 0%  -17.48% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_0-10          1.799Mi ± 0%   1.585Mi ± 0%  -11.91% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_5-10          4.585Mi ± 0%   4.370Mi ± 0%   -4.68% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_10-10         7.438Mi ± 0%   7.223Mi ± 0%   -2.89% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_0-10         17.26Mi ± 0%   10.15Mi ± 0%  -41.20% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_5-10         20.08Mi ± 0%   12.96Mi ± 0%  -35.45% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_10-10        22.97Mi ± 0%   15.84Mi ± 0%  -31.02% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_0-10          9.243Mi ± 0%   6.584Mi ± 0%  -28.76% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_5-10          12.73Mi ± 0%   10.07Mi ± 0%  -20.90% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_10-10         16.32Mi ± 0%   13.66Mi ± 0%  -16.33% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_0-10         78.03Mi ± 0%   23.99Mi ± 0%  -69.26% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_5-10         81.58Mi ± 0%   27.50Mi ± 0%  -66.30% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_10-10        85.27Mi ± 0%   31.10Mi ± 0%  -63.52% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_0-10      10.825Mi ± 0%   7.948Mi ± 0%  -26.58% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_5-10       17.13Mi ± 0%   14.24Mi ± 0%  -16.86% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_10-10      23.61Mi ± 0%   20.71Mi ± 0%  -12.28% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_0-10      95.48Mi ± 0%   34.07Mi ± 0%  -64.32% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_5-10     102.08Mi ± 0%   40.46Mi ± 0%  -60.36% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_10-10    108.78Mi ± 0%   47.05Mi ± 0%  -56.75% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_0-10            24.01Ki ± 0%   13.91Ki ± 0%  -42.06% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_5-10            24.02Ki ± 0%   13.91Ki ± 0%  -42.08% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_10-10           24.01Ki ± 0%   13.91Ki ± 0%  -42.07% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_0-10           24.02Ki ± 0%   13.91Ki ± 0%  -42.07% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_5-10           24.02Ki ± 0%   13.91Ki ± 0%  -42.07% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_10-10          24.02Ki ± 0%   13.91Ki ± 0%  -42.08% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_0-10         1.821Mi ± 0%   1.597Mi ± 0%  -12.29% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_5-10         4.607Mi ± 0%   4.383Mi ± 0%   -4.86% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_10-10        7.460Mi ± 0%   7.236Mi ± 0%   -3.01% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_0-10        17.28Mi ± 0%   10.16Mi ± 0%  -41.21% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_5-10        20.11Mi ± 0%   12.98Mi ± 0%  -35.46% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_10-10       22.99Mi ± 0%   15.86Mi ± 0%  -31.02% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_0-10         9.264Mi ± 0%   6.596Mi ± 0%  -28.80% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_5-10         12.75Mi ± 0%   10.08Mi ± 0%  -20.94% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_10-10        16.35Mi ± 0%   13.67Mi ± 0%  -16.37% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_0-10        78.05Mi ± 0%   24.00Mi ± 0%  -69.25% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_5-10        81.63Mi ± 0%   27.51Mi ± 0%  -66.30% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_10-10       85.29Mi ± 0%   31.11Mi ± 0%  -63.52% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_0-10     10.847Mi ± 0%   7.960Mi ± 0%  -26.61% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_5-10      17.15Mi ± 0%   14.25Mi ± 0%  -16.89% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_10-10     23.64Mi ± 0%   20.73Mi ± 0%  -12.31% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_0-10     95.52Mi ± 0%   34.08Mi ± 0%  -64.32% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_5-10    102.10Mi ± 0%   40.48Mi ± 0%  -60.36% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_10-10   108.78Mi ± 0%   47.07Mi ± 0%  -56.73% (p=0.002 n=6)
geomean                                                                                                                                       5.153Mi        3.202Mi       -37.87%

                                                                                                                                           │  before.txt  │             after.txt              │
                                                                                                                                           │  allocs/op   │  allocs/op   vs base               │
FromMetrics/resource_attribute_count:_0/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_0-10          29.14k ± 0%   22.13k ± 0%  -24.04% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_5-10          64.26k ± 0%   57.25k ± 0%  -10.90% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_10-10         95.38k ± 0%   88.37k ± 0%   -7.34% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_0-10         77.59k ± 0%   62.56k ± 0%  -19.37% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_5-10        113.24k ± 0%   98.11k ± 0%  -13.36% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_10-10        144.6k ± 0%   129.5k ± 0%  -10.45% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_0-10         131.48k ± 0%   73.39k ± 0%  -44.18% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_5-10          170.7k ± 0%   112.6k ± 0%  -34.04% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_10-10         202.8k ± 0%   144.7k ± 0%  -28.65% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_0-10        340.88k ± 0%   94.18k ± 0%  -72.37% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_5-10         380.5k ± 0%   133.7k ± 0%  -64.86% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_10-10        413.6k ± 0%   166.1k ± 0%  -59.84% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_0-10      160.89k ± 0%   95.69k ± 0%  -40.53% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_5-10       235.5k ± 0%   170.2k ± 0%  -27.72% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_10-10      299.1k ± 0%   233.8k ± 0%  -21.85% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_0-10      430.8k ± 0%   161.2k ± 0%  -62.58% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_5-10      508.9k ± 0%   237.2k ± 0%  -53.38% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_10-10     573.4k ± 0%   302.3k ± 2%  -47.28% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_0-10              20.00 ± 0%    15.00 ± 0%  -25.00% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_5-10              20.00 ± 0%    15.00 ± 0%  -25.00% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_10-10             20.00 ± 0%    15.00 ± 0%  -25.00% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_0-10             20.00 ± 0%    15.00 ± 0%  -25.00% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_5-10             20.00 ± 0%    15.00 ± 0%  -25.00% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_10-10            20.00 ± 0%    15.00 ± 0%  -25.00% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_0-10          29.16k ± 0%   22.15k ± 0%  -24.04% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_5-10          64.28k ± 0%   57.26k ± 0%  -10.91% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_10-10         95.40k ± 0%   88.39k ± 0%   -7.35% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_0-10         77.64k ± 0%   62.60k ± 0%  -19.38% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_5-10        113.25k ± 0%   98.12k ± 0%  -13.36% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_10-10        144.7k ± 0%   129.5k ± 0%  -10.46% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_0-10         131.50k ± 0%   73.40k ± 0%  -44.18% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_5-10          170.7k ± 0%   112.6k ± 0%  -34.03% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_10-10         202.9k ± 0%   144.7k ± 0%  -28.65% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_0-10        340.90k ± 0%   94.19k ± 0%  -72.37% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_5-10         380.6k ± 0%   133.7k ± 0%  -64.86% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_10-10        413.6k ± 0%   166.1k ± 0%  -59.84% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_0-10      160.98k ± 0%   95.70k ± 0%  -40.55% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_5-10       235.6k ± 0%   170.2k ± 0%  -27.73% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_10-10      299.2k ± 0%   233.8k ± 0%  -21.85% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_0-10      430.8k ± 0%   161.2k ± 0%  -62.57% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_5-10      508.9k ± 0%   237.3k ± 0%  -53.38% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_10-10     573.4k ± 0%   302.1k ± 0%  -47.32% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_0-10             73.00 ± 0%    62.00 ± 0%  -15.07% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_5-10             73.00 ± 0%    62.00 ± 0%  -15.07% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_10-10            73.00 ± 0%    62.00 ± 0%  -15.07% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_0-10            73.00 ± 0%    62.00 ± 0%  -15.07% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_5-10            73.00 ± 0%    62.00 ± 0%  -15.07% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_10-10           73.00 ± 0%    62.00 ± 0%  -15.07% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_0-10         29.21k ± 0%   22.20k ± 0%  -24.01% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_5-10         64.34k ± 0%   57.32k ± 0%  -10.91% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_10-10        95.46k ± 0%   88.44k ± 0%   -7.35% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_0-10        77.70k ± 0%   62.65k ± 0%  -19.37% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_5-10       113.36k ± 0%   98.20k ± 0%  -13.37% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_10-10       144.7k ± 0%   129.6k ± 0%  -10.43% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_0-10        131.55k ± 0%   73.45k ± 0%  -44.17% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_5-10         170.7k ± 0%   112.6k ± 0%  -34.03% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_10-10        202.9k ± 0%   144.8k ± 0%  -28.65% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_0-10       340.97k ± 0%   94.24k ± 0%  -72.36% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_5-10        381.2k ± 0%   133.8k ± 0%  -64.90% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_10-10       413.7k ± 0%   166.1k ± 0%  -59.84% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_0-10     161.05k ± 0%   95.76k ± 0%  -40.54% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_5-10      235.7k ± 0%   170.3k ± 0%  -27.72% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_10-10     299.3k ± 0%   233.9k ± 0%  -21.85% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_0-10     430.9k ± 0%   161.3k ± 0%  -62.57% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_5-10     509.0k ± 0%   237.3k ± 0%  -53.37% (p=0.002 n=6)
FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_10-10    573.5k ± 0%   302.4k ± 0%  -47.27% (p=0.002 n=6)
geomean                                                                                                                                       39.93k        25.46k       -36.23%

```

</details>

**Link to tracking Issue:** (none)

**Testing:** I've added a benchmark to verify the improvement, and run
the existing tests to ensure these changes have not introduced any
regressions.

**Documentation:** (none required)
  • Loading branch information
charleskorn authored Jul 28, 2023
1 parent 158ee0e commit 1c292e8
Show file tree
Hide file tree
Showing 3 changed files with 216 additions and 49 deletions.
20 changes: 20 additions & 0 deletions .chloggen/prometheus-remote-write-performance.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# Use this changelog template to create an entry for release notes.
# If your change doesn't affect end users, such as a test fix or a tooling change,
# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.

# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
change_type: enhancement

# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
component: prometheusremotewrite

# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
note: improve the latency and memory utilisation of the conversion from OpenTelemetry to Prometheus remote write

# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
issues: [24288]

# (Optional) One or more lines of additional information to render under the primary note.
# These lines will be padded with 2 spaces and then inserted directly into the document.
# Use pipe (|) for multiline entries.
subtext:
122 changes: 73 additions & 49 deletions pkg/translator/prometheusremotewrite/helper.go
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,14 @@ func addExemplar(tsMap map[string]*prompb.TimeSeries, bucketBounds []bucketBound
// the label slice should not contain duplicate label names; this method sorts the slice by label name before creating
// the signature.
func timeSeriesSignature(datatype string, labels *[]prompb.Label) string {
length := len(datatype)

for _, lb := range *labels {
length += 2 + len(lb.GetName()) + len(lb.GetValue())
}

b := strings.Builder{}
b.Grow(length)
b.WriteString(datatype)

sort.Sort(ByLabelName(*labels))
Expand All @@ -149,8 +156,22 @@ func timeSeriesSignature(datatype string, labels *[]prompb.Label) string {
// Unpaired string value is ignored. String pairs overwrites OTLP labels if collision happens, and the overwrite is
// logged. Resultant label names are sanitized.
func createAttributes(resource pcommon.Resource, attributes pcommon.Map, externalLabels map[string]string, extras ...string) []prompb.Label {
serviceName, haveServiceName := resource.Attributes().Get(conventions.AttributeServiceName)
instance, haveInstanceID := resource.Attributes().Get(conventions.AttributeServiceInstanceID)

// Calculate the maximum possible number of labels we could return so we can preallocate l
maxLabelCount := attributes.Len() + len(externalLabels) + len(extras)/2

if haveServiceName {
maxLabelCount++
}

if haveInstanceID {
maxLabelCount++
}

// map ensures no duplicate label name
l := map[string]prompb.Label{}
l := make(map[string]string, maxLabelCount)

// Ensure attributes are sorted by key for consistent merging of keys which
// collide when sanitized.
Expand All @@ -164,44 +185,31 @@ func createAttributes(resource pcommon.Resource, attributes pcommon.Map, externa
for _, label := range labels {
var finalKey = prometheustranslator.NormalizeLabel(label.Name)
if existingLabel, alreadyExists := l[finalKey]; alreadyExists {
existingLabel.Value = existingLabel.Value + ";" + label.Value
l[finalKey] = existingLabel
l[finalKey] = existingLabel + ";" + label.Value
} else {
l[finalKey] = prompb.Label{
Name: finalKey,
Value: label.Value,
}
l[finalKey] = label.Value
}
}

// Map service.name + service.namespace to job
if serviceName, ok := resource.Attributes().Get(conventions.AttributeServiceName); ok {
if haveServiceName {
val := serviceName.AsString()
if serviceNamespace, ok := resource.Attributes().Get(conventions.AttributeServiceNamespace); ok {
val = fmt.Sprintf("%s/%s", serviceNamespace.AsString(), val)
}
l[model.JobLabel] = prompb.Label{
Name: model.JobLabel,
Value: val,
}
l[model.JobLabel] = val
}
// Map service.instance.id to instance
if instance, ok := resource.Attributes().Get(conventions.AttributeServiceInstanceID); ok {
l[model.InstanceLabel] = prompb.Label{
Name: model.InstanceLabel,
Value: instance.AsString(),
}
if haveInstanceID {
l[model.InstanceLabel] = instance.AsString()
}
for key, value := range externalLabels {
// External labels have already been sanitized
if _, alreadyExists := l[key]; alreadyExists {
// Skip external labels if they are overridden by metric attributes
continue
}
l[key] = prompb.Label{
Name: key,
Value: value,
}
l[key] = value
}

for i := 0; i < len(extras); i += 2 {
Expand All @@ -217,15 +225,12 @@ func createAttributes(resource pcommon.Resource, attributes pcommon.Map, externa
if !(len(name) > 4 && name[:2] == "__" && name[len(name)-2:] == "__") {
name = prometheustranslator.NormalizeLabel(name)
}
l[name] = prompb.Label{
Name: name,
Value: extras[i+1],
}
l[name] = extras[i+1]
}

s := make([]prompb.Label, 0, len(l))
for _, lb := range l {
s = append(s, lb)
for k, v := range l {
s = append(s, prompb.Label{Name: k, Value: v})
}

return s
Expand Down Expand Up @@ -253,6 +258,21 @@ func addSingleHistogramDataPoint(pt pmetric.HistogramDataPoint, resource pcommon
timestamp := convertTimeStamp(pt.Timestamp())
// sum, count, and buckets of the histogram should append suffix to baseName
baseName := prometheustranslator.BuildCompliantName(metric, settings.Namespace, settings.AddMetricSuffixes)
baseLabels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels)

createLabels := func(nameSuffix string, extras ...string) []prompb.Label {
extraLabelCount := len(extras) / 2
labels := make([]prompb.Label, len(baseLabels), len(baseLabels)+extraLabelCount+1) // +1 for name
copy(labels, baseLabels)

for extrasIdx := 0; extrasIdx < extraLabelCount; extrasIdx++ {
labels = append(labels, prompb.Label{Name: extras[extrasIdx], Value: extras[extrasIdx+1]})
}

labels = append(labels, prompb.Label{Name: nameStr, Value: baseName + nameSuffix})

return labels
}

// If the sum is unset, it indicates the _sum metric point should be
// omitted
Expand All @@ -266,7 +286,7 @@ func addSingleHistogramDataPoint(pt pmetric.HistogramDataPoint, resource pcommon
sum.Value = math.Float64frombits(value.StaleNaN)
}

sumlabels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels, nameStr, baseName+sumStr)
sumlabels := createLabels(sumStr)
addSample(tsMap, sum, sumlabels, metric.Type().String())

}
Expand All @@ -280,7 +300,7 @@ func addSingleHistogramDataPoint(pt pmetric.HistogramDataPoint, resource pcommon
count.Value = math.Float64frombits(value.StaleNaN)
}

countlabels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels, nameStr, baseName+countStr)
countlabels := createLabels(countStr)
addSample(tsMap, count, countlabels, metric.Type().String())

// cumulative count for conversion to cumulative histogram
Expand All @@ -302,7 +322,7 @@ func addSingleHistogramDataPoint(pt pmetric.HistogramDataPoint, resource pcommon
bucket.Value = math.Float64frombits(value.StaleNaN)
}
boundStr := strconv.FormatFloat(bound, 'f', -1, 64)
labels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels, nameStr, baseName+bucketStr, leStr, boundStr)
labels := createLabels(bucketStr, leStr, boundStr)
sig := addSample(tsMap, bucket, labels, metric.Type().String())

bucketBounds = append(bucketBounds, bucketBoundsData{sig: sig, bound: bound})
Expand All @@ -316,7 +336,7 @@ func addSingleHistogramDataPoint(pt pmetric.HistogramDataPoint, resource pcommon
} else {
infBucket.Value = float64(pt.Count())
}
infLabels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels, nameStr, baseName+bucketStr, leStr, pInfStr)
infLabels := createLabels(bucketStr, leStr, pInfStr)
sig := addSample(tsMap, infBucket, infLabels, metric.Type().String())

bucketBounds = append(bucketBounds, bucketBoundsData{sig: sig, bound: math.Inf(1)})
Expand All @@ -325,14 +345,8 @@ func addSingleHistogramDataPoint(pt pmetric.HistogramDataPoint, resource pcommon
// add _created time series if needed
startTimestamp := pt.StartTimestamp()
if settings.ExportCreatedMetric && startTimestamp != 0 {
createdLabels := createAttributes(
resource,
pt.Attributes(),
settings.ExternalLabels,
nameStr,
baseName+createdSuffix,
)
addCreatedTimeSeriesIfNeeded(tsMap, createdLabels, startTimestamp, metric.Type().String())
labels := createLabels(createdSuffix)
addCreatedTimeSeriesIfNeeded(tsMap, labels, startTimestamp, metric.Type().String())
}
}

Expand Down Expand Up @@ -443,6 +457,22 @@ func addSingleSummaryDataPoint(pt pmetric.SummaryDataPoint, resource pcommon.Res
timestamp := convertTimeStamp(pt.Timestamp())
// sum and count of the summary should append suffix to baseName
baseName := prometheustranslator.BuildCompliantName(metric, settings.Namespace, settings.AddMetricSuffixes)
baseLabels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels)

createLabels := func(name string, extras ...string) []prompb.Label {
extraLabelCount := len(extras) / 2
labels := make([]prompb.Label, len(baseLabels), len(baseLabels)+extraLabelCount+1) // +1 for name
copy(labels, baseLabels)

for extrasIdx := 0; extrasIdx < extraLabelCount; extrasIdx++ {
labels = append(labels, prompb.Label{Name: extras[extrasIdx], Value: extras[extrasIdx+1]})
}

labels = append(labels, prompb.Label{Name: nameStr, Value: name})

return labels
}

// treat sum as a sample in an individual TimeSeries
sum := &prompb.Sample{
Value: pt.Sum(),
Expand All @@ -451,7 +481,7 @@ func addSingleSummaryDataPoint(pt pmetric.SummaryDataPoint, resource pcommon.Res
if pt.Flags().NoRecordedValue() {
sum.Value = math.Float64frombits(value.StaleNaN)
}
sumlabels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels, nameStr, baseName+sumStr)
sumlabels := createLabels(baseName + sumStr)
addSample(tsMap, sum, sumlabels, metric.Type().String())

// treat count as a sample in an individual TimeSeries
Expand All @@ -462,7 +492,7 @@ func addSingleSummaryDataPoint(pt pmetric.SummaryDataPoint, resource pcommon.Res
if pt.Flags().NoRecordedValue() {
count.Value = math.Float64frombits(value.StaleNaN)
}
countlabels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels, nameStr, baseName+countStr)
countlabels := createLabels(baseName + countStr)
addSample(tsMap, count, countlabels, metric.Type().String())

// process each percentile/quantile
Expand All @@ -476,20 +506,14 @@ func addSingleSummaryDataPoint(pt pmetric.SummaryDataPoint, resource pcommon.Res
quantile.Value = math.Float64frombits(value.StaleNaN)
}
percentileStr := strconv.FormatFloat(qt.Quantile(), 'f', -1, 64)
qtlabels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels, nameStr, baseName, quantileStr, percentileStr)
qtlabels := createLabels(baseName, quantileStr, percentileStr)
addSample(tsMap, quantile, qtlabels, metric.Type().String())
}

// add _created time series if needed
startTimestamp := pt.StartTimestamp()
if settings.ExportCreatedMetric && startTimestamp != 0 {
createdLabels := createAttributes(
resource,
pt.Attributes(),
settings.ExternalLabels,
nameStr,
baseName+createdSuffix,
)
createdLabels := createLabels(baseName + createdSuffix)
addCreatedTimeSeriesIfNeeded(tsMap, createdLabels, startTimestamp, metric.Type().String())
}
}
Expand Down
123 changes: 123 additions & 0 deletions pkg/translator/prometheusremotewrite/metrics_to_prw_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
// Copyright The OpenTelemetry Authors
// SPDX-License-Identifier: Apache-2.0

package prometheusremotewrite

import (
"fmt"
"testing"
"time"

"github.com/stretchr/testify/require"
"go.opentelemetry.io/collector/pdata/pcommon"
"go.opentelemetry.io/collector/pdata/pmetric"
"go.opentelemetry.io/collector/pdata/pmetric/pmetricotlp"
)

func BenchmarkFromMetrics(b *testing.B) {
for _, resourceAttributeCount := range []int{0, 5, 50} {
b.Run(fmt.Sprintf("resource attribute count: %v", resourceAttributeCount), func(b *testing.B) {
for _, histogramCount := range []int{0, 1000} {
b.Run(fmt.Sprintf("histogram count: %v", histogramCount), func(b *testing.B) {
nonHistogramCounts := []int{0, 1000}

if resourceAttributeCount == 0 && histogramCount == 0 {
// Don't bother running a scenario where we'll generate no series.
nonHistogramCounts = []int{1000}
}

for _, nonHistogramCount := range nonHistogramCounts {
b.Run(fmt.Sprintf("non-histogram count: %v", nonHistogramCount), func(b *testing.B) {
for _, labelsPerMetric := range []int{2, 20} {
b.Run(fmt.Sprintf("labels per metric: %v", labelsPerMetric), func(b *testing.B) {
for _, exemplarsPerSeries := range []int{0, 5, 10} {
b.Run(fmt.Sprintf("exemplars per series: %v", exemplarsPerSeries), func(b *testing.B) {
payload := createExportRequest(resourceAttributeCount, histogramCount, nonHistogramCount, labelsPerMetric, exemplarsPerSeries)

for i := 0; i < b.N; i++ {
_, err := FromMetrics(payload.Metrics(), Settings{})

if err != nil {
require.NoError(b, err)
}
}
})
}
})
}
})
}
})
}
})
}
}

func createExportRequest(resourceAttributeCount int, histogramCount int, nonHistogramCount int, labelsPerMetric int, exemplarsPerSeries int) pmetricotlp.ExportRequest {
request := pmetricotlp.NewExportRequest()

rm := request.Metrics().ResourceMetrics().AppendEmpty()
generateAttributes(rm.Resource().Attributes(), "resource", resourceAttributeCount)

metrics := rm.ScopeMetrics().AppendEmpty().Metrics()
ts := pcommon.NewTimestampFromTime(time.Now())

for i := 1; i <= histogramCount; i++ {
m := metrics.AppendEmpty()
m.SetEmptyHistogram()
m.SetName(fmt.Sprintf("histogram-%v", i))
m.Histogram().SetAggregationTemporality(pmetric.AggregationTemporalityCumulative)
h := m.Histogram().DataPoints().AppendEmpty()
h.SetTimestamp(ts)

// Set 50 samples, 10 each with values 0.5, 1, 2, 4, and 8
h.SetCount(50)
h.SetSum(155)
h.BucketCounts().FromRaw([]uint64{10, 10, 10, 10, 10, 0})
h.ExplicitBounds().FromRaw([]float64{.5, 1, 2, 4, 8, 16}) // Bucket boundaries include the upper limit (ie. each sample is on the upper limit of its bucket)

generateAttributes(h.Attributes(), "series", labelsPerMetric)
generateExemplars(h.Exemplars(), exemplarsPerSeries, ts)
}

for i := 1; i <= nonHistogramCount; i++ {
m := metrics.AppendEmpty()
m.SetEmptySum()
m.SetName(fmt.Sprintf("sum-%v", i))
m.Sum().SetAggregationTemporality(pmetric.AggregationTemporalityCumulative)
point := m.Sum().DataPoints().AppendEmpty()
point.SetTimestamp(ts)
point.SetDoubleValue(1.23)
generateAttributes(point.Attributes(), "series", labelsPerMetric)
generateExemplars(point.Exemplars(), exemplarsPerSeries, ts)
}

for i := 1; i <= nonHistogramCount; i++ {
m := metrics.AppendEmpty()
m.SetEmptyGauge()
m.SetName(fmt.Sprintf("gauge-%v", i))
point := m.Gauge().DataPoints().AppendEmpty()
point.SetTimestamp(ts)
point.SetDoubleValue(1.23)
generateAttributes(point.Attributes(), "series", labelsPerMetric)
generateExemplars(point.Exemplars(), exemplarsPerSeries, ts)
}

return request
}

func generateAttributes(m pcommon.Map, prefix string, count int) {
for i := 1; i <= count; i++ {
m.PutStr(fmt.Sprintf("%v-name-%v", prefix, i), fmt.Sprintf("value-%v", i))
}
}

func generateExemplars(exemplars pmetric.ExemplarSlice, count int, ts pcommon.Timestamp) {
for i := 1; i <= count; i++ {
e := exemplars.AppendEmpty()
e.SetTimestamp(ts)
e.SetDoubleValue(2.22)
e.SetSpanID(pcommon.SpanID{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08})
e.SetTraceID(pcommon.TraceID{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f})
}
}

0 comments on commit 1c292e8

Please sign in to comment.