From 1c292e882524a225f4c907d3afdd5a9d6e22d9b3 Mon Sep 17 00:00:00 2001 From: Charles Korn Date: Fri, 28 Jul 2023 23:47:47 +1000 Subject: [PATCH] [pkg/translator/prometheusremotewrite] Improve performance of conversion to Prometheus remote write format (#24288) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Description:** This PR improves the performance of the conversion from OpenTelemetry to Prometheus remote write. It reduces latency and memory utilisation by up to 70% in some cases, particularly those involving histograms, summaries or series with many labels.
Benchmark results ``` goos: darwin goarch: arm64 pkg: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite │ before.txt │ after.txt │ │ sec/op │ sec/op vs base │ FromMetrics/resource_attribute_count:_0/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_0-10 2.150m ± 1% 1.987m ± 1% -7.57% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_5-10 4.061m ± 1% 3.980m ± 2% -1.98% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_10-10 5.570m ± 4% 5.443m ± 1% -2.28% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_0-10 16.29m ± 2% 13.04m ± 1% -19.96% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_5-10 18.14m ± 1% 14.98m ± 1% -17.41% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_10-10 19.51m ± 1% 16.36m ± 1% -16.11% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_0-10 10.568m ± 1% 6.405m ± 1% -39.39% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_5-10 14.27m ± 1% 10.25m ± 0% -28.18% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_10-10 16.70m ± 1% 12.75m ± 1% -23.66% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_0-10 71.82m ± 0% 21.41m ± 1% -70.19% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_5-10 76.46m ± 1% 25.36m ± 1% -66.84% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_10-10 78.24m ± 0% 28.07m ± 0% -64.12% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_0-10 12.180m ± 1% 7.912m ± 1% -35.04% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_5-10 17.38m ± 0% 13.26m ± 0% -23.71% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_10-10 21.14m ± 1% 17.07m ± 1% -19.25% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_0-10 86.62m ± 3% 33.35m ± 2% -61.50% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_5-10 91.72m ± 1% 38.95m ± 1% -57.53% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_10-10 95.29m ± 1% 43.16m ± 36% -54.70% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_0-10 1.338µ ± 1% 1.205µ ± 1% -9.94% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_5-10 1.334µ ± 2% 1.225µ ± 2% -8.17% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_10-10 1.328µ ± 1% 1.218µ ± 8% -8.25% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_0-10 1.326µ ± 0% 1.240µ ± 4% -6.49% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_5-10 1.326µ ± 1% 1.205µ ± 1% -9.13% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_10-10 1.327µ ± 0% 1.215µ ± 2% -8.48% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_0-10 2.144m ± 1% 2.008m ± 1% -6.37% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_5-10 4.094m ± 1% 3.969m ± 1% -3.06% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_10-10 5.595m ± 1% 5.465m ± 1% -2.33% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_0-10 16.34m ± 1% 13.02m ± 0% -20.28% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_5-10 18.27m ± 2% 14.99m ± 0% -17.95% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_10-10 19.49m ± 1% 16.38m ± 0% -15.95% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_0-10 10.624m ± 2% 6.389m ± 0% -39.86% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_5-10 14.28m ± 1% 10.26m ± 0% -28.17% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_10-10 16.80m ± 2% 12.75m ± 0% -24.11% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_0-10 72.25m ± 1% 21.52m ± 1% -70.22% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_5-10 76.34m ± 1% 25.29m ± 1% -66.87% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_10-10 78.38m ± 1% 28.11m ± 1% -64.13% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_0-10 12.452m ± 1% 7.930m ± 1% -36.32% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_5-10 17.72m ± 3% 13.29m ± 2% -25.00% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_10-10 21.57m ± 2% 17.13m ± 2% -20.57% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_0-10 87.41m ± 1% 33.19m ± 1% -62.03% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_5-10 92.93m ± 1% 38.91m ± 1% -58.13% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_10-10 95.96m ± 0% 42.89m ± 1% -55.30% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_0-10 16.58µ ± 1% 13.71µ ± 1% -17.35% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_5-10 17.06µ ± 4% 13.69µ ± 0% -19.78% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_10-10 16.96µ ± 2% 13.71µ ± 0% -19.18% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_0-10 16.62µ ± 1% 13.71µ ± 0% -17.49% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_5-10 16.64µ ± 2% 13.72µ ± 6% -17.51% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_10-10 16.55µ ± 1% 13.73µ ± 0% -17.04% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_0-10 2.220m ± 0% 2.148m ± 0% -3.23% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_5-10 4.236m ± 1% 4.119m ± 1% -2.76% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_10-10 5.705m ± 0% 5.605m ± 0% -1.74% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_0-10 16.49m ± 0% 13.20m ± 0% -19.96% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_5-10 18.43m ± 0% 15.13m ± 0% -17.88% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_10-10 19.72m ± 1% 16.52m ± 1% -16.25% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_0-10 11.039m ± 1% 6.469m ± 1% -41.40% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_5-10 14.71m ± 1% 10.33m ± 0% -29.80% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_10-10 17.18m ± 1% 12.82m ± 1% -25.37% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_0-10 73.12m ± 1% 21.63m ± 1% -70.41% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_5-10 77.44m ± 2% 25.49m ± 1% -67.08% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_10-10 79.62m ± 1% 28.22m ± 0% -64.56% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_0-10 12.979m ± 1% 8.137m ± 1% -37.31% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_5-10 18.09m ± 1% 13.53m ± 1% -25.23% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_10-10 21.86m ± 1% 17.35m ± 0% -20.64% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_0-10 87.53m ± 2% 33.70m ± 1% -61.50% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_5-10 93.54m ± 0% 39.12m ± 0% -58.18% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_10-10 97.08m ± 0% 43.04m ± 1% -55.66% (p=0.002 n=6) geomean 4.704m 3.076m -34.61% │ before.txt │ after.txt │ │ B/op │ B/op vs base │ FromMetrics/resource_attribute_count:_0/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_0-10 1.797Mi ± 0% 1.583Mi ± 0% -11.90% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_5-10 4.583Mi ± 0% 4.369Mi ± 0% -4.67% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_10-10 7.436Mi ± 0% 7.221Mi ± 0% -2.88% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_0-10 17.26Mi ± 0% 10.15Mi ± 0% -41.21% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_5-10 20.08Mi ± 0% 12.96Mi ± 0% -35.46% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_10-10 22.97Mi ± 0% 15.84Mi ± 0% -31.02% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_0-10 9.241Mi ± 0% 6.583Mi ± 0% -28.76% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_5-10 12.73Mi ± 0% 10.07Mi ± 0% -20.90% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_10-10 16.32Mi ± 0% 13.66Mi ± 0% -16.32% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_0-10 78.03Mi ± 0% 23.99Mi ± 0% -69.26% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_5-10 81.57Mi ± 0% 27.49Mi ± 0% -66.30% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_10-10 85.26Mi ± 0% 31.10Mi ± 0% -63.52% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_0-10 10.821Mi ± 0% 7.946Mi ± 0% -26.56% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_5-10 17.12Mi ± 0% 14.24Mi ± 0% -16.85% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_10-10 23.61Mi ± 0% 20.71Mi ± 0% -12.27% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_0-10 95.50Mi ± 0% 34.07Mi ± 0% -64.33% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_5-10 102.08Mi ± 0% 40.46Mi ± 0% -60.36% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_10-10 108.76Mi ± 0% 47.06Mi ± 1% -56.73% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_0-10 1.922Ki ± 0% 1.586Ki ± 0% -17.48% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_5-10 1.922Ki ± 0% 1.586Ki ± 0% -17.48% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_10-10 1.922Ki ± 0% 1.586Ki ± 0% -17.48% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_0-10 1.922Ki ± 0% 1.586Ki ± 0% -17.48% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_5-10 1.922Ki ± 0% 1.586Ki ± 0% -17.48% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_10-10 1.922Ki ± 0% 1.586Ki ± 0% -17.48% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_0-10 1.799Mi ± 0% 1.585Mi ± 0% -11.91% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_5-10 4.585Mi ± 0% 4.370Mi ± 0% -4.68% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_10-10 7.438Mi ± 0% 7.223Mi ± 0% -2.89% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_0-10 17.26Mi ± 0% 10.15Mi ± 0% -41.20% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_5-10 20.08Mi ± 0% 12.96Mi ± 0% -35.45% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_10-10 22.97Mi ± 0% 15.84Mi ± 0% -31.02% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_0-10 9.243Mi ± 0% 6.584Mi ± 0% -28.76% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_5-10 12.73Mi ± 0% 10.07Mi ± 0% -20.90% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_10-10 16.32Mi ± 0% 13.66Mi ± 0% -16.33% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_0-10 78.03Mi ± 0% 23.99Mi ± 0% -69.26% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_5-10 81.58Mi ± 0% 27.50Mi ± 0% -66.30% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_10-10 85.27Mi ± 0% 31.10Mi ± 0% -63.52% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_0-10 10.825Mi ± 0% 7.948Mi ± 0% -26.58% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_5-10 17.13Mi ± 0% 14.24Mi ± 0% -16.86% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_10-10 23.61Mi ± 0% 20.71Mi ± 0% -12.28% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_0-10 95.48Mi ± 0% 34.07Mi ± 0% -64.32% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_5-10 102.08Mi ± 0% 40.46Mi ± 0% -60.36% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_10-10 108.78Mi ± 0% 47.05Mi ± 0% -56.75% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_0-10 24.01Ki ± 0% 13.91Ki ± 0% -42.06% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_5-10 24.02Ki ± 0% 13.91Ki ± 0% -42.08% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_10-10 24.01Ki ± 0% 13.91Ki ± 0% -42.07% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_0-10 24.02Ki ± 0% 13.91Ki ± 0% -42.07% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_5-10 24.02Ki ± 0% 13.91Ki ± 0% -42.07% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_10-10 24.02Ki ± 0% 13.91Ki ± 0% -42.08% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_0-10 1.821Mi ± 0% 1.597Mi ± 0% -12.29% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_5-10 4.607Mi ± 0% 4.383Mi ± 0% -4.86% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_10-10 7.460Mi ± 0% 7.236Mi ± 0% -3.01% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_0-10 17.28Mi ± 0% 10.16Mi ± 0% -41.21% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_5-10 20.11Mi ± 0% 12.98Mi ± 0% -35.46% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_10-10 22.99Mi ± 0% 15.86Mi ± 0% -31.02% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_0-10 9.264Mi ± 0% 6.596Mi ± 0% -28.80% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_5-10 12.75Mi ± 0% 10.08Mi ± 0% -20.94% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_10-10 16.35Mi ± 0% 13.67Mi ± 0% -16.37% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_0-10 78.05Mi ± 0% 24.00Mi ± 0% -69.25% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_5-10 81.63Mi ± 0% 27.51Mi ± 0% -66.30% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_10-10 85.29Mi ± 0% 31.11Mi ± 0% -63.52% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_0-10 10.847Mi ± 0% 7.960Mi ± 0% -26.61% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_5-10 17.15Mi ± 0% 14.25Mi ± 0% -16.89% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_10-10 23.64Mi ± 0% 20.73Mi ± 0% -12.31% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_0-10 95.52Mi ± 0% 34.08Mi ± 0% -64.32% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_5-10 102.10Mi ± 0% 40.48Mi ± 0% -60.36% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_10-10 108.78Mi ± 0% 47.07Mi ± 0% -56.73% (p=0.002 n=6) geomean 5.153Mi 3.202Mi -37.87% │ before.txt │ after.txt │ │ allocs/op │ allocs/op vs base │ FromMetrics/resource_attribute_count:_0/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_0-10 29.14k ± 0% 22.13k ± 0% -24.04% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_5-10 64.26k ± 0% 57.25k ± 0% -10.90% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_10-10 95.38k ± 0% 88.37k ± 0% -7.34% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_0-10 77.59k ± 0% 62.56k ± 0% -19.37% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_5-10 113.24k ± 0% 98.11k ± 0% -13.36% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_10-10 144.6k ± 0% 129.5k ± 0% -10.45% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_0-10 131.48k ± 0% 73.39k ± 0% -44.18% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_5-10 170.7k ± 0% 112.6k ± 0% -34.04% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_10-10 202.8k ± 0% 144.7k ± 0% -28.65% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_0-10 340.88k ± 0% 94.18k ± 0% -72.37% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_5-10 380.5k ± 0% 133.7k ± 0% -64.86% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_10-10 413.6k ± 0% 166.1k ± 0% -59.84% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_0-10 160.89k ± 0% 95.69k ± 0% -40.53% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_5-10 235.5k ± 0% 170.2k ± 0% -27.72% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_10-10 299.1k ± 0% 233.8k ± 0% -21.85% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_0-10 430.8k ± 0% 161.2k ± 0% -62.58% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_5-10 508.9k ± 0% 237.2k ± 0% -53.38% (p=0.002 n=6) FromMetrics/resource_attribute_count:_0/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_10-10 573.4k ± 0% 302.3k ± 2% -47.28% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_0-10 20.00 ± 0% 15.00 ± 0% -25.00% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_5-10 20.00 ± 0% 15.00 ± 0% -25.00% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_10-10 20.00 ± 0% 15.00 ± 0% -25.00% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_0-10 20.00 ± 0% 15.00 ± 0% -25.00% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_5-10 20.00 ± 0% 15.00 ± 0% -25.00% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_10-10 20.00 ± 0% 15.00 ± 0% -25.00% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_0-10 29.16k ± 0% 22.15k ± 0% -24.04% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_5-10 64.28k ± 0% 57.26k ± 0% -10.91% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_10-10 95.40k ± 0% 88.39k ± 0% -7.35% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_0-10 77.64k ± 0% 62.60k ± 0% -19.38% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_5-10 113.25k ± 0% 98.12k ± 0% -13.36% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_10-10 144.7k ± 0% 129.5k ± 0% -10.46% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_0-10 131.50k ± 0% 73.40k ± 0% -44.18% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_5-10 170.7k ± 0% 112.6k ± 0% -34.03% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_10-10 202.9k ± 0% 144.7k ± 0% -28.65% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_0-10 340.90k ± 0% 94.19k ± 0% -72.37% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_5-10 380.6k ± 0% 133.7k ± 0% -64.86% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_10-10 413.6k ± 0% 166.1k ± 0% -59.84% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_0-10 160.98k ± 0% 95.70k ± 0% -40.55% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_5-10 235.6k ± 0% 170.2k ± 0% -27.73% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_10-10 299.2k ± 0% 233.8k ± 0% -21.85% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_0-10 430.8k ± 0% 161.2k ± 0% -62.57% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_5-10 508.9k ± 0% 237.3k ± 0% -53.38% (p=0.002 n=6) FromMetrics/resource_attribute_count:_5/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_10-10 573.4k ± 0% 302.1k ± 0% -47.32% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_0-10 73.00 ± 0% 62.00 ± 0% -15.07% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_5-10 73.00 ± 0% 62.00 ± 0% -15.07% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_10-10 73.00 ± 0% 62.00 ± 0% -15.07% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_0-10 73.00 ± 0% 62.00 ± 0% -15.07% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_5-10 73.00 ± 0% 62.00 ± 0% -15.07% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_10-10 73.00 ± 0% 62.00 ± 0% -15.07% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_0-10 29.21k ± 0% 22.20k ± 0% -24.01% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_5-10 64.34k ± 0% 57.32k ± 0% -10.91% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_10-10 95.46k ± 0% 88.44k ± 0% -7.35% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_0-10 77.70k ± 0% 62.65k ± 0% -19.37% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_5-10 113.36k ± 0% 98.20k ± 0% -13.37% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_0/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_10-10 144.7k ± 0% 129.6k ± 0% -10.43% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_0-10 131.55k ± 0% 73.45k ± 0% -44.17% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_5-10 170.7k ± 0% 112.6k ± 0% -34.03% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_2/exemplars_per_series:_10-10 202.9k ± 0% 144.8k ± 0% -28.65% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_0-10 340.97k ± 0% 94.24k ± 0% -72.36% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_5-10 381.2k ± 0% 133.8k ± 0% -64.90% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_0/labels_per_metric:_20/exemplars_per_series:_10-10 413.7k ± 0% 166.1k ± 0% -59.84% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_0-10 161.05k ± 0% 95.76k ± 0% -40.54% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_5-10 235.7k ± 0% 170.3k ± 0% -27.72% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_2/exemplars_per_series:_10-10 299.3k ± 0% 233.9k ± 0% -21.85% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_0-10 430.9k ± 0% 161.3k ± 0% -62.57% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_5-10 509.0k ± 0% 237.3k ± 0% -53.37% (p=0.002 n=6) FromMetrics/resource_attribute_count:_50/histogram_count:_1000/non-histogram_count:_1000/labels_per_metric:_20/exemplars_per_series:_10-10 573.5k ± 0% 302.4k ± 0% -47.27% (p=0.002 n=6) geomean 39.93k 25.46k -36.23% ```
**Link to tracking Issue:** (none) **Testing:** I've added a benchmark to verify the improvement, and run the existing tests to ensure these changes have not introduced any regressions. **Documentation:** (none required) --- .../prometheus-remote-write-performance.yaml | 20 +++ .../prometheusremotewrite/helper.go | 122 ++++++++++------- .../metrics_to_prw_test.go | 123 ++++++++++++++++++ 3 files changed, 216 insertions(+), 49 deletions(-) create mode 100755 .chloggen/prometheus-remote-write-performance.yaml create mode 100644 pkg/translator/prometheusremotewrite/metrics_to_prw_test.go diff --git a/.chloggen/prometheus-remote-write-performance.yaml b/.chloggen/prometheus-remote-write-performance.yaml new file mode 100755 index 000000000000..d96c994cbeb5 --- /dev/null +++ b/.chloggen/prometheus-remote-write-performance.yaml @@ -0,0 +1,20 @@ +# Use this changelog template to create an entry for release notes. +# If your change doesn't affect end users, such as a test fix or a tooling change, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: prometheusremotewrite + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: improve the latency and memory utilisation of the conversion from OpenTelemetry to Prometheus remote write + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [24288] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: diff --git a/pkg/translator/prometheusremotewrite/helper.go b/pkg/translator/prometheusremotewrite/helper.go index f37f29c81d9b..058f72a1c8d3 100644 --- a/pkg/translator/prometheusremotewrite/helper.go +++ b/pkg/translator/prometheusremotewrite/helper.go @@ -130,7 +130,14 @@ func addExemplar(tsMap map[string]*prompb.TimeSeries, bucketBounds []bucketBound // the label slice should not contain duplicate label names; this method sorts the slice by label name before creating // the signature. func timeSeriesSignature(datatype string, labels *[]prompb.Label) string { + length := len(datatype) + + for _, lb := range *labels { + length += 2 + len(lb.GetName()) + len(lb.GetValue()) + } + b := strings.Builder{} + b.Grow(length) b.WriteString(datatype) sort.Sort(ByLabelName(*labels)) @@ -149,8 +156,22 @@ func timeSeriesSignature(datatype string, labels *[]prompb.Label) string { // Unpaired string value is ignored. String pairs overwrites OTLP labels if collision happens, and the overwrite is // logged. Resultant label names are sanitized. func createAttributes(resource pcommon.Resource, attributes pcommon.Map, externalLabels map[string]string, extras ...string) []prompb.Label { + serviceName, haveServiceName := resource.Attributes().Get(conventions.AttributeServiceName) + instance, haveInstanceID := resource.Attributes().Get(conventions.AttributeServiceInstanceID) + + // Calculate the maximum possible number of labels we could return so we can preallocate l + maxLabelCount := attributes.Len() + len(externalLabels) + len(extras)/2 + + if haveServiceName { + maxLabelCount++ + } + + if haveInstanceID { + maxLabelCount++ + } + // map ensures no duplicate label name - l := map[string]prompb.Label{} + l := make(map[string]string, maxLabelCount) // Ensure attributes are sorted by key for consistent merging of keys which // collide when sanitized. @@ -164,33 +185,23 @@ func createAttributes(resource pcommon.Resource, attributes pcommon.Map, externa for _, label := range labels { var finalKey = prometheustranslator.NormalizeLabel(label.Name) if existingLabel, alreadyExists := l[finalKey]; alreadyExists { - existingLabel.Value = existingLabel.Value + ";" + label.Value - l[finalKey] = existingLabel + l[finalKey] = existingLabel + ";" + label.Value } else { - l[finalKey] = prompb.Label{ - Name: finalKey, - Value: label.Value, - } + l[finalKey] = label.Value } } // Map service.name + service.namespace to job - if serviceName, ok := resource.Attributes().Get(conventions.AttributeServiceName); ok { + if haveServiceName { val := serviceName.AsString() if serviceNamespace, ok := resource.Attributes().Get(conventions.AttributeServiceNamespace); ok { val = fmt.Sprintf("%s/%s", serviceNamespace.AsString(), val) } - l[model.JobLabel] = prompb.Label{ - Name: model.JobLabel, - Value: val, - } + l[model.JobLabel] = val } // Map service.instance.id to instance - if instance, ok := resource.Attributes().Get(conventions.AttributeServiceInstanceID); ok { - l[model.InstanceLabel] = prompb.Label{ - Name: model.InstanceLabel, - Value: instance.AsString(), - } + if haveInstanceID { + l[model.InstanceLabel] = instance.AsString() } for key, value := range externalLabels { // External labels have already been sanitized @@ -198,10 +209,7 @@ func createAttributes(resource pcommon.Resource, attributes pcommon.Map, externa // Skip external labels if they are overridden by metric attributes continue } - l[key] = prompb.Label{ - Name: key, - Value: value, - } + l[key] = value } for i := 0; i < len(extras); i += 2 { @@ -217,15 +225,12 @@ func createAttributes(resource pcommon.Resource, attributes pcommon.Map, externa if !(len(name) > 4 && name[:2] == "__" && name[len(name)-2:] == "__") { name = prometheustranslator.NormalizeLabel(name) } - l[name] = prompb.Label{ - Name: name, - Value: extras[i+1], - } + l[name] = extras[i+1] } s := make([]prompb.Label, 0, len(l)) - for _, lb := range l { - s = append(s, lb) + for k, v := range l { + s = append(s, prompb.Label{Name: k, Value: v}) } return s @@ -253,6 +258,21 @@ func addSingleHistogramDataPoint(pt pmetric.HistogramDataPoint, resource pcommon timestamp := convertTimeStamp(pt.Timestamp()) // sum, count, and buckets of the histogram should append suffix to baseName baseName := prometheustranslator.BuildCompliantName(metric, settings.Namespace, settings.AddMetricSuffixes) + baseLabels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels) + + createLabels := func(nameSuffix string, extras ...string) []prompb.Label { + extraLabelCount := len(extras) / 2 + labels := make([]prompb.Label, len(baseLabels), len(baseLabels)+extraLabelCount+1) // +1 for name + copy(labels, baseLabels) + + for extrasIdx := 0; extrasIdx < extraLabelCount; extrasIdx++ { + labels = append(labels, prompb.Label{Name: extras[extrasIdx], Value: extras[extrasIdx+1]}) + } + + labels = append(labels, prompb.Label{Name: nameStr, Value: baseName + nameSuffix}) + + return labels + } // If the sum is unset, it indicates the _sum metric point should be // omitted @@ -266,7 +286,7 @@ func addSingleHistogramDataPoint(pt pmetric.HistogramDataPoint, resource pcommon sum.Value = math.Float64frombits(value.StaleNaN) } - sumlabels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels, nameStr, baseName+sumStr) + sumlabels := createLabels(sumStr) addSample(tsMap, sum, sumlabels, metric.Type().String()) } @@ -280,7 +300,7 @@ func addSingleHistogramDataPoint(pt pmetric.HistogramDataPoint, resource pcommon count.Value = math.Float64frombits(value.StaleNaN) } - countlabels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels, nameStr, baseName+countStr) + countlabels := createLabels(countStr) addSample(tsMap, count, countlabels, metric.Type().String()) // cumulative count for conversion to cumulative histogram @@ -302,7 +322,7 @@ func addSingleHistogramDataPoint(pt pmetric.HistogramDataPoint, resource pcommon bucket.Value = math.Float64frombits(value.StaleNaN) } boundStr := strconv.FormatFloat(bound, 'f', -1, 64) - labels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels, nameStr, baseName+bucketStr, leStr, boundStr) + labels := createLabels(bucketStr, leStr, boundStr) sig := addSample(tsMap, bucket, labels, metric.Type().String()) bucketBounds = append(bucketBounds, bucketBoundsData{sig: sig, bound: bound}) @@ -316,7 +336,7 @@ func addSingleHistogramDataPoint(pt pmetric.HistogramDataPoint, resource pcommon } else { infBucket.Value = float64(pt.Count()) } - infLabels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels, nameStr, baseName+bucketStr, leStr, pInfStr) + infLabels := createLabels(bucketStr, leStr, pInfStr) sig := addSample(tsMap, infBucket, infLabels, metric.Type().String()) bucketBounds = append(bucketBounds, bucketBoundsData{sig: sig, bound: math.Inf(1)}) @@ -325,14 +345,8 @@ func addSingleHistogramDataPoint(pt pmetric.HistogramDataPoint, resource pcommon // add _created time series if needed startTimestamp := pt.StartTimestamp() if settings.ExportCreatedMetric && startTimestamp != 0 { - createdLabels := createAttributes( - resource, - pt.Attributes(), - settings.ExternalLabels, - nameStr, - baseName+createdSuffix, - ) - addCreatedTimeSeriesIfNeeded(tsMap, createdLabels, startTimestamp, metric.Type().String()) + labels := createLabels(createdSuffix) + addCreatedTimeSeriesIfNeeded(tsMap, labels, startTimestamp, metric.Type().String()) } } @@ -443,6 +457,22 @@ func addSingleSummaryDataPoint(pt pmetric.SummaryDataPoint, resource pcommon.Res timestamp := convertTimeStamp(pt.Timestamp()) // sum and count of the summary should append suffix to baseName baseName := prometheustranslator.BuildCompliantName(metric, settings.Namespace, settings.AddMetricSuffixes) + baseLabels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels) + + createLabels := func(name string, extras ...string) []prompb.Label { + extraLabelCount := len(extras) / 2 + labels := make([]prompb.Label, len(baseLabels), len(baseLabels)+extraLabelCount+1) // +1 for name + copy(labels, baseLabels) + + for extrasIdx := 0; extrasIdx < extraLabelCount; extrasIdx++ { + labels = append(labels, prompb.Label{Name: extras[extrasIdx], Value: extras[extrasIdx+1]}) + } + + labels = append(labels, prompb.Label{Name: nameStr, Value: name}) + + return labels + } + // treat sum as a sample in an individual TimeSeries sum := &prompb.Sample{ Value: pt.Sum(), @@ -451,7 +481,7 @@ func addSingleSummaryDataPoint(pt pmetric.SummaryDataPoint, resource pcommon.Res if pt.Flags().NoRecordedValue() { sum.Value = math.Float64frombits(value.StaleNaN) } - sumlabels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels, nameStr, baseName+sumStr) + sumlabels := createLabels(baseName + sumStr) addSample(tsMap, sum, sumlabels, metric.Type().String()) // treat count as a sample in an individual TimeSeries @@ -462,7 +492,7 @@ func addSingleSummaryDataPoint(pt pmetric.SummaryDataPoint, resource pcommon.Res if pt.Flags().NoRecordedValue() { count.Value = math.Float64frombits(value.StaleNaN) } - countlabels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels, nameStr, baseName+countStr) + countlabels := createLabels(baseName + countStr) addSample(tsMap, count, countlabels, metric.Type().String()) // process each percentile/quantile @@ -476,20 +506,14 @@ func addSingleSummaryDataPoint(pt pmetric.SummaryDataPoint, resource pcommon.Res quantile.Value = math.Float64frombits(value.StaleNaN) } percentileStr := strconv.FormatFloat(qt.Quantile(), 'f', -1, 64) - qtlabels := createAttributes(resource, pt.Attributes(), settings.ExternalLabels, nameStr, baseName, quantileStr, percentileStr) + qtlabels := createLabels(baseName, quantileStr, percentileStr) addSample(tsMap, quantile, qtlabels, metric.Type().String()) } // add _created time series if needed startTimestamp := pt.StartTimestamp() if settings.ExportCreatedMetric && startTimestamp != 0 { - createdLabels := createAttributes( - resource, - pt.Attributes(), - settings.ExternalLabels, - nameStr, - baseName+createdSuffix, - ) + createdLabels := createLabels(baseName + createdSuffix) addCreatedTimeSeriesIfNeeded(tsMap, createdLabels, startTimestamp, metric.Type().String()) } } diff --git a/pkg/translator/prometheusremotewrite/metrics_to_prw_test.go b/pkg/translator/prometheusremotewrite/metrics_to_prw_test.go new file mode 100644 index 000000000000..8be6bdcd60a9 --- /dev/null +++ b/pkg/translator/prometheusremotewrite/metrics_to_prw_test.go @@ -0,0 +1,123 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package prometheusremotewrite + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/pmetric" + "go.opentelemetry.io/collector/pdata/pmetric/pmetricotlp" +) + +func BenchmarkFromMetrics(b *testing.B) { + for _, resourceAttributeCount := range []int{0, 5, 50} { + b.Run(fmt.Sprintf("resource attribute count: %v", resourceAttributeCount), func(b *testing.B) { + for _, histogramCount := range []int{0, 1000} { + b.Run(fmt.Sprintf("histogram count: %v", histogramCount), func(b *testing.B) { + nonHistogramCounts := []int{0, 1000} + + if resourceAttributeCount == 0 && histogramCount == 0 { + // Don't bother running a scenario where we'll generate no series. + nonHistogramCounts = []int{1000} + } + + for _, nonHistogramCount := range nonHistogramCounts { + b.Run(fmt.Sprintf("non-histogram count: %v", nonHistogramCount), func(b *testing.B) { + for _, labelsPerMetric := range []int{2, 20} { + b.Run(fmt.Sprintf("labels per metric: %v", labelsPerMetric), func(b *testing.B) { + for _, exemplarsPerSeries := range []int{0, 5, 10} { + b.Run(fmt.Sprintf("exemplars per series: %v", exemplarsPerSeries), func(b *testing.B) { + payload := createExportRequest(resourceAttributeCount, histogramCount, nonHistogramCount, labelsPerMetric, exemplarsPerSeries) + + for i := 0; i < b.N; i++ { + _, err := FromMetrics(payload.Metrics(), Settings{}) + + if err != nil { + require.NoError(b, err) + } + } + }) + } + }) + } + }) + } + }) + } + }) + } +} + +func createExportRequest(resourceAttributeCount int, histogramCount int, nonHistogramCount int, labelsPerMetric int, exemplarsPerSeries int) pmetricotlp.ExportRequest { + request := pmetricotlp.NewExportRequest() + + rm := request.Metrics().ResourceMetrics().AppendEmpty() + generateAttributes(rm.Resource().Attributes(), "resource", resourceAttributeCount) + + metrics := rm.ScopeMetrics().AppendEmpty().Metrics() + ts := pcommon.NewTimestampFromTime(time.Now()) + + for i := 1; i <= histogramCount; i++ { + m := metrics.AppendEmpty() + m.SetEmptyHistogram() + m.SetName(fmt.Sprintf("histogram-%v", i)) + m.Histogram().SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) + h := m.Histogram().DataPoints().AppendEmpty() + h.SetTimestamp(ts) + + // Set 50 samples, 10 each with values 0.5, 1, 2, 4, and 8 + h.SetCount(50) + h.SetSum(155) + h.BucketCounts().FromRaw([]uint64{10, 10, 10, 10, 10, 0}) + h.ExplicitBounds().FromRaw([]float64{.5, 1, 2, 4, 8, 16}) // Bucket boundaries include the upper limit (ie. each sample is on the upper limit of its bucket) + + generateAttributes(h.Attributes(), "series", labelsPerMetric) + generateExemplars(h.Exemplars(), exemplarsPerSeries, ts) + } + + for i := 1; i <= nonHistogramCount; i++ { + m := metrics.AppendEmpty() + m.SetEmptySum() + m.SetName(fmt.Sprintf("sum-%v", i)) + m.Sum().SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) + point := m.Sum().DataPoints().AppendEmpty() + point.SetTimestamp(ts) + point.SetDoubleValue(1.23) + generateAttributes(point.Attributes(), "series", labelsPerMetric) + generateExemplars(point.Exemplars(), exemplarsPerSeries, ts) + } + + for i := 1; i <= nonHistogramCount; i++ { + m := metrics.AppendEmpty() + m.SetEmptyGauge() + m.SetName(fmt.Sprintf("gauge-%v", i)) + point := m.Gauge().DataPoints().AppendEmpty() + point.SetTimestamp(ts) + point.SetDoubleValue(1.23) + generateAttributes(point.Attributes(), "series", labelsPerMetric) + generateExemplars(point.Exemplars(), exemplarsPerSeries, ts) + } + + return request +} + +func generateAttributes(m pcommon.Map, prefix string, count int) { + for i := 1; i <= count; i++ { + m.PutStr(fmt.Sprintf("%v-name-%v", prefix, i), fmt.Sprintf("value-%v", i)) + } +} + +func generateExemplars(exemplars pmetric.ExemplarSlice, count int, ts pcommon.Timestamp) { + for i := 1; i <= count; i++ { + e := exemplars.AppendEmpty() + e.SetTimestamp(ts) + e.SetDoubleValue(2.22) + e.SetSpanID(pcommon.SpanID{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08}) + e.SetTraceID(pcommon.TraceID{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f}) + } +}