From 2bc9904b7ae86261fcf7633ffa7ed5f549087595 Mon Sep 17 00:00:00 2001 From: David Ashpole Date: Tue, 18 Jul 2023 15:33:18 -0400 Subject: [PATCH] prometheus exporters: Add add_metric_suffixes configuration option (#24260) **Description:** Add add_metric_suffixes configuration option, which can disable the addition of type and unit suffixes. This is backwards-compatible, since the default is true. This is recommended by the specification for sum suffixes in https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/compatibility/prometheus_and_openmetrics.md#sums and allowed in metadata https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/compatibility/prometheus_and_openmetrics.md#metric-metadata-1: `Exporters SHOULD provide a configuration option to disable the addition of _total suffixes.` `The resulting unit SHOULD be added to the metric as OpenMetrics UNIT metadata and as a suffix to the metric name unless the metric name already contains the unit, or the unit MUST be omitted` This deprecates the BuildPromCompliantName function in-favor of BuildCompliantName, which includes the additional argument for configuring suffixes. **Link to tracking Issue:** Fixes https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/21743 Part of https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/8950 --------- Co-authored-by: Dmitrii Anoshin --- ...metheusexporters-normalization-config.yaml | 20 ++++++++ exporter/prometheusexporter/README.md | 2 + exporter/prometheusexporter/collector.go | 20 ++++---- exporter/prometheusexporter/config.go | 3 ++ exporter/prometheusexporter/config_test.go | 5 +- exporter/prometheusexporter/factory.go | 1 + .../prometheusexporter/testdata/config.yaml | 1 + .../prometheusremotewriteexporter/README.md | 1 + .../prometheusremotewriteexporter/config.go | 3 ++ .../config_test.go | 5 +- .../prometheusremotewriteexporter/exporter.go | 1 + .../prometheusremotewriteexporter/factory.go | 1 + .../testdata/config.yaml | 1 + pkg/translator/prometheus/normalize_name.go | 11 +++-- .../prometheus/normalize_name_test.go | 46 +++++++++++++------ .../prometheusremotewrite/helper.go | 4 +- .../prometheusremotewrite/histograms_test.go | 2 +- .../prometheusremotewrite/metrics_to_prw.go | 3 +- .../number_data_points.go | 4 +- 19 files changed, 97 insertions(+), 37 deletions(-) create mode 100644 .chloggen/prometheusexporters-normalization-config.yaml diff --git a/.chloggen/prometheusexporters-normalization-config.yaml b/.chloggen/prometheusexporters-normalization-config.yaml new file mode 100644 index 000000000000..c3231d1aece1 --- /dev/null +++ b/.chloggen/prometheusexporters-normalization-config.yaml @@ -0,0 +1,20 @@ +# Use this changelog template to create an entry for release notes. +# If your change doesn't affect end users, such as a test fix or a tooling change, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: 'enhancement' + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: prometheusremotewriteexporter, prometheusexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: "Add `add_metric_suffixes` configuration option, which can disable the addition of type and unit suffixes." + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [21743, 8950] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: diff --git a/exporter/prometheusexporter/README.md b/exporter/prometheusexporter/README.md index 6ba9c487b976..1951c23e07f9 100644 --- a/exporter/prometheusexporter/README.md +++ b/exporter/prometheusexporter/README.md @@ -33,6 +33,7 @@ The following settings can be optionally configured: - `resource_to_telemetry_conversion` - `enabled` (default = false): If `enabled` is `true`, all the resource attributes will be converted to metric labels by default. - `enable_open_metrics`: (default = `false`): If true, metrics will be exported using the OpenMetrics format. Exemplars are only exported in the OpenMetrics format, and only for histogram and monotonic sum (i.e. counter) metrics. +- `add_metric_suffixes`: (default = `true`): If false, addition of type and unit suffixes is disabled. Example: @@ -51,6 +52,7 @@ exporters: send_timestamps: true metric_expiration: 180m enable_open_metrics: true + add_metric_suffixes: false resource_to_telemetry_conversion: enabled: true ``` diff --git a/exporter/prometheusexporter/collector.go b/exporter/prometheusexporter/collector.go index 660c934c5502..957cf044d998 100644 --- a/exporter/prometheusexporter/collector.go +++ b/exporter/prometheusexporter/collector.go @@ -30,18 +30,20 @@ type collector struct { accumulator accumulator logger *zap.Logger - sendTimestamps bool - namespace string - constLabels prometheus.Labels + sendTimestamps bool + addMetricSuffixes bool + namespace string + constLabels prometheus.Labels } func newCollector(config *Config, logger *zap.Logger) *collector { return &collector{ - accumulator: newAccumulator(logger, config.MetricExpiration), - logger: logger, - namespace: prometheustranslator.CleanUpString(config.Namespace), - sendTimestamps: config.SendTimestamps, - constLabels: config.ConstLabels, + accumulator: newAccumulator(logger, config.MetricExpiration), + logger: logger, + namespace: prometheustranslator.CleanUpString(config.Namespace), + sendTimestamps: config.SendTimestamps, + constLabels: config.ConstLabels, + addMetricSuffixes: config.AddMetricSuffixes, } } @@ -127,7 +129,7 @@ func (c *collector) getMetricMetadata(metric pmetric.Metric, attributes pcommon. } return prometheus.NewDesc( - prometheustranslator.BuildPromCompliantName(metric, c.namespace), + prometheustranslator.BuildCompliantName(metric, c.namespace, c.addMetricSuffixes), metric.Description(), keys, c.constLabels, diff --git a/exporter/prometheusexporter/config.go b/exporter/prometheusexporter/config.go index 419aa38cab1b..80a8bc285dd6 100644 --- a/exporter/prometheusexporter/config.go +++ b/exporter/prometheusexporter/config.go @@ -34,6 +34,9 @@ type Config struct { // EnableOpenMetrics enables the use of the OpenMetrics encoding option for the prometheus exporter. EnableOpenMetrics bool `mapstructure:"enable_open_metrics"` + + // AddMetricSuffixes controls whether suffixes are added to metric names. Defaults to true. + AddMetricSuffixes bool `mapstructure:"add_metric_suffixes"` } var _ component.Config = (*Config)(nil) diff --git a/exporter/prometheusexporter/config_test.go b/exporter/prometheusexporter/config_test.go index 5bb2ac9ec0e5..1be8076596c0 100644 --- a/exporter/prometheusexporter/config_test.go +++ b/exporter/prometheusexporter/config_test.go @@ -50,8 +50,9 @@ func TestLoadConfig(t *testing.T) { "label1": "value1", "another label": "spaced value", }, - SendTimestamps: true, - MetricExpiration: 60 * time.Minute, + SendTimestamps: true, + MetricExpiration: 60 * time.Minute, + AddMetricSuffixes: false, }, }, } diff --git a/exporter/prometheusexporter/factory.go b/exporter/prometheusexporter/factory.go index 8185b31ccbdf..ab1581e4a872 100644 --- a/exporter/prometheusexporter/factory.go +++ b/exporter/prometheusexporter/factory.go @@ -29,6 +29,7 @@ func createDefaultConfig() component.Config { SendTimestamps: false, MetricExpiration: time.Minute * 5, EnableOpenMetrics: false, + AddMetricSuffixes: true, } } diff --git a/exporter/prometheusexporter/testdata/config.yaml b/exporter/prometheusexporter/testdata/config.yaml index 47a8ad574b3f..602b569cd910 100644 --- a/exporter/prometheusexporter/testdata/config.yaml +++ b/exporter/prometheusexporter/testdata/config.yaml @@ -11,3 +11,4 @@ prometheus/2: "another label": spaced value send_timestamps: true metric_expiration: 60m + add_metric_suffixes: false diff --git a/exporter/prometheusremotewriteexporter/README.md b/exporter/prometheusremotewriteexporter/README.md index 33cc93fa0261..f74697a90486 100644 --- a/exporter/prometheusremotewriteexporter/README.md +++ b/exporter/prometheusremotewriteexporter/README.md @@ -50,6 +50,7 @@ The following settings can be optionally configured: - `headers`: additional headers attached to each HTTP request. - *Note the following headers cannot be changed: `Content-Encoding`, `Content-Type`, `X-Prometheus-Remote-Write-Version`, and `User-Agent`.* - `namespace`: prefix attached to each exported metric name. +- `add_metric_suffixes`: If set to false, type and unit suffixes will not be added to metrics. Default: true. - `remote_write_queue`: fine tuning for queueing and sending of the outgoing remote writes. - `enabled`: enable the sending queue - `queue_size`: number of OTLP metrics that can be queued. Ignored if `enabled` is `false` diff --git a/exporter/prometheusremotewriteexporter/config.go b/exporter/prometheusremotewriteexporter/config.go index f70b635ad8d2..ec576e07c084 100644 --- a/exporter/prometheusremotewriteexporter/config.go +++ b/exporter/prometheusremotewriteexporter/config.go @@ -42,6 +42,9 @@ type Config struct { // CreatedMetric allows customizing creation of _created metrics CreatedMetric *CreatedMetric `mapstructure:"export_created_metric,omitempty"` + + // AddMetricSuffixes controls whether unit and type suffixes are added to metrics on export + AddMetricSuffixes bool `mapstructure:"add_metric_suffixes"` } type CreatedMetric struct { diff --git a/exporter/prometheusremotewriteexporter/config_test.go b/exporter/prometheusremotewriteexporter/config_test.go index 822d516185a2..2dcee0eeac49 100644 --- a/exporter/prometheusremotewriteexporter/config_test.go +++ b/exporter/prometheusremotewriteexporter/config_test.go @@ -54,8 +54,9 @@ func TestLoadConfig(t *testing.T) { QueueSize: 2000, NumConsumers: 10, }, - Namespace: "test-space", - ExternalLabels: map[string]string{"key1": "value1", "key2": "value2"}, + AddMetricSuffixes: false, + Namespace: "test-space", + ExternalLabels: map[string]string{"key1": "value1", "key2": "value2"}, HTTPClientSettings: confighttp.HTTPClientSettings{ Endpoint: "localhost:8888", TLSSetting: configtls.TLSClientSetting{ diff --git a/exporter/prometheusremotewriteexporter/exporter.go b/exporter/prometheusremotewriteexporter/exporter.go index 102110994296..296e13d3dac8 100644 --- a/exporter/prometheusremotewriteexporter/exporter.go +++ b/exporter/prometheusremotewriteexporter/exporter.go @@ -73,6 +73,7 @@ func newPRWExporter(cfg *Config, set exporter.CreateSettings) (*prwExporter, err ExternalLabels: sanitizedLabels, DisableTargetInfo: !cfg.TargetInfo.Enabled, ExportCreatedMetric: cfg.CreatedMetric.Enabled, + AddMetricSuffixes: cfg.AddMetricSuffixes, }, } if cfg.WAL == nil { diff --git a/exporter/prometheusremotewriteexporter/factory.go b/exporter/prometheusremotewriteexporter/factory.go index bfbd1b6e222f..859612a8a36a 100644 --- a/exporter/prometheusremotewriteexporter/factory.go +++ b/exporter/prometheusremotewriteexporter/factory.go @@ -80,6 +80,7 @@ func createDefaultConfig() component.Config { RandomizationFactor: backoff.DefaultRandomizationFactor, Multiplier: backoff.DefaultMultiplier, }, + AddMetricSuffixes: true, HTTPClientSettings: confighttp.HTTPClientSettings{ Endpoint: "http://some.url:9411/api/prom/push", // We almost read 0 bytes, so no need to tune ReadBufferSize. diff --git a/exporter/prometheusremotewriteexporter/testdata/config.yaml b/exporter/prometheusremotewriteexporter/testdata/config.yaml index 17df70aa8cbf..4bd9b2bf11ff 100644 --- a/exporter/prometheusremotewriteexporter/testdata/config.yaml +++ b/exporter/prometheusremotewriteexporter/testdata/config.yaml @@ -11,6 +11,7 @@ prometheusremotewrite/2: tls: ca_file: "/var/lib/mycert.pem" write_buffer_size: 524288 + add_metric_suffixes: false headers: Prometheus-Remote-Write-Version: "0.1.0" X-Scope-OrgID: 234 diff --git a/pkg/translator/prometheus/normalize_name.go b/pkg/translator/prometheus/normalize_name.go index da924cd424da..dfb965e9793f 100644 --- a/pkg/translator/prometheus/normalize_name.go +++ b/pkg/translator/prometheus/normalize_name.go @@ -78,7 +78,12 @@ var normalizeNameGate = featuregate.GlobalRegistry().MustRegister( featuregate.WithRegisterReferenceURL("https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/8950"), ) -// Build a Prometheus-compliant metric name for the specified metric +// Deprecated: use BuildCompliantName instead. +func BuildPromCompliantName(metric pmetric.Metric, namespace string) string { + return BuildCompliantName(metric, namespace, true) +} + +// BuildCompliantName builds a Prometheus-compliant metric name for the specified metric // // Metric name is prefixed with specified namespace and underscore (if any). // Namespace is not cleaned up. Make sure specified namespace follows Prometheus @@ -86,11 +91,11 @@ var normalizeNameGate = featuregate.GlobalRegistry().MustRegister( // // See rules at https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels // and https://prometheus.io/docs/practices/naming/#metric-and-label-naming -func BuildPromCompliantName(metric pmetric.Metric, namespace string) string { +func BuildCompliantName(metric pmetric.Metric, namespace string, addMetricSuffixes bool) string { var metricName string // Full normalization following standard Prometheus naming conventions - if normalizeNameGate.IsEnabled() { + if addMetricSuffixes && normalizeNameGate.IsEnabled() { return normalizeName(metric, namespace) } diff --git a/pkg/translator/prometheus/normalize_name_test.go b/pkg/translator/prometheus/normalize_name_test.go index 8ef01f2a4b34..3175e9e04ed5 100644 --- a/pkg/translator/prometheus/normalize_name_test.go +++ b/pkg/translator/prometheus/normalize_name_test.go @@ -210,27 +210,43 @@ func TestRemoveItem(t *testing.T) { } -func TestBuildPromCompliantNameWithNormalize(t *testing.T) { +func TestBuildCompliantNameWithNormalize(t *testing.T) { defer testutil.SetFeatureGateForTest(t, normalizeNameGate, true)() - require.Equal(t, "system_io_bytes_total", BuildPromCompliantName(createCounter("system.io", "By"), "")) - require.Equal(t, "system_network_io_bytes_total", BuildPromCompliantName(createCounter("network.io", "By"), "system")) - require.Equal(t, "_3_14_digits", BuildPromCompliantName(createGauge("3.14 digits", ""), "")) - require.Equal(t, "envoy_rule_engine_zlib_buf_error", BuildPromCompliantName(createGauge("envoy__rule_engine_zlib_buf_error", ""), "")) - require.Equal(t, "foo_bar", BuildPromCompliantName(createGauge(":foo::bar", ""), "")) - require.Equal(t, "foo_bar_total", BuildPromCompliantName(createCounter(":foo::bar", ""), "")) + addUnitAndTypeSuffixes := true + require.Equal(t, "system_io_bytes_total", BuildCompliantName(createCounter("system.io", "By"), "", addUnitAndTypeSuffixes)) + require.Equal(t, "system_network_io_bytes_total", BuildCompliantName(createCounter("network.io", "By"), "system", addUnitAndTypeSuffixes)) + require.Equal(t, "_3_14_digits", BuildCompliantName(createGauge("3.14 digits", ""), "", addUnitAndTypeSuffixes)) + require.Equal(t, "envoy_rule_engine_zlib_buf_error", BuildCompliantName(createGauge("envoy__rule_engine_zlib_buf_error", ""), "", addUnitAndTypeSuffixes)) + require.Equal(t, "foo_bar", BuildCompliantName(createGauge(":foo::bar", ""), "", addUnitAndTypeSuffixes)) + require.Equal(t, "foo_bar_total", BuildCompliantName(createCounter(":foo::bar", ""), "", addUnitAndTypeSuffixes)) } -func TestBuildPromCompliantNameWithoutNormalize(t *testing.T) { +func TestBuildCompliantNameWithSuffixesFeatureGateDisabled(t *testing.T) { defer testutil.SetFeatureGateForTest(t, normalizeNameGate, false)() - require.Equal(t, "system_io", BuildPromCompliantName(createCounter("system.io", "By"), "")) - require.Equal(t, "system_network_io", BuildPromCompliantName(createCounter("network.io", "By"), "system")) - require.Equal(t, "system_network_I_O", BuildPromCompliantName(createCounter("network (I/O)", "By"), "system")) - require.Equal(t, "_3_14_digits", BuildPromCompliantName(createGauge("3.14 digits", "By"), "")) - require.Equal(t, "envoy__rule_engine_zlib_buf_error", BuildPromCompliantName(createGauge("envoy__rule_engine_zlib_buf_error", ""), "")) - require.Equal(t, ":foo::bar", BuildPromCompliantName(createGauge(":foo::bar", ""), "")) - require.Equal(t, ":foo::bar", BuildPromCompliantName(createCounter(":foo::bar", ""), "")) + addUnitAndTypeSuffixes := true + require.Equal(t, "system_io", BuildCompliantName(createCounter("system.io", "By"), "", addUnitAndTypeSuffixes)) + require.Equal(t, "system_network_io", BuildCompliantName(createCounter("network.io", "By"), "system", addUnitAndTypeSuffixes)) + require.Equal(t, "system_network_I_O", BuildCompliantName(createCounter("network (I/O)", "By"), "system", addUnitAndTypeSuffixes)) + require.Equal(t, "_3_14_digits", BuildCompliantName(createGauge("3.14 digits", "By"), "", addUnitAndTypeSuffixes)) + require.Equal(t, "envoy__rule_engine_zlib_buf_error", BuildCompliantName(createGauge("envoy__rule_engine_zlib_buf_error", ""), "", addUnitAndTypeSuffixes)) + require.Equal(t, ":foo::bar", BuildCompliantName(createGauge(":foo::bar", ""), "", addUnitAndTypeSuffixes)) + require.Equal(t, ":foo::bar", BuildCompliantName(createCounter(":foo::bar", ""), "", addUnitAndTypeSuffixes)) + +} + +func TestBuildCompliantNameWithoutSuffixes(t *testing.T) { + + defer testutil.SetFeatureGateForTest(t, normalizeNameGate, false)() + addUnitAndTypeSuffixes := false + require.Equal(t, "system_io", BuildCompliantName(createCounter("system.io", "By"), "", addUnitAndTypeSuffixes)) + require.Equal(t, "system_network_io", BuildCompliantName(createCounter("network.io", "By"), "system", addUnitAndTypeSuffixes)) + require.Equal(t, "system_network_I_O", BuildCompliantName(createCounter("network (I/O)", "By"), "system", addUnitAndTypeSuffixes)) + require.Equal(t, "_3_14_digits", BuildCompliantName(createGauge("3.14 digits", "By"), "", addUnitAndTypeSuffixes)) + require.Equal(t, "envoy__rule_engine_zlib_buf_error", BuildCompliantName(createGauge("envoy__rule_engine_zlib_buf_error", ""), "", addUnitAndTypeSuffixes)) + require.Equal(t, ":foo::bar", BuildCompliantName(createGauge(":foo::bar", ""), "", addUnitAndTypeSuffixes)) + require.Equal(t, ":foo::bar", BuildCompliantName(createCounter(":foo::bar", ""), "", addUnitAndTypeSuffixes)) } diff --git a/pkg/translator/prometheusremotewrite/helper.go b/pkg/translator/prometheusremotewrite/helper.go index b1ad78fc0c7e..f37f29c81d9b 100644 --- a/pkg/translator/prometheusremotewrite/helper.go +++ b/pkg/translator/prometheusremotewrite/helper.go @@ -252,7 +252,7 @@ func isValidAggregationTemporality(metric pmetric.Metric) bool { func addSingleHistogramDataPoint(pt pmetric.HistogramDataPoint, resource pcommon.Resource, metric pmetric.Metric, settings Settings, tsMap map[string]*prompb.TimeSeries) { timestamp := convertTimeStamp(pt.Timestamp()) // sum, count, and buckets of the histogram should append suffix to baseName - baseName := prometheustranslator.BuildPromCompliantName(metric, settings.Namespace) + baseName := prometheustranslator.BuildCompliantName(metric, settings.Namespace, settings.AddMetricSuffixes) // If the sum is unset, it indicates the _sum metric point should be // omitted @@ -442,7 +442,7 @@ func addSingleSummaryDataPoint(pt pmetric.SummaryDataPoint, resource pcommon.Res tsMap map[string]*prompb.TimeSeries) { timestamp := convertTimeStamp(pt.Timestamp()) // sum and count of the summary should append suffix to baseName - baseName := prometheustranslator.BuildPromCompliantName(metric, settings.Namespace) + baseName := prometheustranslator.BuildCompliantName(metric, settings.Namespace, settings.AddMetricSuffixes) // treat sum as a sample in an individual TimeSeries sum := &prompb.Sample{ Value: pt.Sum(), diff --git a/pkg/translator/prometheusremotewrite/histograms_test.go b/pkg/translator/prometheusremotewrite/histograms_test.go index 49e4768e5383..fcef1d4dc924 100644 --- a/pkg/translator/prometheusremotewrite/histograms_test.go +++ b/pkg/translator/prometheusremotewrite/histograms_test.go @@ -742,7 +742,7 @@ func TestAddSingleExponentialHistogramDataPoint(t *testing.T) { for x := 0; x < metric.ExponentialHistogram().DataPoints().Len(); x++ { err := addSingleExponentialHistogramDataPoint( - prometheustranslator.BuildPromCompliantName(metric, ""), + prometheustranslator.BuildCompliantName(metric, "", true), metric.ExponentialHistogram().DataPoints().At(x), pcommon.NewResource(), Settings{}, diff --git a/pkg/translator/prometheusremotewrite/metrics_to_prw.go b/pkg/translator/prometheusremotewrite/metrics_to_prw.go index 205e279020f4..14780dc186fb 100644 --- a/pkg/translator/prometheusremotewrite/metrics_to_prw.go +++ b/pkg/translator/prometheusremotewrite/metrics_to_prw.go @@ -20,6 +20,7 @@ type Settings struct { ExternalLabels map[string]string DisableTargetInfo bool ExportCreatedMetric bool + AddMetricSuffixes bool } // FromMetrics converts pmetric.Metrics to prometheus remote write format. @@ -79,7 +80,7 @@ func FromMetrics(md pmetric.Metrics, settings Settings) (tsMap map[string]*promp if dataPoints.Len() == 0 { errs = multierr.Append(errs, fmt.Errorf("empty data points. %s is dropped", metric.Name())) } - name := prometheustranslator.BuildPromCompliantName(metric, settings.Namespace) + name := prometheustranslator.BuildCompliantName(metric, settings.Namespace, settings.AddMetricSuffixes) for x := 0; x < dataPoints.Len(); x++ { errs = multierr.Append( errs, diff --git a/pkg/translator/prometheusremotewrite/number_data_points.go b/pkg/translator/prometheusremotewrite/number_data_points.go index d50ed6526513..5614424c6ead 100644 --- a/pkg/translator/prometheusremotewrite/number_data_points.go +++ b/pkg/translator/prometheusremotewrite/number_data_points.go @@ -25,7 +25,7 @@ func addSingleGaugeNumberDataPoint( settings Settings, series map[string]*prompb.TimeSeries, ) { - name := prometheustranslator.BuildPromCompliantName(metric, settings.Namespace) + name := prometheustranslator.BuildCompliantName(metric, settings.Namespace, settings.AddMetricSuffixes) labels := createAttributes( resource, pt.Attributes(), @@ -58,7 +58,7 @@ func addSingleSumNumberDataPoint( settings Settings, series map[string]*prompb.TimeSeries, ) { - name := prometheustranslator.BuildPromCompliantName(metric, settings.Namespace) + name := prometheustranslator.BuildCompliantName(metric, settings.Namespace, settings.AddMetricSuffixes) labels := createAttributes( resource, pt.Attributes(),