From 0c3571c98c238780b553128131b220672956d2a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Geisend=C3=B6rfer?= Date: Thu, 26 Jan 2023 18:35:27 +0100 Subject: [PATCH 01/75] WIP: add profiling signal --- .../profiling/v1/profiling_service.proto | 65 ++++++++++ .../profiling/v1/trace_service_http.yaml | 9 ++ .../proto/profiling/v1/profiling.proto | 120 ++++++++++++++++++ 3 files changed, 194 insertions(+) create mode 100644 opentelemetry/proto/collector/profiling/v1/profiling_service.proto create mode 100644 opentelemetry/proto/collector/profiling/v1/trace_service_http.yaml create mode 100644 opentelemetry/proto/profiling/v1/profiling.proto diff --git a/opentelemetry/proto/collector/profiling/v1/profiling_service.proto b/opentelemetry/proto/collector/profiling/v1/profiling_service.proto new file mode 100644 index 000000000..d074cfac9 --- /dev/null +++ b/opentelemetry/proto/collector/profiling/v1/profiling_service.proto @@ -0,0 +1,65 @@ +// Copyright 2019, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package opentelemetry.proto.collector.profiling.v1; + +import "opentelemetry/proto/profiling/v1/profiling.proto"; + +option csharp_namespace = "OpenTelemetry.Proto.Collector.Profiling.V1"; +option java_multiple_files = true; +option java_package = "io.opentelemetry.proto.collector.profiling.v1"; +option java_outer_classname = "ProfilingServiceProto"; +option go_package = "go.opentelemetry.io/proto/otlp/collector/profiling/v1"; + +// Service that can be used to push profiles between one Application instrumented with +// OpenTelemetry and a collector, or between a collector and a central collector (in this +// case spans are sent/received to/from multiple Applications). +service ProfilingService { + // For performance reasons, it is recommended to keep this RPC + // alive for the entire life of the application. + rpc Export(ExportProfilingServiceRequest) returns (ExportProfilingServiceResponse) {} +} + +message ExportProfilingServiceRequest { + // An array of ResourceProfiles. + // For data coming from a single resource this array will typically contain one + // element. Intermediary nodes (such as OpenTelemetry Collector) that receive + // data from multiple origins typically batch the data before forwarding further and + // in that case this array will contain multiple elements. + repeated opentelemetry.proto.profiling.v1.ResourceProfiles resource_profiles = 1; +} + +message ExportProfilingServiceResponse { + // The details of a partially successful export request. + // + // If the request is only partially accepted + // (i.e. when the server accepts only parts of the data and rejects the rest) + // the server MUST initialize the `partial_success` field and MUST + // set the `rejected_` with the number of items it rejected. + // + // Servers MAY also make use of the `partial_success` field to convey + // warnings/suggestions to senders even when the request was fully accepted. + // In such cases, the `rejected_` MUST have a value of `0` and + // the `error_message` MUST be non-empty. + // + // A `partial_success` message with an empty value (rejected_ = 0 and + // `error_message` = "") is equivalent to it not being set/present. Senders + // SHOULD interpret it the same way as in the full success case. + ExportProfilingPartialSuccess partial_success = 1; +} + +message ExportProfilingPartialSuccess { +} diff --git a/opentelemetry/proto/collector/profiling/v1/trace_service_http.yaml b/opentelemetry/proto/collector/profiling/v1/trace_service_http.yaml new file mode 100644 index 000000000..287473597 --- /dev/null +++ b/opentelemetry/proto/collector/profiling/v1/trace_service_http.yaml @@ -0,0 +1,9 @@ +# This is an API configuration to generate an HTTP/JSON -> gRPC gateway for the +# OpenTelemetry service using github.com/grpc-ecosystem/grpc-gateway. +type: google.api.Service +config_version: 3 +http: + rules: + - selector: opentelemetry.proto.collector.trace.v1.TraceService.Export + post: /v1/trace + body: "*" \ No newline at end of file diff --git a/opentelemetry/proto/profiling/v1/profiling.proto b/opentelemetry/proto/profiling/v1/profiling.proto new file mode 100644 index 000000000..ed665d4bf --- /dev/null +++ b/opentelemetry/proto/profiling/v1/profiling.proto @@ -0,0 +1,120 @@ +// Copyright 2019, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package opentelemetry.proto.profiling.v1; + +import "opentelemetry/proto/common/v1/common.proto"; +import "opentelemetry/proto/resource/v1/resource.proto"; + +option csharp_namespace = "OpenTelemetry.Proto.Profiling.V1"; +option java_multiple_files = true; +option java_package = "io.opentelemetry.proto.profiling.v1"; +option java_outer_classname = "ProfilingProto"; +option go_package = "go.opentelemetry.io/proto/otlp/profiling/v1"; + + +// A collection of ScopeProfiles from a Resource. +message ResourceProfiles { + reserved 1000; + + // The resource for the profiles in this message. + // If this field is not set then no resource info is known. + opentelemetry.proto.resource.v1.Resource resource = 1; + + // A list of ScopeProfiles that originate from a resource. + repeated ScopeProfiles scope_profiles = 2; + + // This schema_url applies to the data in the "resource" field. It does not apply + // to the data in the "scope_profiles" field which have their own schema_url field. + string schema_url = 3; +} + +// A collection of Profiles produced by an InstrumentationScope. +message ScopeProfiles { + // The instrumentation scope information for the profiles in this message. + // Semantically when InstrumentationScope isn't set, it is equivalent with + // an empty instrumentation scope name (unknown). + opentelemetry.proto.common.v1.InstrumentationScope scope = 1; + + // A list of Profiles that originate from an instrumentation scope. + repeated Profile profiles = 2; + + // This schema_url applies to all profiles and profile events in the "profiles" field. + string schema_url = 3; +} + +// A Profile represents a single profile. +message Profile { + // A unique identifier for a profile. The ID is a 16-byte array. An ID with + // all zeroes is considered invalid. + // + // This field is required. + bytes profile_id = 1; + + // start_time_unix_nano is the start time of the profile. + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. + // + // This field is semantically required and it is expected that end_time >= start_time. + fixed64 start_time_unix_nano = 2; + + // end_time_unix_nano is the end time of the span. + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. + // + // This field is semantically required and it is expected that end_time >= start_time. + fixed64 end_time_unix_nano = 3; + + // attributes is a collection of key/value pairs. Note, global attributes + // like server name can be set using the resource API. Examples of attributes: + // + // "/http/user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36" + // "/http/server_latency": 300 + // "abc.com/myattribute": true + // "abc.com/score": 10.239 + // + // The OpenTelemetry API specification further restricts the allowed value types: + // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/common/README.md#attribute + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + repeated opentelemetry.proto.common.v1.KeyValue attributes = 4; + + // dropped_attributes_count is the number of attributes that were discarded. Attributes + // can be discarded because their keys are too long or because there are too many + // attributes. If this value is 0, then no attributes were dropped. + uint32 dropped_attributes_count = 5; + + reserved 6 to 15; + + // PPROF contains a pprof profile. + message PPROF { + // payload contains a pprof file. + bytes payload = 1; + } + + // JFR contains a JFR file. + message JFR { + // payload contains a JFR file. + bytes payload = 1; + } + + // OPROF contains a batch of oprof events (details TBD) + message OPROF {} + + oneof profile_kind { + PPROF pprof = 16; + JFR jfr = 17; + OPROF oprof = 18; + } +} From 912021b15b8d1a2c305df0f20977a9ebadafb518 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Wed, 14 Jun 2023 14:13:22 -0700 Subject: [PATCH 02/75] snapshot --- .../v1/profiles_service.proto} | 26 +++++++++---------- .../v1/profiles_service_http.yaml} | 6 ++--- .../v1/profiles.proto} | 10 +++---- 3 files changed, 21 insertions(+), 21 deletions(-) rename opentelemetry/proto/collector/{profiling/v1/profiling_service.proto => profiles/v1/profiles_service.proto} (76%) rename opentelemetry/proto/collector/{profiling/v1/trace_service_http.yaml => profiles/v1/profiles_service_http.yaml} (65%) rename opentelemetry/proto/{profiling/v1/profiling.proto => profiles/v1/profiles.proto} (93%) diff --git a/opentelemetry/proto/collector/profiling/v1/profiling_service.proto b/opentelemetry/proto/collector/profiles/v1/profiles_service.proto similarity index 76% rename from opentelemetry/proto/collector/profiling/v1/profiling_service.proto rename to opentelemetry/proto/collector/profiles/v1/profiles_service.proto index d074cfac9..538d6f47b 100644 --- a/opentelemetry/proto/collector/profiling/v1/profiling_service.proto +++ b/opentelemetry/proto/collector/profiles/v1/profiles_service.proto @@ -14,35 +14,35 @@ syntax = "proto3"; -package opentelemetry.proto.collector.profiling.v1; +package opentelemetry.proto.collector.profiles.v1; -import "opentelemetry/proto/profiling/v1/profiling.proto"; +import "opentelemetry/proto/profiles/v1/profiles.proto"; -option csharp_namespace = "OpenTelemetry.Proto.Collector.Profiling.V1"; +option csharp_namespace = "OpenTelemetry.Proto.Collector.Profiles.V1"; option java_multiple_files = true; -option java_package = "io.opentelemetry.proto.collector.profiling.v1"; -option java_outer_classname = "ProfilingServiceProto"; -option go_package = "go.opentelemetry.io/proto/otlp/collector/profiling/v1"; +option java_package = "io.opentelemetry.proto.collector.profiles.v1"; +option java_outer_classname = "ProfilesServiceProto"; +option go_package = "go.opentelemetry.io/proto/otlp/collector/profiles/v1"; // Service that can be used to push profiles between one Application instrumented with // OpenTelemetry and a collector, or between a collector and a central collector (in this // case spans are sent/received to/from multiple Applications). -service ProfilingService { +service ProfilesService { // For performance reasons, it is recommended to keep this RPC // alive for the entire life of the application. - rpc Export(ExportProfilingServiceRequest) returns (ExportProfilingServiceResponse) {} + rpc Export(ExportProfilesServiceRequest) returns (ExportProfilesServiceResponse) {} } -message ExportProfilingServiceRequest { +message ExportProfilesServiceRequest { // An array of ResourceProfiles. // For data coming from a single resource this array will typically contain one // element. Intermediary nodes (such as OpenTelemetry Collector) that receive // data from multiple origins typically batch the data before forwarding further and // in that case this array will contain multiple elements. - repeated opentelemetry.proto.profiling.v1.ResourceProfiles resource_profiles = 1; + repeated opentelemetry.proto.profiles.v1.ResourceProfiles resource_profiles = 1; } -message ExportProfilingServiceResponse { +message ExportProfilesServiceResponse { // The details of a partially successful export request. // // If the request is only partially accepted @@ -58,8 +58,8 @@ message ExportProfilingServiceResponse { // A `partial_success` message with an empty value (rejected_ = 0 and // `error_message` = "") is equivalent to it not being set/present. Senders // SHOULD interpret it the same way as in the full success case. - ExportProfilingPartialSuccess partial_success = 1; + ExportProfilesPartialSuccess partial_success = 1; } -message ExportProfilingPartialSuccess { +message ExportProfilesPartialSuccess { } diff --git a/opentelemetry/proto/collector/profiling/v1/trace_service_http.yaml b/opentelemetry/proto/collector/profiles/v1/profiles_service_http.yaml similarity index 65% rename from opentelemetry/proto/collector/profiling/v1/trace_service_http.yaml rename to opentelemetry/proto/collector/profiles/v1/profiles_service_http.yaml index 287473597..64a314f1c 100644 --- a/opentelemetry/proto/collector/profiling/v1/trace_service_http.yaml +++ b/opentelemetry/proto/collector/profiles/v1/profiles_service_http.yaml @@ -4,6 +4,6 @@ type: google.api.Service config_version: 3 http: rules: - - selector: opentelemetry.proto.collector.trace.v1.TraceService.Export - post: /v1/trace - body: "*" \ No newline at end of file + - selector: opentelemetry.proto.collector.profiles.v1.TraceService.Export + post: /v1/profiles + body: "*" diff --git a/opentelemetry/proto/profiling/v1/profiling.proto b/opentelemetry/proto/profiles/v1/profiles.proto similarity index 93% rename from opentelemetry/proto/profiling/v1/profiling.proto rename to opentelemetry/proto/profiles/v1/profiles.proto index ed665d4bf..c1afc8d20 100644 --- a/opentelemetry/proto/profiling/v1/profiling.proto +++ b/opentelemetry/proto/profiles/v1/profiles.proto @@ -14,16 +14,16 @@ syntax = "proto3"; -package opentelemetry.proto.profiling.v1; +package opentelemetry.proto.profiles.v1; import "opentelemetry/proto/common/v1/common.proto"; import "opentelemetry/proto/resource/v1/resource.proto"; -option csharp_namespace = "OpenTelemetry.Proto.Profiling.V1"; +option csharp_namespace = "OpenTelemetry.Proto.Profiles.V1"; option java_multiple_files = true; -option java_package = "io.opentelemetry.proto.profiling.v1"; -option java_outer_classname = "ProfilingProto"; -option go_package = "go.opentelemetry.io/proto/otlp/profiling/v1"; +option java_package = "io.opentelemetry.proto.profiles.v1"; +option java_outer_classname = "ProfilesProto"; +option go_package = "go.opentelemetry.io/proto/otlp/profiles/v1"; // A collection of ScopeProfiles from a Resource. From 72ddb55bbab2a048d2fda0664124eb488fa68476 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Wed, 14 Jun 2023 14:14:52 -0700 Subject: [PATCH 03/75] snapshot --- .../proto/collector/profiles/v1/profiles_service_http.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opentelemetry/proto/collector/profiles/v1/profiles_service_http.yaml b/opentelemetry/proto/collector/profiles/v1/profiles_service_http.yaml index 64a314f1c..eed628920 100644 --- a/opentelemetry/proto/collector/profiles/v1/profiles_service_http.yaml +++ b/opentelemetry/proto/collector/profiles/v1/profiles_service_http.yaml @@ -4,6 +4,6 @@ type: google.api.Service config_version: 3 http: rules: - - selector: opentelemetry.proto.collector.profiles.v1.TraceService.Export + - selector: opentelemetry.proto.collector.profiles.v1.ProfilesService.Export post: /v1/profiles body: "*" From 05ecb3c0e54b534b606189f5d2371bbd309d3cf7 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Wed, 14 Jun 2023 15:16:52 -0700 Subject: [PATCH 04/75] empty profile proto --- .../profiles/v1/profiles_service.proto | 14 +++++ .../proto/profiles/v1/profiles.proto | 55 +++++++++++++------ 2 files changed, 51 insertions(+), 18 deletions(-) diff --git a/opentelemetry/proto/collector/profiles/v1/profiles_service.proto b/opentelemetry/proto/collector/profiles/v1/profiles_service.proto index 538d6f47b..aaaefdef7 100644 --- a/opentelemetry/proto/collector/profiles/v1/profiles_service.proto +++ b/opentelemetry/proto/collector/profiles/v1/profiles_service.proto @@ -62,4 +62,18 @@ message ExportProfilesServiceResponse { } message ExportProfilesPartialSuccess { + // The number of rejected profile records. + // + // A `rejected_` field holding a `0` value indicates that the + // request was fully accepted. + int64 rejected_profile_records = 1; + + // A developer-facing human-readable message in English. It should be used + // either to explain why the server rejected parts of the data during a partial + // success or to convey warnings/suggestions during a full success. The message + // should offer guidance on how users can address such issues. + // + // error_message is an optional field. An error_message with an empty value + // is equivalent to it not being set. + string error_message = 2; } diff --git a/opentelemetry/proto/profiles/v1/profiles.proto b/opentelemetry/proto/profiles/v1/profiles.proto index c1afc8d20..08dfdffc3 100644 --- a/opentelemetry/proto/profiles/v1/profiles.proto +++ b/opentelemetry/proto/profiles/v1/profiles.proto @@ -25,6 +25,25 @@ option java_package = "io.opentelemetry.proto.profiles.v1"; option java_outer_classname = "ProfilesProto"; option go_package = "go.opentelemetry.io/proto/otlp/profiles/v1"; +// ProfilesData represents the profiles data that can be stored in a persistent storage, +// OR can be embedded by other protocols that transfer OTLP profiles data but do not +// implement the OTLP protocol. +// +// The main difference between this message and collector protocol is that +// in this message there will not be any "control" or "metadata" specific to +// OTLP protocol. +// +// When new fields are added into this message, the OTLP request MUST be updated +// as well. +message ProfilesData { + // An array of ResourceProfiles. + // For data coming from a single resource this array will typically contain + // one element. Intermediary nodes that receive data from multiple origins + // typically batch the data before forwarding further and in that case this + // array will contain multiple elements. + repeated ResourceProfiles resource_profiles = 1; +} + // A collection of ScopeProfiles from a Resource. message ResourceProfiles { @@ -95,26 +114,26 @@ message Profile { // attributes. If this value is 0, then no attributes were dropped. uint32 dropped_attributes_count = 5; - reserved 6 to 15; + // reserved 6 to 15; - // PPROF contains a pprof profile. - message PPROF { - // payload contains a pprof file. - bytes payload = 1; - } + // // PPROF contains a pprof profile. + // message PPROF { + // // payload contains a pprof file. + // bytes payload = 1; + // } - // JFR contains a JFR file. - message JFR { - // payload contains a JFR file. - bytes payload = 1; - } + // // JFR contains a JFR file. + // message JFR { + // // payload contains a JFR file. + // bytes payload = 1; + // } - // OPROF contains a batch of oprof events (details TBD) - message OPROF {} + // // OPROF contains a batch of oprof events (details TBD) + // message OPROF {} - oneof profile_kind { - PPROF pprof = 16; - JFR jfr = 17; - OPROF oprof = 18; - } + // oneof profile_kind { + // PPROF pprof = 16; + // JFR jfr = 17; + // OPROF oprof = 18; + // } } From 1ebf7eb59ed5dd82ed5a1144c5fdaac0013ba879 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Tue, 20 Jun 2023 11:08:47 -0700 Subject: [PATCH 05/75] improved naming (profile_records came from log_records) --- .../proto/collector/profiles/v1/profiles_service.proto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opentelemetry/proto/collector/profiles/v1/profiles_service.proto b/opentelemetry/proto/collector/profiles/v1/profiles_service.proto index aaaefdef7..bfbd0aa29 100644 --- a/opentelemetry/proto/collector/profiles/v1/profiles_service.proto +++ b/opentelemetry/proto/collector/profiles/v1/profiles_service.proto @@ -62,11 +62,11 @@ message ExportProfilesServiceResponse { } message ExportProfilesPartialSuccess { - // The number of rejected profile records. + // The number of rejected profiles. // // A `rejected_` field holding a `0` value indicates that the // request was fully accepted. - int64 rejected_profile_records = 1; + int64 rejected_profiles = 1; // A developer-facing human-readable message in English. It should be used // either to explain why the server rejected parts of the data during a partial From fb6e2152bc0ff20943686d4ff27110cb4848d4c0 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Mon, 26 Jun 2023 22:49:17 -0700 Subject: [PATCH 06/75] adds alternative variations of profiles proto --- .../denormalized/denormalized.proto | 131 +++++++++++ .../alternatives/normalized/normalized.proto | 142 +++++++++++ .../v1/alternatives/pprof/pprof.proto | 221 ++++++++++++++++++ .../proto/profiles/v1/profiles.proto | 38 ++- 4 files changed, 510 insertions(+), 22 deletions(-) create mode 100644 opentelemetry/proto/profiles/v1/alternatives/denormalized/denormalized.proto create mode 100644 opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto create mode 100644 opentelemetry/proto/profiles/v1/alternatives/pprof/pprof.proto diff --git a/opentelemetry/proto/profiles/v1/alternatives/denormalized/denormalized.proto b/opentelemetry/proto/profiles/v1/alternatives/denormalized/denormalized.proto new file mode 100644 index 000000000..605c913e3 --- /dev/null +++ b/opentelemetry/proto/profiles/v1/alternatives/denormalized/denormalized.proto @@ -0,0 +1,131 @@ +syntax = "proto3"; + +package opentelemetry.proto.profiles.v1.alternatives.denormalized; + +import "opentelemetry/proto/common/v1/common.proto"; +import "opentelemetry/proto/resource/v1/resource.proto"; + +option go_package = "go.opentelemetry.io/proto/otlp/profiles/v1/alternatives/denormalized"; + + +// A pointer from a profile to a trace span. +message Link { + // A unique identifier of a trace that this linked span is part of. The ID is a + // 16-byte array. + bytes trace_id = 1; + // A unique identifier for the linked span. The ID is an 8-byte array. + bytes span_id = 2; +} + +message Sample { + repeated Location locations = 1; + repeated Link links = 2; + repeated opentelemetry.proto.common.v1.KeyValue attributes = 3; + uint32 dropped_attributes_count = 4; + + // this one is repeated because there can be multiple profile kinds in one profile. Typical example is memory profiles in go that contain: + // * alloc_objects + // * alloc_bytes + // * inuse_objects + // * inuse_bytes + repeated fixed64 values = 5; + + // optional + fixed64 timestamp_unix_nano = 6; +} + +// borrowed from metrics proto +enum AggregationTemporality { + AGGREGATION_TEMPORALITY_UNSPECIFIED = 0; + AGGREGATION_TEMPORALITY_DELTA = 1; + AGGREGATION_TEMPORALITY_CUMULATIVE = 2; +} + +message SampleType { + AggregationTemporality aggregation_temporality = 1; + uint64 sample_size = 2; + + // CPU / memory /etc + string type = 3; + string unit = 4; +} + +message Profile { + repeated SampleType sample_types = 1; + repeated Sample samples = 2; +} + +// borrowed from pprof proto +message Mapping { + // Address at which the binary (or DLL) is loaded into memory. + uint64 memory_start = 1; + // The limit of the address range occupied by this mapping. + uint64 memory_limit = 2; + // Offset in the binary that corresponds to the first mapped address. + uint64 file_offset = 3; + // The object this entry is loaded from. This can be a filename on + // disk for the main binary and shared libraries, or virtual + // abstractions like "[vdso]". + string filename = 4; + // A string that uniquely identifies a particular program version + // with high probability. E.g., for binaries generated by GNU tools, + // it could be the contents of the .note.gnu.build-id field. + string build_id = 5; + + // The following fields indicate the resolution of symbolic info. + bool has_functions = 6; + bool has_filenames = 7; + bool has_line_numbers = 8; + bool has_inline_frames = 9; +} + +// borrowed from pprof proto +// Describes function and line table debug information. +message Location { + // profile.Mapping for this location. + // It can be unset if the mapping is unknown or not applicable for + // this profile type. + Mapping mapping = 1; + + // The instruction address for this location, if available. It + // should be within [Mapping.memory_start...Mapping.memory_limit] + // for the corresponding mapping. A non-leaf address may be in the + // middle of a call instruction. It is up to display tools to find + // the beginning of the instruction if necessary. + uint64 address = 2; + // Multiple line indicates this location has inlined functions, + // where the last entry represents the caller into which the + // preceding entries were inlined. + // + // E.g., if memcpy() is inlined into printf: + // line[0].function_name == "memcpy" + // line[1].function_name == "printf" + repeated Line line = 3; + // Provides an indication that multiple symbols map to this location's + // address, for example due to identical code folding by the linker. In that + // case the line information above represents one of the multiple + // symbols. This field must be recomputed when the symbolization state of the + // profile changes. + bool is_folded = 4; +} + +// borrowed from pprof proto +message Line { + // Function for this line. + Function function = 1; + // Line number in source code. + int64 line = 2; +} + +// borrowed from pprof proto +message Function { + // Name of the function, in human-readable form if available. + string name = 1; + // Name of the function, as identified by the system. + // For instance, it can be a C++ mangled name. + string system_name = 2; + // Source file containing the function. + string filename = 3; + // Line number in source file. + string start_line = 4; +} diff --git a/opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto b/opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto new file mode 100644 index 000000000..7fd156f0e --- /dev/null +++ b/opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto @@ -0,0 +1,142 @@ +syntax = "proto3"; + +package opentelemetry.proto.profiles.v1.alternatives.normalized; + +import "opentelemetry/proto/common/v1/common.proto"; +import "opentelemetry/proto/resource/v1/resource.proto"; + +option go_package = "go.opentelemetry.io/proto/otlp/profiles/v1/normalized"; + + +// A pointer from a profile to a trace span. +message Link { + // A unique identifier of a trace that this linked span is part of. The ID is a + // 16-byte array. + bytes trace_id = 1; + // A unique identifier for the linked span. The ID is an 8-byte array. + bytes span_id = 2; +} + +// we link to attribute sets because +message AttributeSet { + repeated opentelemetry.proto.common.v1.KeyValue attributes = 1; + uint32 dropped_attributes_count = 2; +} + +message Sample { + repeated uint64 location_ids = 1; + repeated uint64 link_ids = 2; + repeated uint64 attribute_set_ids = 3; + + // this one is repeated because there can be multiple profile kinds in one profile. Typical example is memory profiles in go that contain: + // * alloc_objects + // * alloc_bytes + // * inuse_objects + // * inuse_bytes + repeated fixed64 values = 4; + + // optional + fixed64 timestamp_unix_nano = 5; +} + +// borrowed from metrics proto +enum AggregationTemporality { + AGGREGATION_TEMPORALITY_UNSPECIFIED = 0; + AGGREGATION_TEMPORALITY_DELTA = 1; + AGGREGATION_TEMPORALITY_CUMULATIVE = 2; +} + +message SampleType { + AggregationTemporality aggregation_temporality = 1; + uint64 sample_size = 2; + + // CPU / memory /etc + int64 type = 3; // Index into string table. + int64 unit = 4; // Index into string table. +} + +message Profile { + repeated SampleType sample_types = 1; + repeated Sample samples = 2; + repeated Mapping mappings = 3; + repeated Location locations = 4; + repeated Function functions = 5; + repeated Link links = 6; + repeated AttributeSet attribute_sets = 7; + repeated string string_table = 8; +} + + +// borrowed from pprof proto +message Mapping { + // Address at which the binary (or DLL) is loaded into memory. + uint64 memory_start = 1; + // The limit of the address range occupied by this mapping. + uint64 memory_limit = 2; + // Offset in the binary that corresponds to the first mapped address. + uint64 file_offset = 3; + // The object this entry is loaded from. This can be a filename on + // disk for the main binary and shared libraries, or virtual + // abstractions like "[vdso]". + int64 filename = 4; // Index into string table + // A string that uniquely identifies a particular program version + // with high probability. E.g., for binaries generated by GNU tools, + // it could be the contents of the .note.gnu.build-id field. + int64 build_id = 5; // Index into string table + + // The following fields indicate the resolution of symbolic info. + bool has_functions = 6; + bool has_filenames = 7; + bool has_line_numbers = 8; + bool has_inline_frames = 9; +} + +// borrowed from pprof proto +// Describes function and line table debug information. +message Location { + // The id of the corresponding profile.Mapping for this location. + // It can be unset if the mapping is unknown or not applicable for + // this profile type. + uint64 mapping_id = 1; + // The instruction address for this location, if available. It + // should be within [Mapping.memory_start...Mapping.memory_limit] + // for the corresponding mapping. A non-leaf address may be in the + // middle of a call instruction. It is up to display tools to find + // the beginning of the instruction if necessary. + uint64 address = 2; + // Multiple line indicates this location has inlined functions, + // where the last entry represents the caller into which the + // preceding entries were inlined. + // + // E.g., if memcpy() is inlined into printf: + // line[0].function_name == "memcpy" + // line[1].function_name == "printf" + repeated Line line = 3; + // Provides an indication that multiple symbols map to this location's + // address, for example due to identical code folding by the linker. In that + // case the line information above represents one of the multiple + // symbols. This field must be recomputed when the symbolization state of the + // profile changes. + bool is_folded = 4; +} + +// borrowed from pprof proto +message Line { + // The id of the corresponding profile.Function for this line. + uint64 function_id = 1; + // Line number in source code. + int64 line = 2; +} + +// borrowed from pprof proto +message Function { + // Name of the function, in human-readable form if available. + int64 name = 1; // Index into string table + // Name of the function, as identified by the system. + // For instance, it can be a C++ mangled name. + int64 system_name = 2; // Index into string table + // Source file containing the function. + int64 filename = 3; // Index into string table + // Line number in source file. + int64 start_line = 4; +} diff --git a/opentelemetry/proto/profiles/v1/alternatives/pprof/pprof.proto b/opentelemetry/proto/profiles/v1/alternatives/pprof/pprof.proto new file mode 100644 index 000000000..ec55bc207 --- /dev/null +++ b/opentelemetry/proto/profiles/v1/alternatives/pprof/pprof.proto @@ -0,0 +1,221 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Profile is a common stacktrace profile format. +// +// Measurements represented with this format should follow the +// following conventions: +// +// - Consumers should treat unset optional fields as if they had been +// set with their default value. +// +// - When possible, measurements should be stored in "unsampled" form +// that is most useful to humans. There should be enough +// information present to determine the original sampled values. +// +// - On-disk, the serialized proto must be gzip-compressed. +// +// - The profile is represented as a set of samples, where each sample +// references a sequence of locations, and where each location belongs +// to a mapping. +// - There is a N->1 relationship from sample.location_id entries to +// locations. For every sample.location_id entry there must be a +// unique Location with that id. +// - There is an optional N->1 relationship from locations to +// mappings. For every nonzero Location.mapping_id there must be a +// unique Mapping with that id. + +syntax = "proto3"; + +package opentelemetry.proto.profiles.v1.alternatives.pprof; + +option go_package = "go.opentelemetry.io/proto/otlp/profiles/v1/alternatives/pprof"; + +message Profile { + // A description of the samples associated with each Sample.value. + // For a cpu profile this might be: + // [["cpu","nanoseconds"]] or [["wall","seconds"]] or [["syscall","count"]] + // For a heap profile, this might be: + // [["allocations","count"], ["space","bytes"]], + // If one of the values represents the number of events represented + // by the sample, by convention it should be at index 0 and use + // sample_type.unit == "count". + repeated ValueType sample_type = 1; + // The set of samples recorded in this profile. + repeated Sample sample = 2; + // Mapping from address ranges to the image/binary/library mapped + // into that address range. mapping[0] will be the main binary. + repeated Mapping mapping = 3; + // Locations referenced by samples. + repeated Location location = 4; + // Functions referenced by locations. + repeated Function function = 5; + // A common table for strings referenced by various messages. + // string_table[0] must always be "". + repeated string string_table = 6; + // frames with Function.function_name fully matching the following + // regexp will be dropped from the samples, along with their successors. + int64 drop_frames = 7; // Index into string table. + // frames with Function.function_name fully matching the following + // regexp will be kept, even if it matches drop_frames. + int64 keep_frames = 8; // Index into string table. + + // The following fields are informational, do not affect + // interpretation of results. + + // Time of collection (UTC) represented as nanoseconds past the epoch. + int64 time_nanos = 9; + // Duration of the profile, if a duration makes sense. + int64 duration_nanos = 10; + // The kind of events between sampled occurrences. + // e.g [ "cpu","cycles" ] or [ "heap","bytes" ] + ValueType period_type = 11; + // The number of events between sampled occurrences. + int64 period = 12; + // Free-form text associated with the profile. The text is displayed as is + // to the user by the tools that read profiles (e.g. by pprof). This field + // should not be used to store any machine-readable information, it is only + // for human-friendly content. The profile must stay functional if this field + // is cleaned. + repeated int64 comment = 13; // Indices into string table. + // Index into the string table of the type of the preferred sample + // value. If unset, clients should default to the last sample value. + int64 default_sample_type = 14; +} + +// ValueType describes the semantics and measurement units of a value. +message ValueType { + int64 type = 1; // Index into string table. + int64 unit = 2; // Index into string table. +} + +// Each Sample records values encountered in some program +// context. The program context is typically a stack trace, perhaps +// augmented with auxiliary information like the thread-id, some +// indicator of a higher level request being handled etc. +message Sample { + // The ids recorded here correspond to a Profile.location.id. + // The leaf is at location_id[0]. + repeated uint64 location_id = 1; + // The type and unit of each value is defined by the corresponding + // entry in Profile.sample_type. All samples must have the same + // number of values, the same as the length of Profile.sample_type. + // When aggregating multiple samples into a single sample, the + // result has a list of values that is the element-wise sum of the + // lists of the originals. + repeated int64 value = 2; + // label includes additional context for this sample. It can include + // things like a thread id, allocation size, etc. + // + // NOTE: While possible, having multiple values for the same label key is + // strongly discouraged and should never be used. Most tools (e.g. pprof) do + // not have good (or any) support for multi-value labels. And an even more + // discouraged case is having a string label and a numeric label of the same + // name on a sample. Again, possible to express, but should not be used. + repeated Label label = 3; +} + +message Label { + int64 key = 1; // Index into string table + + // At most one of the following must be present + int64 str = 2; // Index into string table + int64 num = 3; + + // Should only be present when num is present. + // Specifies the units of num. + // Use arbitrary string (for example, "requests") as a custom count unit. + // If no unit is specified, consumer may apply heuristic to deduce the unit. + // Consumers may also interpret units like "bytes" and "kilobytes" as memory + // units and units like "seconds" and "nanoseconds" as time units, + // and apply appropriate unit conversions to these. + int64 num_unit = 4; // Index into string table +} + +message Mapping { + // Unique nonzero id for the mapping. + uint64 id = 1; + // Address at which the binary (or DLL) is loaded into memory. + uint64 memory_start = 2; + // The limit of the address range occupied by this mapping. + uint64 memory_limit = 3; + // Offset in the binary that corresponds to the first mapped address. + uint64 file_offset = 4; + // The object this entry is loaded from. This can be a filename on + // disk for the main binary and shared libraries, or virtual + // abstractions like "[vdso]". + int64 filename = 5; // Index into string table + // A string that uniquely identifies a particular program version + // with high probability. E.g., for binaries generated by GNU tools, + // it could be the contents of the .note.gnu.build-id field. + int64 build_id = 6; // Index into string table + + // The following fields indicate the resolution of symbolic info. + bool has_functions = 7; + bool has_filenames = 8; + bool has_line_numbers = 9; + bool has_inline_frames = 10; +} + +// Describes function and line table debug information. +message Location { + // Unique nonzero id for the location. A profile could use + // instruction addresses or any integer sequence as ids. + uint64 id = 1; + // The id of the corresponding profile.Mapping for this location. + // It can be unset if the mapping is unknown or not applicable for + // this profile type. + uint64 mapping_id = 2; + // The instruction address for this location, if available. It + // should be within [Mapping.memory_start...Mapping.memory_limit] + // for the corresponding mapping. A non-leaf address may be in the + // middle of a call instruction. It is up to display tools to find + // the beginning of the instruction if necessary. + uint64 address = 3; + // Multiple line indicates this location has inlined functions, + // where the last entry represents the caller into which the + // preceding entries were inlined. + // + // E.g., if memcpy() is inlined into printf: + // line[0].function_name == "memcpy" + // line[1].function_name == "printf" + repeated Line line = 4; + // Provides an indication that multiple symbols map to this location's + // address, for example due to identical code folding by the linker. In that + // case the line information above represents one of the multiple + // symbols. This field must be recomputed when the symbolization state of the + // profile changes. + bool is_folded = 5; +} + +message Line { + // The id of the corresponding profile.Function for this line. + uint64 function_id = 1; + // Line number in source code. + int64 line = 2; +} + +message Function { + // Unique nonzero id for the function. + uint64 id = 1; + // Name of the function, in human-readable form if available. + int64 name = 2; // Index into string table + // Name of the function, as identified by the system. + // For instance, it can be a C++ mangled name. + int64 system_name = 3; // Index into string table + // Source file containing the function. + int64 filename = 4; // Index into string table + // Line number in source file. + int64 start_line = 5; +} diff --git a/opentelemetry/proto/profiles/v1/profiles.proto b/opentelemetry/proto/profiles/v1/profiles.proto index 08dfdffc3..1adae179f 100644 --- a/opentelemetry/proto/profiles/v1/profiles.proto +++ b/opentelemetry/proto/profiles/v1/profiles.proto @@ -19,6 +19,12 @@ package opentelemetry.proto.profiles.v1; import "opentelemetry/proto/common/v1/common.proto"; import "opentelemetry/proto/resource/v1/resource.proto"; +// TODO(@petethepig): idea here is that we will eventually get rid of these +// and have just version in this (profiles.proto) file +import "opentelemetry/proto/profiles/v1/alternatives/pprof/pprof.proto"; +import "opentelemetry/proto/profiles/v1/alternatives/denormalized/denormalized.proto"; +import "opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto"; + option csharp_namespace = "OpenTelemetry.Proto.Profiles.V1"; option java_multiple_files = true; option java_package = "io.opentelemetry.proto.profiles.v1"; @@ -89,7 +95,7 @@ message Profile { // This field is semantically required and it is expected that end_time >= start_time. fixed64 start_time_unix_nano = 2; - // end_time_unix_nano is the end time of the span. + // end_time_unix_nano is the end time of the profile. // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. // // This field is semantically required and it is expected that end_time >= start_time. @@ -114,26 +120,14 @@ message Profile { // attributes. If this value is 0, then no attributes were dropped. uint32 dropped_attributes_count = 5; - // reserved 6 to 15; - - // // PPROF contains a pprof profile. - // message PPROF { - // // payload contains a pprof file. - // bytes payload = 1; - // } - - // // JFR contains a JFR file. - // message JFR { - // // payload contains a JFR file. - // bytes payload = 1; - // } - - // // OPROF contains a batch of oprof events (details TBD) - // message OPROF {} + // This can be original pprof or jfr encoded profile. + bytes original_payload = 7; - // oneof profile_kind { - // PPROF pprof = 16; - // JFR jfr = 17; - // OPROF oprof = 18; - // } + // this is temporary until we pick one of these formats + // fields from one of these will be embedded in Profile message: + oneof alternative_profile { + opentelemetry.proto.profiles.v1.alternatives.pprof.Profile pprof = 8; + opentelemetry.proto.profiles.v1.alternatives.normalized.Profile normalized = 9; + opentelemetry.proto.profiles.v1.alternatives.denormalized.Profile denormalized = 10; + } } From 25dce2ebcc39508e4e29b9eb5e92e606f3d142b7 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Mon, 26 Jun 2023 23:03:56 -0700 Subject: [PATCH 07/75] fix --- .../proto/profiles/v1/alternatives/normalized/normalized.proto | 1 - 1 file changed, 1 deletion(-) diff --git a/opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto b/opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto index 7fd156f0e..ab69254e2 100644 --- a/opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto +++ b/opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto @@ -17,7 +17,6 @@ message Link { bytes span_id = 2; } -// we link to attribute sets because message AttributeSet { repeated opentelemetry.proto.common.v1.KeyValue attributes = 1; uint32 dropped_attributes_count = 2; From 076244ac5490dd262a966c91907a76f45cadca28 Mon Sep 17 00:00:00 2001 From: Ryan Perry Date: Fri, 30 Jun 2023 13:26:27 -0400 Subject: [PATCH 08/75] Test PR merging abilities --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index d22b11899..1bc502643 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ and the corresponding Language Independent Interface Types ([.proto files](opent ## Language Independent Interface Types -The proto files can be consumed as GIT submodules or copied and built directly in the consumer project. +The `.proto` files can be consumed as GIT submodules or copied and built directly into the consumer project. The compiled files are published to central repositories (Maven, ...) from OpenTelemetry client libraries. @@ -15,11 +15,11 @@ See [contribution guidelines](CONTRIBUTING.md) if you would like to make any cha ## OTLP/JSON -See additional requirements for [OTLP/JSON wire representation here](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/otlp.md#json-protobuf-encoding). +For additional requirements regarding the OTLP/JSON wire representation, refer to [OTLP/JSON wire representation here](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/otlp.md#json-protobuf-encoding). ## Generate gRPC Client Libraries -To generate the raw gRPC client libraries, use `make gen-${LANGUAGE}`. Currently supported languages are: +To generate the raw gRPC client libraries, use the command `make gen-${LANGUAGE}`. The currently supported languages are: * cpp * csharp @@ -33,8 +33,8 @@ To generate the raw gRPC client libraries, use `make gen-${LANGUAGE}`. Currently ## Maturity Level -1.0.0 and newer releases from this repository may contain unstable (alpha or beta) -components as indicated by the Maturity table below. +Releases 1.0.0 and newer releases from this repository may contain unstable (alpha or beta) +components as indicated by the Maturity table below: | Component | Binary Protobuf Maturity | JSON Maturity | | --------- |--------------- | ------------- | @@ -56,7 +56,7 @@ Components marked `Stable` provide the following guarantees: - Service method names will not change. [from 1.0.0] - Service method parameter names will not change. [from 1.0.0] - Service method parameter types and return types will not change. [from 1.0.0] -- Service method kind (unary vs streaming) will not change. +- Service method kind (unary vs. streaming) will not change. - Names of messages and enums will not change. [from 1.0.0] - Numbers assigned to enum choices will not change. - Names of enum choices will not change. [from 1.0.0] @@ -66,7 +66,7 @@ Components marked `Stable` provide the following guarantees: - `optional` and `repeated` declarators of existing fields will not change. [from 1.0.0] - No existing symbol will be deleted. [from 1.0.0] -Note: guarantees marked [from 1.0.0] will go into effect when this repository is tagged +Note: The guarantees marked as [from 1.0.0] will go into effect when this repository is tagged with version number 1.0.0. The following additive changes are allowed: @@ -87,7 +87,7 @@ is generated from the .proto files by any particular code generator. ## Experiments -In some cases we are trying to experiment with different features. In this case, +In some cases, we are trying to experiment with different features. In this case, we recommend using an "experimental" sub-directory instead of adding them to any protocol version. These protocols should not be used, except for development/testing purposes. From 169500224961e4b3f8dacb81d8923b1388c45c30 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Wed, 5 Jul 2023 11:36:25 -0700 Subject: [PATCH 09/75] incorporating the feedback from the last meeting --- .../alternatives/normalized/normalized.proto | 62 +++++++++++-------- 1 file changed, 35 insertions(+), 27 deletions(-) diff --git a/opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto b/opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto index ab69254e2..2746db3ae 100644 --- a/opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto +++ b/opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto @@ -17,41 +17,44 @@ message Link { bytes span_id = 2; } +// AttributeSet represents a set of attributes. It is referenced from Samples +// and not embedded because it is common for multiple samples to have the same +// set of attributes message AttributeSet { repeated opentelemetry.proto.common.v1.KeyValue attributes = 1; uint32 dropped_attributes_count = 2; } message Sample { - repeated uint64 location_ids = 1; - repeated uint64 link_ids = 2; - repeated uint64 attribute_set_ids = 3; + repeated uint64 location_indices = 1; + repeated uint64 link_indices = 2; + repeated uint64 attribute_set_indices = 3; // this one is repeated because there can be multiple profile kinds in one profile. Typical example is memory profiles in go that contain: // * alloc_objects // * alloc_bytes // * inuse_objects // * inuse_bytes - repeated fixed64 values = 4; + repeated int64 values = 4; // optional fixed64 timestamp_unix_nano = 5; } -// borrowed from metrics proto -enum AggregationTemporality { - AGGREGATION_TEMPORALITY_UNSPECIFIED = 0; - AGGREGATION_TEMPORALITY_DELTA = 1; - AGGREGATION_TEMPORALITY_CUMULATIVE = 2; -} - message SampleType { + // borrowed from metrics proto + enum AggregationTemporality { + AGGREGATION_TEMPORALITY_UNSPECIFIED = 0; + AGGREGATION_TEMPORALITY_DELTA = 1; + AGGREGATION_TEMPORALITY_CUMULATIVE = 2; + } + AggregationTemporality aggregation_temporality = 1; uint64 sample_size = 2; // CPU / memory /etc - int64 type = 3; // Index into string table. - int64 unit = 4; // Index into string table. + uint32 type_index = 3; // Index into string table. + uint32 unit_index = 4; // Index into string table. } message Profile { @@ -62,6 +65,8 @@ message Profile { repeated Function functions = 5; repeated Link links = 6; repeated AttributeSet attribute_sets = 7; + + // 0-th element must be an empty string ("") repeated string string_table = 8; } @@ -77,17 +82,20 @@ message Mapping { // The object this entry is loaded from. This can be a filename on // disk for the main binary and shared libraries, or virtual // abstractions like "[vdso]". - int64 filename = 4; // Index into string table + uint32 filename_index = 4; // Index into string table // A string that uniquely identifies a particular program version // with high probability. E.g., for binaries generated by GNU tools, // it could be the contents of the .note.gnu.build-id field. - int64 build_id = 5; // Index into string table + uint32 build_id_index = 5; // Index into string table + + enum SymbolicInfo { + SYMBOLIC_INFO_UNSPECIFIED = 0; + SYMBOLIC_INFO_FULL = 1; + SYMBOLIC_INFO_FUNCTIONS_ONLY = 2; + SYMBOLIC_INFO_NO_INLINE_FRAMES = 3; + } - // The following fields indicate the resolution of symbolic info. - bool has_functions = 6; - bool has_filenames = 7; - bool has_line_numbers = 8; - bool has_inline_frames = 9; + SymbolicInfo symbolic_info = 6; } // borrowed from pprof proto @@ -96,7 +104,7 @@ message Location { // The id of the corresponding profile.Mapping for this location. // It can be unset if the mapping is unknown or not applicable for // this profile type. - uint64 mapping_id = 1; + uint32 mapping_index = 1; // The instruction address for this location, if available. It // should be within [Mapping.memory_start...Mapping.memory_limit] // for the corresponding mapping. A non-leaf address may be in the @@ -122,20 +130,20 @@ message Location { // borrowed from pprof proto message Line { // The id of the corresponding profile.Function for this line. - uint64 function_id = 1; + uint32 function_index = 1; // Line number in source code. - int64 line = 2; + uint32 line = 2; } // borrowed from pprof proto message Function { // Name of the function, in human-readable form if available. - int64 name = 1; // Index into string table + uint32 name_index = 1; // Index into string table // Name of the function, as identified by the system. // For instance, it can be a C++ mangled name. - int64 system_name = 2; // Index into string table + uint32 system_name_index = 2; // Index into string table // Source file containing the function. - int64 filename = 3; // Index into string table + uint32 filename_index = 3; // Index into string table // Line number in source file. - int64 start_line = 4; + uint32 start_line = 4; } From 7faab0e9b4298ccdd985b06550ee308457a0f7a3 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Thu, 6 Jul 2023 09:05:35 -0700 Subject: [PATCH 10/75] copyright year fix --- opentelemetry/proto/profiles/v1/profiles.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opentelemetry/proto/profiles/v1/profiles.proto b/opentelemetry/proto/profiles/v1/profiles.proto index 1adae179f..b2d4b83a3 100644 --- a/opentelemetry/proto/profiles/v1/profiles.proto +++ b/opentelemetry/proto/profiles/v1/profiles.proto @@ -1,4 +1,4 @@ -// Copyright 2019, OpenTelemetry Authors +// Copyright 2023, OpenTelemetry Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. From a87c23641d10a8d255558c22448428d700e5cb6c Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Thu, 6 Jul 2023 09:06:12 -0700 Subject: [PATCH 11/75] uint64 -> uint32 for indices --- .../profiles/v1/alternatives/normalized/normalized.proto | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto b/opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto index 2746db3ae..e40f3f961 100644 --- a/opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto +++ b/opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto @@ -26,9 +26,9 @@ message AttributeSet { } message Sample { - repeated uint64 location_indices = 1; - repeated uint64 link_indices = 2; - repeated uint64 attribute_set_indices = 3; + repeated uint32 location_indices = 1; + repeated uint32 link_indices = 2; + repeated uint32 attribute_set_indices = 3; // this one is repeated because there can be multiple profile kinds in one profile. Typical example is memory profiles in go that contain: // * alloc_objects From 980fc893bf5c0ebfe8bc9e77af2b36971f6ba581 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Thu, 6 Jul 2023 09:08:08 -0700 Subject: [PATCH 12/75] adds attributes to Location and Mapping --- .../profiles/v1/alternatives/normalized/normalized.proto | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto b/opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto index e40f3f961..675cff548 100644 --- a/opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto +++ b/opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto @@ -96,6 +96,9 @@ message Mapping { } SymbolicInfo symbolic_info = 6; + + // TODO(@petethepig): I wonder if we need something a little more specialized here + repeated uint32 attribute_set_indices = 7; } // borrowed from pprof proto @@ -125,6 +128,9 @@ message Location { // symbols. This field must be recomputed when the symbolization state of the // profile changes. bool is_folded = 4; + + // TODO(@petethepig): I wonder if we need something a little more specialized here + repeated uint32 attribute_set_indices = 5; } // borrowed from pprof proto From 6b98f0ca3b2205eed6ea5459b01ecd3ac5de68ac Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Thu, 6 Jul 2023 20:38:42 -0700 Subject: [PATCH 13/75] move stacktraces to a separate message --- .../v1/alternatives/denormalized/denormalized.proto | 6 +++++- .../profiles/v1/alternatives/normalized/normalized.proto | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/opentelemetry/proto/profiles/v1/alternatives/denormalized/denormalized.proto b/opentelemetry/proto/profiles/v1/alternatives/denormalized/denormalized.proto index 605c913e3..b2d10abad 100644 --- a/opentelemetry/proto/profiles/v1/alternatives/denormalized/denormalized.proto +++ b/opentelemetry/proto/profiles/v1/alternatives/denormalized/denormalized.proto @@ -17,8 +17,12 @@ message Link { bytes span_id = 2; } -message Sample { +message StackTrace { repeated Location locations = 1; +} + +message Sample { + repeated StackTrace stacktrace = 1; repeated Link links = 2; repeated opentelemetry.proto.common.v1.KeyValue attributes = 3; uint32 dropped_attributes_count = 4; diff --git a/opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto b/opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto index 675cff548..a840a092e 100644 --- a/opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto +++ b/opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto @@ -25,8 +25,12 @@ message AttributeSet { uint32 dropped_attributes_count = 2; } -message Sample { +message StackTrace { repeated uint32 location_indices = 1; +} + +message Sample { + StackTrace stacktrace = 1; repeated uint32 link_indices = 2; repeated uint32 attribute_set_indices = 3; From efc572cff93ef943d7f949e7fd361d8d10da4b01 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Thu, 6 Jul 2023 22:41:48 -0700 Subject: [PATCH 14/75] follow up to prev commit --- .../denormalized/denormalized.proto | 6 +++--- .../alternatives/normalized/normalized.proto | 19 ++++++++++--------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/opentelemetry/proto/profiles/v1/alternatives/denormalized/denormalized.proto b/opentelemetry/proto/profiles/v1/alternatives/denormalized/denormalized.proto index b2d10abad..471aca88d 100644 --- a/opentelemetry/proto/profiles/v1/alternatives/denormalized/denormalized.proto +++ b/opentelemetry/proto/profiles/v1/alternatives/denormalized/denormalized.proto @@ -17,12 +17,12 @@ message Link { bytes span_id = 2; } -message StackTrace { +message Stacktrace { repeated Location locations = 1; } message Sample { - repeated StackTrace stacktrace = 1; + Stacktrace stacktrace = 1; repeated Link links = 2; repeated opentelemetry.proto.common.v1.KeyValue attributes = 3; uint32 dropped_attributes_count = 4; @@ -131,5 +131,5 @@ message Function { // Source file containing the function. string filename = 3; // Line number in source file. - string start_line = 4; + uint32 start_line = 4; } diff --git a/opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto b/opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto index a840a092e..6688997b6 100644 --- a/opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto +++ b/opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto @@ -25,12 +25,12 @@ message AttributeSet { uint32 dropped_attributes_count = 2; } -message StackTrace { +message Stacktrace { repeated uint32 location_indices = 1; } message Sample { - StackTrace stacktrace = 1; + uint32 stacktrace_index = 1; repeated uint32 link_indices = 2; repeated uint32 attribute_set_indices = 3; @@ -63,15 +63,16 @@ message SampleType { message Profile { repeated SampleType sample_types = 1; - repeated Sample samples = 2; - repeated Mapping mappings = 3; - repeated Location locations = 4; - repeated Function functions = 5; - repeated Link links = 6; - repeated AttributeSet attribute_sets = 7; + repeated Stacktrace stacktraces = 2; + repeated Sample samples = 3; + repeated Mapping mappings = 4; + repeated Location locations = 5; + repeated Function functions = 6; + repeated Link links = 7; + repeated AttributeSet attribute_sets = 8; // 0-th element must be an empty string ("") - repeated string string_table = 8; + repeated string string_table = 9; } From 8fe43e9b6a04ecdebf7bb3352927846e2796b82b Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Thu, 6 Jul 2023 22:42:00 -0700 Subject: [PATCH 15/75] adds another alternative (arrays) --- .../v1/alternatives/arrays/arrays.proto | 169 ++++++++++++++++++ .../proto/profiles/v1/profiles.proto | 4 +- 2 files changed, 172 insertions(+), 1 deletion(-) create mode 100644 opentelemetry/proto/profiles/v1/alternatives/arrays/arrays.proto diff --git a/opentelemetry/proto/profiles/v1/alternatives/arrays/arrays.proto b/opentelemetry/proto/profiles/v1/alternatives/arrays/arrays.proto new file mode 100644 index 000000000..d66107e37 --- /dev/null +++ b/opentelemetry/proto/profiles/v1/alternatives/arrays/arrays.proto @@ -0,0 +1,169 @@ +syntax = "proto3"; + +package opentelemetry.proto.profiles.v1.alternatives.arrays; + +import "opentelemetry/proto/common/v1/common.proto"; +import "opentelemetry/proto/resource/v1/resource.proto"; + +option go_package = "go.opentelemetry.io/proto/otlp/profiles/v1/arrays"; + + +// A pointer from a profile to a trace span. +message Link { + // A unique identifier of a trace that this linked span is part of. The ID is a + // 16-byte array. + bytes trace_id = 1; + // A unique identifier for the linked span. The ID is an 8-byte array. + bytes span_id = 2; +} + +// AttributeSet represents a set of attributes. It is referenced from Samples +// and not embedded because it is common for multiple samples to have the same +// set of attributes +message AttributeSet { + repeated opentelemetry.proto.common.v1.KeyValue attributes = 1; + uint32 dropped_attributes_count = 2; +} + +message Stacktrace { + repeated uint32 location_indices = 1; +} + +// message Sample { +// uint32 stacktrace_index = 1; +// repeated uint32 link_indices = 2; +// repeated uint32 attribute_set_indices = 3; +// repeated int64 values = 4; + +// // optional +// fixed64 timestamp_unix_nano = 5; +// } + +message SampleType { + // borrowed from metrics proto + enum AggregationTemporality { + AGGREGATION_TEMPORALITY_UNSPECIFIED = 0; + AGGREGATION_TEMPORALITY_DELTA = 1; + AGGREGATION_TEMPORALITY_CUMULATIVE = 2; + } + + AggregationTemporality aggregation_temporality = 1; + uint64 sample_size = 2; + + // CPU / memory /etc + uint32 type_index = 3; // Index into string table. + uint32 unit_index = 4; // Index into string table. +} + +message ProfileType { + repeated uint32 stacktrace_indices = 10; + repeated uint32 link_indices = 11; + repeated uint32 attribute_set_indices = 12; + repeated int64 values = 13; + repeated uint64 timestamps = 14; +} + +message Profile { + repeated SampleType sample_types = 1; + repeated Stacktrace stacktraces = 2; + repeated Mapping mappings = 4; + repeated Location locations = 5; + repeated Function functions = 6; + repeated Link links = 7; + repeated AttributeSet attribute_sets = 8; + + // 0-th element must be an empty string ("") + repeated string string_table = 9; + + repeated ProfileType profile_types = 10; + + // stacktrace_index + // link_indices + // attribute_set_indices + // values + // timestamp_unix_nano +} + + +// borrowed from pprof proto +message Mapping { + // Address at which the binary (or DLL) is loaded into memory. + uint64 memory_start = 1; + // The limit of the address range occupied by this mapping. + uint64 memory_limit = 2; + // Offset in the binary that corresponds to the first mapped address. + uint64 file_offset = 3; + // The object this entry is loaded from. This can be a filename on + // disk for the main binary and shared libraries, or virtual + // abstractions like "[vdso]". + uint32 filename_index = 4; // Index into string table + // A string that uniquely identifies a particular program version + // with high probability. E.g., for binaries generated by GNU tools, + // it could be the contents of the .note.gnu.build-id field. + uint32 build_id_index = 5; // Index into string table + + enum SymbolicInfo { + SYMBOLIC_INFO_UNSPECIFIED = 0; + SYMBOLIC_INFO_FULL = 1; + SYMBOLIC_INFO_FUNCTIONS_ONLY = 2; + SYMBOLIC_INFO_NO_INLINE_FRAMES = 3; + } + + SymbolicInfo symbolic_info = 6; + + // TODO(@petethepig): I wonder if we need something a little more specialized here + repeated uint32 attribute_set_indices = 7; +} + +// borrowed from pprof proto +// Describes function and line table debug information. +message Location { + // The id of the corresponding profile.Mapping for this location. + // It can be unset if the mapping is unknown or not applicable for + // this profile type. + uint32 mapping_index = 1; + // The instruction address for this location, if available. It + // should be within [Mapping.memory_start...Mapping.memory_limit] + // for the corresponding mapping. A non-leaf address may be in the + // middle of a call instruction. It is up to display tools to find + // the beginning of the instruction if necessary. + uint64 address = 2; + // Multiple line indicates this location has inlined functions, + // where the last entry represents the caller into which the + // preceding entries were inlined. + // + // E.g., if memcpy() is inlined into printf: + // line[0].function_name == "memcpy" + // line[1].function_name == "printf" + repeated Line line = 3; + // Provides an indication that multiple symbols map to this location's + // address, for example due to identical code folding by the linker. In that + // case the line information above represents one of the multiple + // symbols. This field must be recomputed when the symbolization state of the + // profile changes. + bool is_folded = 4; + + // TODO(@petethepig): I wonder if we need something a little more specialized here + repeated uint32 attribute_set_indices = 5; +} + +// borrowed from pprof proto +message Line { + // The id of the corresponding profile.Function for this line. + uint32 function_index = 1; + // Line number in source code. + uint32 line = 2; +} + +// borrowed from pprof proto +message Function { + // Name of the function, in human-readable form if available. + uint32 name_index = 1; // Index into string table + // Name of the function, as identified by the system. + // For instance, it can be a C++ mangled name. + uint32 system_name_index = 2; // Index into string table + // Source file containing the function. + uint32 filename_index = 3; // Index into string table + // Line number in source file. + uint32 start_line = 4; +} diff --git a/opentelemetry/proto/profiles/v1/profiles.proto b/opentelemetry/proto/profiles/v1/profiles.proto index b2d4b83a3..2f902d5ec 100644 --- a/opentelemetry/proto/profiles/v1/profiles.proto +++ b/opentelemetry/proto/profiles/v1/profiles.proto @@ -24,6 +24,7 @@ import "opentelemetry/proto/resource/v1/resource.proto"; import "opentelemetry/proto/profiles/v1/alternatives/pprof/pprof.proto"; import "opentelemetry/proto/profiles/v1/alternatives/denormalized/denormalized.proto"; import "opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto"; +import "opentelemetry/proto/profiles/v1/alternatives/arrays/arrays.proto"; option csharp_namespace = "OpenTelemetry.Proto.Profiles.V1"; option java_multiple_files = true; @@ -128,6 +129,7 @@ message Profile { oneof alternative_profile { opentelemetry.proto.profiles.v1.alternatives.pprof.Profile pprof = 8; opentelemetry.proto.profiles.v1.alternatives.normalized.Profile normalized = 9; - opentelemetry.proto.profiles.v1.alternatives.denormalized.Profile denormalized = 10; + opentelemetry.proto.profiles.v1.alternatives.arrays.Profile arrays = 10; + opentelemetry.proto.profiles.v1.alternatives.denormalized.Profile denormalized = 11; } } From b0166761b45030d12f00e2e2723d1bd6d0ff269c Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Tue, 18 Jul 2023 13:08:35 -0700 Subject: [PATCH 16/75] adds a decision log --- .../proto/profiles/v1/decision-log.md | 116 ++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 opentelemetry/proto/profiles/v1/decision-log.md diff --git a/opentelemetry/proto/profiles/v1/decision-log.md b/opentelemetry/proto/profiles/v1/decision-log.md new file mode 100644 index 000000000..310b82862 --- /dev/null +++ b/opentelemetry/proto/profiles/v1/decision-log.md @@ -0,0 +1,116 @@ +# Decision #1 +Spec must include an optional field for the original payload (e.g original JFR/pprof profile) + +### Date +Jun 01 2023 + +### Context +Some formats (JFR) have advanced functinality that is too hard to generalize (e.g in JFR you can include custom events in your profiles) + +### Alternatives Considered +* try to generalize it anyway, so that OTEL format is a subset of JFR +* discard such data + +### Reasoning +* generalizing it is too hard, JFR allows for pretty much any kind of message +* discarding is not good, some percentage of users depend on custom JFR events + +### Details +[SIG Meeting Notes from June 1 2023](https://docs.google.com/document/d/19UqPPPlGE83N37MhS93uRlxsP1_wGxQ33Qv6CDHaEp0/edit?usp=sharing) + + + + + +# Decision #2 +Use normalized representation of profiling data + +### Date +Jun 29 2023 + +### Context +Profiling data is different than other signals in a way that it has heavy structures with many links between them. + +### Alternatives Considered +Use denormalized represenation of profiling data + +### Reasoning +Because profiling data has many heavy structures with many links between them, denormalized representation takes 10x more space compared to normalized version where each object references other objects by index + +### Details +[Github Conversation](https://github.com/open-telemetry/opentelemetry-proto/pull/488#discussion_r1246163710) +[Github Conversation](https://github.com/open-telemetry/opentelemetry-proto/pull/488#pullrequestreview-1505265763) + + + + + +# Decision #3 +When referring to other structs use array indexes for references instead of ids + +### Date +Jun 29 2023 + +### Context +in pprof references happen using ids, e.g {function_id=123}. this leads to confusion, as these ids almost always match array indexes. e.g "function" with id=1 will usually be in functions array at index 1. + +### Alternatives Considered +Using ids instead of indexes + +### Reasoning +* reduces confusion +* reduces serialized size +* improves performance since array lookup is faster than search (O(1) vs O(n) + +### Details +[Github Conversation](https://github.com/open-telemetry/opentelemetry-proto/pull/488#discussion_r1248116971) + + + + + +# Decision #4 +Spec must include an optional Mapping struct from pprof + +### Date +Jun 29 2023 + +### Context +pprof includes Mapping struct that describes relationships between functions and where those functions are located in the binary. This struct is rarely used in client implementations + +### Alternatives Considered +Not include it to simplify spec + +### Reasoning +It is useful when symbol information is supplied separately, e.g very common in linux + +### Details +[SIG Meeting Notes from Jun 29 2023](https://docs.google.com/document/d/19UqPPPlGE83N37MhS93uRlxsP1_wGxQ33Qv6CDHaEp0/edit?usp=sharing) + + + + + +# Decision #5 +Internal references should be uint32 + +### Date +Jun 29 2023 + +### Context +there's many internal reference fields, e.g: +* location_id +* function_id +if they are > 32 bits then the payload is going to be too huge anyway + +### Alternatives Considered +* use uint64 + + +### Reasoning +* uint32 is big enough +* but it's better than uint64 because it will reduce struct size in go +* this will reduct collector memory footprint + +### Details +[Github Conversation](https://github.com/open-telemetry/opentelemetry-proto/pull/488/files#r1246155907) From 8cd1e60d9857e66e08c094356583c59222b95b7b Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Tue, 25 Jul 2023 22:44:36 -0700 Subject: [PATCH 17/75] included recent suggestions --- .../v1/alternatives/arrays/arrays.proto | 16 ++++------------ .../denormalized/denormalized.proto | 6 ------ .../v1/alternatives/normalized/normalized.proto | 17 ++++------------- 3 files changed, 8 insertions(+), 31 deletions(-) diff --git a/opentelemetry/proto/profiles/v1/alternatives/arrays/arrays.proto b/opentelemetry/proto/profiles/v1/alternatives/arrays/arrays.proto index d66107e37..571d0644e 100644 --- a/opentelemetry/proto/profiles/v1/alternatives/arrays/arrays.proto +++ b/opentelemetry/proto/profiles/v1/alternatives/arrays/arrays.proto @@ -102,14 +102,12 @@ message Mapping { // it could be the contents of the .note.gnu.build-id field. uint32 build_id_index = 5; // Index into string table - enum SymbolicInfo { - SYMBOLIC_INFO_UNSPECIFIED = 0; - SYMBOLIC_INFO_FULL = 1; - SYMBOLIC_INFO_FUNCTIONS_ONLY = 2; - SYMBOLIC_INFO_NO_INLINE_FRAMES = 3; + enum SymbolFidelity { + SYMBOL_FIDELITY_UNSPECIFIED = 0; + SYMBOL_FIDELITY_FULL = 1; } - SymbolicInfo symbolic_info = 6; + SymbolFidelity symbolic_info = 6; // TODO(@petethepig): I wonder if we need something a little more specialized here repeated uint32 attribute_set_indices = 7; @@ -136,12 +134,6 @@ message Location { // line[0].function_name == "memcpy" // line[1].function_name == "printf" repeated Line line = 3; - // Provides an indication that multiple symbols map to this location's - // address, for example due to identical code folding by the linker. In that - // case the line information above represents one of the multiple - // symbols. This field must be recomputed when the symbolization state of the - // profile changes. - bool is_folded = 4; // TODO(@petethepig): I wonder if we need something a little more specialized here repeated uint32 attribute_set_indices = 5; diff --git a/opentelemetry/proto/profiles/v1/alternatives/denormalized/denormalized.proto b/opentelemetry/proto/profiles/v1/alternatives/denormalized/denormalized.proto index 471aca88d..8f0576766 100644 --- a/opentelemetry/proto/profiles/v1/alternatives/denormalized/denormalized.proto +++ b/opentelemetry/proto/profiles/v1/alternatives/denormalized/denormalized.proto @@ -105,12 +105,6 @@ message Location { // line[0].function_name == "memcpy" // line[1].function_name == "printf" repeated Line line = 3; - // Provides an indication that multiple symbols map to this location's - // address, for example due to identical code folding by the linker. In that - // case the line information above represents one of the multiple - // symbols. This field must be recomputed when the symbolization state of the - // profile changes. - bool is_folded = 4; } // borrowed from pprof proto diff --git a/opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto b/opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto index 6688997b6..1e1508bab 100644 --- a/opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto +++ b/opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto @@ -93,14 +93,12 @@ message Mapping { // it could be the contents of the .note.gnu.build-id field. uint32 build_id_index = 5; // Index into string table - enum SymbolicInfo { - SYMBOLIC_INFO_UNSPECIFIED = 0; - SYMBOLIC_INFO_FULL = 1; - SYMBOLIC_INFO_FUNCTIONS_ONLY = 2; - SYMBOLIC_INFO_NO_INLINE_FRAMES = 3; + enum SymbolFidelity { + SYMBOL_FIDELITY_UNSPECIFIED = 0; + SYMBOL_FIDELITY_FULL = 1; } - SymbolicInfo symbolic_info = 6; + SymbolFidelity symbolic_info = 6; // TODO(@petethepig): I wonder if we need something a little more specialized here repeated uint32 attribute_set_indices = 7; @@ -127,13 +125,6 @@ message Location { // line[0].function_name == "memcpy" // line[1].function_name == "printf" repeated Line line = 3; - // Provides an indication that multiple symbols map to this location's - // address, for example due to identical code folding by the linker. In that - // case the line information above represents one of the multiple - // symbols. This field must be recomputed when the symbolization state of the - // profile changes. - bool is_folded = 4; - // TODO(@petethepig): I wonder if we need something a little more specialized here repeated uint32 attribute_set_indices = 5; } From 4fd6976aed2b69fee25c2e1ecad19bea9ec3dd6c Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Wed, 23 Aug 2023 21:20:16 -0700 Subject: [PATCH 18/75] more changes to the spec --- .../v1/alternatives/arrays/arrays.proto | 161 ------------- .../denormalized/denormalized.proto | 129 ---------- .../alternatives/normalized/normalized.proto | 151 ------------ .../v1/alternatives/pprof/pprof.proto | 221 ------------------ .../proto/profiles/v1/decision-log.md | 52 +++++ .../proto/profiles/v1/profiles.proto | 169 +++++++++++++- 6 files changed, 210 insertions(+), 673 deletions(-) delete mode 100644 opentelemetry/proto/profiles/v1/alternatives/arrays/arrays.proto delete mode 100644 opentelemetry/proto/profiles/v1/alternatives/denormalized/denormalized.proto delete mode 100644 opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto delete mode 100644 opentelemetry/proto/profiles/v1/alternatives/pprof/pprof.proto diff --git a/opentelemetry/proto/profiles/v1/alternatives/arrays/arrays.proto b/opentelemetry/proto/profiles/v1/alternatives/arrays/arrays.proto deleted file mode 100644 index 571d0644e..000000000 --- a/opentelemetry/proto/profiles/v1/alternatives/arrays/arrays.proto +++ /dev/null @@ -1,161 +0,0 @@ -syntax = "proto3"; - -package opentelemetry.proto.profiles.v1.alternatives.arrays; - -import "opentelemetry/proto/common/v1/common.proto"; -import "opentelemetry/proto/resource/v1/resource.proto"; - -option go_package = "go.opentelemetry.io/proto/otlp/profiles/v1/arrays"; - - -// A pointer from a profile to a trace span. -message Link { - // A unique identifier of a trace that this linked span is part of. The ID is a - // 16-byte array. - bytes trace_id = 1; - // A unique identifier for the linked span. The ID is an 8-byte array. - bytes span_id = 2; -} - -// AttributeSet represents a set of attributes. It is referenced from Samples -// and not embedded because it is common for multiple samples to have the same -// set of attributes -message AttributeSet { - repeated opentelemetry.proto.common.v1.KeyValue attributes = 1; - uint32 dropped_attributes_count = 2; -} - -message Stacktrace { - repeated uint32 location_indices = 1; -} - -// message Sample { -// uint32 stacktrace_index = 1; -// repeated uint32 link_indices = 2; -// repeated uint32 attribute_set_indices = 3; -// repeated int64 values = 4; - -// // optional -// fixed64 timestamp_unix_nano = 5; -// } - -message SampleType { - // borrowed from metrics proto - enum AggregationTemporality { - AGGREGATION_TEMPORALITY_UNSPECIFIED = 0; - AGGREGATION_TEMPORALITY_DELTA = 1; - AGGREGATION_TEMPORALITY_CUMULATIVE = 2; - } - - AggregationTemporality aggregation_temporality = 1; - uint64 sample_size = 2; - - // CPU / memory /etc - uint32 type_index = 3; // Index into string table. - uint32 unit_index = 4; // Index into string table. -} - -message ProfileType { - repeated uint32 stacktrace_indices = 10; - repeated uint32 link_indices = 11; - repeated uint32 attribute_set_indices = 12; - repeated int64 values = 13; - repeated uint64 timestamps = 14; -} - -message Profile { - repeated SampleType sample_types = 1; - repeated Stacktrace stacktraces = 2; - repeated Mapping mappings = 4; - repeated Location locations = 5; - repeated Function functions = 6; - repeated Link links = 7; - repeated AttributeSet attribute_sets = 8; - - // 0-th element must be an empty string ("") - repeated string string_table = 9; - - repeated ProfileType profile_types = 10; - - // stacktrace_index - // link_indices - // attribute_set_indices - // values - // timestamp_unix_nano -} - - -// borrowed from pprof proto -message Mapping { - // Address at which the binary (or DLL) is loaded into memory. - uint64 memory_start = 1; - // The limit of the address range occupied by this mapping. - uint64 memory_limit = 2; - // Offset in the binary that corresponds to the first mapped address. - uint64 file_offset = 3; - // The object this entry is loaded from. This can be a filename on - // disk for the main binary and shared libraries, or virtual - // abstractions like "[vdso]". - uint32 filename_index = 4; // Index into string table - // A string that uniquely identifies a particular program version - // with high probability. E.g., for binaries generated by GNU tools, - // it could be the contents of the .note.gnu.build-id field. - uint32 build_id_index = 5; // Index into string table - - enum SymbolFidelity { - SYMBOL_FIDELITY_UNSPECIFIED = 0; - SYMBOL_FIDELITY_FULL = 1; - } - - SymbolFidelity symbolic_info = 6; - - // TODO(@petethepig): I wonder if we need something a little more specialized here - repeated uint32 attribute_set_indices = 7; -} - -// borrowed from pprof proto -// Describes function and line table debug information. -message Location { - // The id of the corresponding profile.Mapping for this location. - // It can be unset if the mapping is unknown or not applicable for - // this profile type. - uint32 mapping_index = 1; - // The instruction address for this location, if available. It - // should be within [Mapping.memory_start...Mapping.memory_limit] - // for the corresponding mapping. A non-leaf address may be in the - // middle of a call instruction. It is up to display tools to find - // the beginning of the instruction if necessary. - uint64 address = 2; - // Multiple line indicates this location has inlined functions, - // where the last entry represents the caller into which the - // preceding entries were inlined. - // - // E.g., if memcpy() is inlined into printf: - // line[0].function_name == "memcpy" - // line[1].function_name == "printf" - repeated Line line = 3; - - // TODO(@petethepig): I wonder if we need something a little more specialized here - repeated uint32 attribute_set_indices = 5; -} - -// borrowed from pprof proto -message Line { - // The id of the corresponding profile.Function for this line. - uint32 function_index = 1; - // Line number in source code. - uint32 line = 2; -} - -// borrowed from pprof proto -message Function { - // Name of the function, in human-readable form if available. - uint32 name_index = 1; // Index into string table - // Name of the function, as identified by the system. - // For instance, it can be a C++ mangled name. - uint32 system_name_index = 2; // Index into string table - // Source file containing the function. - uint32 filename_index = 3; // Index into string table - // Line number in source file. - uint32 start_line = 4; -} diff --git a/opentelemetry/proto/profiles/v1/alternatives/denormalized/denormalized.proto b/opentelemetry/proto/profiles/v1/alternatives/denormalized/denormalized.proto deleted file mode 100644 index 8f0576766..000000000 --- a/opentelemetry/proto/profiles/v1/alternatives/denormalized/denormalized.proto +++ /dev/null @@ -1,129 +0,0 @@ -syntax = "proto3"; - -package opentelemetry.proto.profiles.v1.alternatives.denormalized; - -import "opentelemetry/proto/common/v1/common.proto"; -import "opentelemetry/proto/resource/v1/resource.proto"; - -option go_package = "go.opentelemetry.io/proto/otlp/profiles/v1/alternatives/denormalized"; - - -// A pointer from a profile to a trace span. -message Link { - // A unique identifier of a trace that this linked span is part of. The ID is a - // 16-byte array. - bytes trace_id = 1; - // A unique identifier for the linked span. The ID is an 8-byte array. - bytes span_id = 2; -} - -message Stacktrace { - repeated Location locations = 1; -} - -message Sample { - Stacktrace stacktrace = 1; - repeated Link links = 2; - repeated opentelemetry.proto.common.v1.KeyValue attributes = 3; - uint32 dropped_attributes_count = 4; - - // this one is repeated because there can be multiple profile kinds in one profile. Typical example is memory profiles in go that contain: - // * alloc_objects - // * alloc_bytes - // * inuse_objects - // * inuse_bytes - repeated fixed64 values = 5; - - // optional - fixed64 timestamp_unix_nano = 6; -} - -// borrowed from metrics proto -enum AggregationTemporality { - AGGREGATION_TEMPORALITY_UNSPECIFIED = 0; - AGGREGATION_TEMPORALITY_DELTA = 1; - AGGREGATION_TEMPORALITY_CUMULATIVE = 2; -} - -message SampleType { - AggregationTemporality aggregation_temporality = 1; - uint64 sample_size = 2; - - // CPU / memory /etc - string type = 3; - string unit = 4; -} - -message Profile { - repeated SampleType sample_types = 1; - repeated Sample samples = 2; -} - -// borrowed from pprof proto -message Mapping { - // Address at which the binary (or DLL) is loaded into memory. - uint64 memory_start = 1; - // The limit of the address range occupied by this mapping. - uint64 memory_limit = 2; - // Offset in the binary that corresponds to the first mapped address. - uint64 file_offset = 3; - // The object this entry is loaded from. This can be a filename on - // disk for the main binary and shared libraries, or virtual - // abstractions like "[vdso]". - string filename = 4; - // A string that uniquely identifies a particular program version - // with high probability. E.g., for binaries generated by GNU tools, - // it could be the contents of the .note.gnu.build-id field. - string build_id = 5; - - // The following fields indicate the resolution of symbolic info. - bool has_functions = 6; - bool has_filenames = 7; - bool has_line_numbers = 8; - bool has_inline_frames = 9; -} - -// borrowed from pprof proto -// Describes function and line table debug information. -message Location { - // profile.Mapping for this location. - // It can be unset if the mapping is unknown or not applicable for - // this profile type. - Mapping mapping = 1; - - // The instruction address for this location, if available. It - // should be within [Mapping.memory_start...Mapping.memory_limit] - // for the corresponding mapping. A non-leaf address may be in the - // middle of a call instruction. It is up to display tools to find - // the beginning of the instruction if necessary. - uint64 address = 2; - // Multiple line indicates this location has inlined functions, - // where the last entry represents the caller into which the - // preceding entries were inlined. - // - // E.g., if memcpy() is inlined into printf: - // line[0].function_name == "memcpy" - // line[1].function_name == "printf" - repeated Line line = 3; -} - -// borrowed from pprof proto -message Line { - // Function for this line. - Function function = 1; - // Line number in source code. - int64 line = 2; -} - -// borrowed from pprof proto -message Function { - // Name of the function, in human-readable form if available. - string name = 1; - // Name of the function, as identified by the system. - // For instance, it can be a C++ mangled name. - string system_name = 2; - // Source file containing the function. - string filename = 3; - // Line number in source file. - uint32 start_line = 4; -} diff --git a/opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto b/opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto deleted file mode 100644 index 1e1508bab..000000000 --- a/opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto +++ /dev/null @@ -1,151 +0,0 @@ -syntax = "proto3"; - -package opentelemetry.proto.profiles.v1.alternatives.normalized; - -import "opentelemetry/proto/common/v1/common.proto"; -import "opentelemetry/proto/resource/v1/resource.proto"; - -option go_package = "go.opentelemetry.io/proto/otlp/profiles/v1/normalized"; - - -// A pointer from a profile to a trace span. -message Link { - // A unique identifier of a trace that this linked span is part of. The ID is a - // 16-byte array. - bytes trace_id = 1; - // A unique identifier for the linked span. The ID is an 8-byte array. - bytes span_id = 2; -} - -// AttributeSet represents a set of attributes. It is referenced from Samples -// and not embedded because it is common for multiple samples to have the same -// set of attributes -message AttributeSet { - repeated opentelemetry.proto.common.v1.KeyValue attributes = 1; - uint32 dropped_attributes_count = 2; -} - -message Stacktrace { - repeated uint32 location_indices = 1; -} - -message Sample { - uint32 stacktrace_index = 1; - repeated uint32 link_indices = 2; - repeated uint32 attribute_set_indices = 3; - - // this one is repeated because there can be multiple profile kinds in one profile. Typical example is memory profiles in go that contain: - // * alloc_objects - // * alloc_bytes - // * inuse_objects - // * inuse_bytes - repeated int64 values = 4; - - // optional - fixed64 timestamp_unix_nano = 5; -} - -message SampleType { - // borrowed from metrics proto - enum AggregationTemporality { - AGGREGATION_TEMPORALITY_UNSPECIFIED = 0; - AGGREGATION_TEMPORALITY_DELTA = 1; - AGGREGATION_TEMPORALITY_CUMULATIVE = 2; - } - - AggregationTemporality aggregation_temporality = 1; - uint64 sample_size = 2; - - // CPU / memory /etc - uint32 type_index = 3; // Index into string table. - uint32 unit_index = 4; // Index into string table. -} - -message Profile { - repeated SampleType sample_types = 1; - repeated Stacktrace stacktraces = 2; - repeated Sample samples = 3; - repeated Mapping mappings = 4; - repeated Location locations = 5; - repeated Function functions = 6; - repeated Link links = 7; - repeated AttributeSet attribute_sets = 8; - - // 0-th element must be an empty string ("") - repeated string string_table = 9; -} - - -// borrowed from pprof proto -message Mapping { - // Address at which the binary (or DLL) is loaded into memory. - uint64 memory_start = 1; - // The limit of the address range occupied by this mapping. - uint64 memory_limit = 2; - // Offset in the binary that corresponds to the first mapped address. - uint64 file_offset = 3; - // The object this entry is loaded from. This can be a filename on - // disk for the main binary and shared libraries, or virtual - // abstractions like "[vdso]". - uint32 filename_index = 4; // Index into string table - // A string that uniquely identifies a particular program version - // with high probability. E.g., for binaries generated by GNU tools, - // it could be the contents of the .note.gnu.build-id field. - uint32 build_id_index = 5; // Index into string table - - enum SymbolFidelity { - SYMBOL_FIDELITY_UNSPECIFIED = 0; - SYMBOL_FIDELITY_FULL = 1; - } - - SymbolFidelity symbolic_info = 6; - - // TODO(@petethepig): I wonder if we need something a little more specialized here - repeated uint32 attribute_set_indices = 7; -} - -// borrowed from pprof proto -// Describes function and line table debug information. -message Location { - // The id of the corresponding profile.Mapping for this location. - // It can be unset if the mapping is unknown or not applicable for - // this profile type. - uint32 mapping_index = 1; - // The instruction address for this location, if available. It - // should be within [Mapping.memory_start...Mapping.memory_limit] - // for the corresponding mapping. A non-leaf address may be in the - // middle of a call instruction. It is up to display tools to find - // the beginning of the instruction if necessary. - uint64 address = 2; - // Multiple line indicates this location has inlined functions, - // where the last entry represents the caller into which the - // preceding entries were inlined. - // - // E.g., if memcpy() is inlined into printf: - // line[0].function_name == "memcpy" - // line[1].function_name == "printf" - repeated Line line = 3; - // TODO(@petethepig): I wonder if we need something a little more specialized here - repeated uint32 attribute_set_indices = 5; -} - -// borrowed from pprof proto -message Line { - // The id of the corresponding profile.Function for this line. - uint32 function_index = 1; - // Line number in source code. - uint32 line = 2; -} - -// borrowed from pprof proto -message Function { - // Name of the function, in human-readable form if available. - uint32 name_index = 1; // Index into string table - // Name of the function, as identified by the system. - // For instance, it can be a C++ mangled name. - uint32 system_name_index = 2; // Index into string table - // Source file containing the function. - uint32 filename_index = 3; // Index into string table - // Line number in source file. - uint32 start_line = 4; -} diff --git a/opentelemetry/proto/profiles/v1/alternatives/pprof/pprof.proto b/opentelemetry/proto/profiles/v1/alternatives/pprof/pprof.proto deleted file mode 100644 index ec55bc207..000000000 --- a/opentelemetry/proto/profiles/v1/alternatives/pprof/pprof.proto +++ /dev/null @@ -1,221 +0,0 @@ -// Copyright 2016 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Profile is a common stacktrace profile format. -// -// Measurements represented with this format should follow the -// following conventions: -// -// - Consumers should treat unset optional fields as if they had been -// set with their default value. -// -// - When possible, measurements should be stored in "unsampled" form -// that is most useful to humans. There should be enough -// information present to determine the original sampled values. -// -// - On-disk, the serialized proto must be gzip-compressed. -// -// - The profile is represented as a set of samples, where each sample -// references a sequence of locations, and where each location belongs -// to a mapping. -// - There is a N->1 relationship from sample.location_id entries to -// locations. For every sample.location_id entry there must be a -// unique Location with that id. -// - There is an optional N->1 relationship from locations to -// mappings. For every nonzero Location.mapping_id there must be a -// unique Mapping with that id. - -syntax = "proto3"; - -package opentelemetry.proto.profiles.v1.alternatives.pprof; - -option go_package = "go.opentelemetry.io/proto/otlp/profiles/v1/alternatives/pprof"; - -message Profile { - // A description of the samples associated with each Sample.value. - // For a cpu profile this might be: - // [["cpu","nanoseconds"]] or [["wall","seconds"]] or [["syscall","count"]] - // For a heap profile, this might be: - // [["allocations","count"], ["space","bytes"]], - // If one of the values represents the number of events represented - // by the sample, by convention it should be at index 0 and use - // sample_type.unit == "count". - repeated ValueType sample_type = 1; - // The set of samples recorded in this profile. - repeated Sample sample = 2; - // Mapping from address ranges to the image/binary/library mapped - // into that address range. mapping[0] will be the main binary. - repeated Mapping mapping = 3; - // Locations referenced by samples. - repeated Location location = 4; - // Functions referenced by locations. - repeated Function function = 5; - // A common table for strings referenced by various messages. - // string_table[0] must always be "". - repeated string string_table = 6; - // frames with Function.function_name fully matching the following - // regexp will be dropped from the samples, along with their successors. - int64 drop_frames = 7; // Index into string table. - // frames with Function.function_name fully matching the following - // regexp will be kept, even if it matches drop_frames. - int64 keep_frames = 8; // Index into string table. - - // The following fields are informational, do not affect - // interpretation of results. - - // Time of collection (UTC) represented as nanoseconds past the epoch. - int64 time_nanos = 9; - // Duration of the profile, if a duration makes sense. - int64 duration_nanos = 10; - // The kind of events between sampled occurrences. - // e.g [ "cpu","cycles" ] or [ "heap","bytes" ] - ValueType period_type = 11; - // The number of events between sampled occurrences. - int64 period = 12; - // Free-form text associated with the profile. The text is displayed as is - // to the user by the tools that read profiles (e.g. by pprof). This field - // should not be used to store any machine-readable information, it is only - // for human-friendly content. The profile must stay functional if this field - // is cleaned. - repeated int64 comment = 13; // Indices into string table. - // Index into the string table of the type of the preferred sample - // value. If unset, clients should default to the last sample value. - int64 default_sample_type = 14; -} - -// ValueType describes the semantics and measurement units of a value. -message ValueType { - int64 type = 1; // Index into string table. - int64 unit = 2; // Index into string table. -} - -// Each Sample records values encountered in some program -// context. The program context is typically a stack trace, perhaps -// augmented with auxiliary information like the thread-id, some -// indicator of a higher level request being handled etc. -message Sample { - // The ids recorded here correspond to a Profile.location.id. - // The leaf is at location_id[0]. - repeated uint64 location_id = 1; - // The type and unit of each value is defined by the corresponding - // entry in Profile.sample_type. All samples must have the same - // number of values, the same as the length of Profile.sample_type. - // When aggregating multiple samples into a single sample, the - // result has a list of values that is the element-wise sum of the - // lists of the originals. - repeated int64 value = 2; - // label includes additional context for this sample. It can include - // things like a thread id, allocation size, etc. - // - // NOTE: While possible, having multiple values for the same label key is - // strongly discouraged and should never be used. Most tools (e.g. pprof) do - // not have good (or any) support for multi-value labels. And an even more - // discouraged case is having a string label and a numeric label of the same - // name on a sample. Again, possible to express, but should not be used. - repeated Label label = 3; -} - -message Label { - int64 key = 1; // Index into string table - - // At most one of the following must be present - int64 str = 2; // Index into string table - int64 num = 3; - - // Should only be present when num is present. - // Specifies the units of num. - // Use arbitrary string (for example, "requests") as a custom count unit. - // If no unit is specified, consumer may apply heuristic to deduce the unit. - // Consumers may also interpret units like "bytes" and "kilobytes" as memory - // units and units like "seconds" and "nanoseconds" as time units, - // and apply appropriate unit conversions to these. - int64 num_unit = 4; // Index into string table -} - -message Mapping { - // Unique nonzero id for the mapping. - uint64 id = 1; - // Address at which the binary (or DLL) is loaded into memory. - uint64 memory_start = 2; - // The limit of the address range occupied by this mapping. - uint64 memory_limit = 3; - // Offset in the binary that corresponds to the first mapped address. - uint64 file_offset = 4; - // The object this entry is loaded from. This can be a filename on - // disk for the main binary and shared libraries, or virtual - // abstractions like "[vdso]". - int64 filename = 5; // Index into string table - // A string that uniquely identifies a particular program version - // with high probability. E.g., for binaries generated by GNU tools, - // it could be the contents of the .note.gnu.build-id field. - int64 build_id = 6; // Index into string table - - // The following fields indicate the resolution of symbolic info. - bool has_functions = 7; - bool has_filenames = 8; - bool has_line_numbers = 9; - bool has_inline_frames = 10; -} - -// Describes function and line table debug information. -message Location { - // Unique nonzero id for the location. A profile could use - // instruction addresses or any integer sequence as ids. - uint64 id = 1; - // The id of the corresponding profile.Mapping for this location. - // It can be unset if the mapping is unknown or not applicable for - // this profile type. - uint64 mapping_id = 2; - // The instruction address for this location, if available. It - // should be within [Mapping.memory_start...Mapping.memory_limit] - // for the corresponding mapping. A non-leaf address may be in the - // middle of a call instruction. It is up to display tools to find - // the beginning of the instruction if necessary. - uint64 address = 3; - // Multiple line indicates this location has inlined functions, - // where the last entry represents the caller into which the - // preceding entries were inlined. - // - // E.g., if memcpy() is inlined into printf: - // line[0].function_name == "memcpy" - // line[1].function_name == "printf" - repeated Line line = 4; - // Provides an indication that multiple symbols map to this location's - // address, for example due to identical code folding by the linker. In that - // case the line information above represents one of the multiple - // symbols. This field must be recomputed when the symbolization state of the - // profile changes. - bool is_folded = 5; -} - -message Line { - // The id of the corresponding profile.Function for this line. - uint64 function_id = 1; - // Line number in source code. - int64 line = 2; -} - -message Function { - // Unique nonzero id for the function. - uint64 id = 1; - // Name of the function, in human-readable form if available. - int64 name = 2; // Index into string table - // Name of the function, as identified by the system. - // For instance, it can be a C++ mangled name. - int64 system_name = 3; // Index into string table - // Source file containing the function. - int64 filename = 4; // Index into string table - // Line number in source file. - int64 start_line = 5; -} diff --git a/opentelemetry/proto/profiles/v1/decision-log.md b/opentelemetry/proto/profiles/v1/decision-log.md index 310b82862..6c1f4fb84 100644 --- a/opentelemetry/proto/profiles/v1/decision-log.md +++ b/opentelemetry/proto/profiles/v1/decision-log.md @@ -114,3 +114,55 @@ if they are > 32 bits then the payload is going to be too huge anyway ### Details [Github Conversation](https://github.com/open-telemetry/opentelemetry-proto/pull/488/files#r1246155907) + + + + + +# Decision #6 +Arrays of integers should be used instead of arrays of structs + +### Date +Aug 23 2023 + +### Context + +Samples are a collection of references to other messages plus a value. The standard way of representing those is to put each `Sample` into a separate message, and link from `Sample` to other messages. Parsing / generating such payloads creates many individual objects that runtime has to track. + +An "arrays of integers" representation puts values of the same kind into separate arrays. This reduces the size of the resulting protobuf payload and the number of objects that need to be allocated to parse / generate such payload. + +Here's pseudocode that illustrates the approach. Note that this example is simplified for the sake of clarity: + +``` +// normalized +"samples": [ + { + "stacktrace_id": 1, + "value": 100 + }, { + "stacktrace_id": 2, + "value": 200 + } +], + +// arrays +"stacktrace_ids": [1, 2], +"values": [100, 200] +``` + +Benchmarking shows that this approach is more efficient in terms of CPU utilization, memory consumption and size of the resulting protobuf payload. + +### Alternatives Considered + +`normalized` represenation where we don't use arrays of integers, but arrays of structures instead + + +### Reasoning + +* `arrays` representation is the most efficient in terms of CPU utilization, memory consumption and size of the resulting protobuf payload. It is however cognitively more complex to implement and understand. +* as presented in [Design Goals](#design-goals) section, the performance characteristics of the format are very important for the profiling signal +* the format is not intended to be used directly by the end users, but rather by the developers of profiling systems that are used to and are expected to be able to handle the complexity. It is not more complex than other existing formats + +### Details + +* [Benchmarking Results](https://docs.google.com/spreadsheets/d/1Q-6MlegV8xLYdz5WD5iPxQU2tsfodX1-CDV1WeGzyQ0/edit?usp=sharing) diff --git a/opentelemetry/proto/profiles/v1/profiles.proto b/opentelemetry/proto/profiles/v1/profiles.proto index 2f902d5ec..a9eb666da 100644 --- a/opentelemetry/proto/profiles/v1/profiles.proto +++ b/opentelemetry/proto/profiles/v1/profiles.proto @@ -82,7 +82,9 @@ message ScopeProfiles { string schema_url = 3; } -// A Profile represents a single profile. +// A Profile represents a single profile generated by a profiler. This is similar to a pprof profile or a jfr profile. It has an ID and it is bound by time. Profile contains reference tables for stack traces, mappings, locations, functions, links, attribute sets, and string values. + +// Profile embeds one or more ProfileType messages — this allows to represent multiple profiles of different types (e.g allocated objects and allocated bytes) in a single Profile message. message Profile { // A unique identifier for a profile. The ID is a 16-byte array. An ID with // all zeroes is considered invalid. @@ -122,14 +124,159 @@ message Profile { uint32 dropped_attributes_count = 5; // This can be original pprof or jfr encoded profile. - bytes original_payload = 7; - - // this is temporary until we pick one of these formats - // fields from one of these will be embedded in Profile message: - oneof alternative_profile { - opentelemetry.proto.profiles.v1.alternatives.pprof.Profile pprof = 8; - opentelemetry.proto.profiles.v1.alternatives.normalized.Profile normalized = 9; - opentelemetry.proto.profiles.v1.alternatives.arrays.Profile arrays = 10; - opentelemetry.proto.profiles.v1.alternatives.denormalized.Profile denormalized = 11; - } + bytes original_payload = 6; + + // List of stack traces included in this profile. + repeated Stacktrace stacktraces = 7; + + // List of mappings included in this profile. + repeated Mapping mappings = 8; + + // List of locations included in this profile. + repeated Location locations = 9; + + // List of functions included in this profile. + repeated Function functions = 10; + + // List of links to trace spans associated with this profile. + repeated Link links = 11; + + // List of attribute sets in this profile. + repeated AttributeSet attribute_sets = 12; + + // A table of string values referenced by indices. + // The 0-th element must be an empty string (""). + repeated string string_table = 13; + + // List of profile types included in this profile. The first item in the list is consider to be the "default" profile type. + repeated ProfileType profile_types = 14; +} + +// A pointer from a profile to a trace span. This allows for linking between profiles and traces. +message Link { + // A unique identifier of a trace that this linked span is part of. The ID is a + // 16-byte array. + bytes trace_id = 1; + // A unique identifier for the linked span. The ID is an 8-byte array. + bytes span_id = 2; +} + +// AttributeSet represents a set of attributes. It is referenced from other tables +// and is not embedded because it is common for multiple samples to have the same +// set of attributes. +message AttributeSet { + // Attributes associated with a specific sample. [Optional]. + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + repeated opentelemetry.proto.common.v1.KeyValue attributes = 1; + uint32 dropped_attributes_count = 2; +} + +// A stack trace is a sequence of locations. Order of locations is important and goes from callers to callees. Many stacktraces will point to the same locations, and that's why we use references. It does not contain any attribute sets or links or values or timestamps. This is because of the "arrays of integers" optimization. The link between stacktraces, attribute sets, links, values and timestamps is implicit and is based on the order of the elements in the corresponding tables. +message Stacktrace { + repeated uint32 location_indices = 1; +} + +// Temporality of aggregation for the sample. +enum AggregationTemporality { + AGGREGATION_TEMPORALITY_UNSPECIFIED = 0; // Temporality unspecified. + AGGREGATION_TEMPORALITY_DELTA = 1; // Delta aggregation over time. + AGGREGATION_TEMPORALITY_CUMULATIVE = 2; // Cumulative aggregation over time. +} + +// Represents a single profile type. It implicitely creates a connection between stacktraces, links, attribute sets, values and timestamps. The connection is based on the order of the elements in the corresponding tables. This implicit connection is what is commonly refferred to as a "Sample". The length of tables must be the same. It is acceptable however for timestamps, links and attribute set tables to be empty. It is not acceptable for stacktrace or values tables to be empty. +message ProfileType { + // Temporality of aggregation for the sample. + AggregationTemporality aggregation_temporality = 1; + + // Profiler sample rate. [Optional]. + uint64 sample_rate = 2; + + // Index into the string table for the type of the sample. + uint32 type_index = 3; + + // Index into the string table for the unit of the sample. + uint32 unit_index = 4; + + // List of indices referring to stack traces in the profile's stack trace table. + repeated uint32 stacktrace_indices = 10; + + // List of indices referring to links in the profile's link table. + repeated uint32 link_indices = 11; + + // List of indices referring to attribute sets in the profile's attribute set table. + repeated uint32 attribute_set_indices = 12; + + // List of values associated with the profile type. + repeated int64 values = 13; + + // List of timestamps associated with the profile type. + repeated uint64 timestamps = 14; +} + +enum SymbolFidelity { + SYMBOL_FIDELITY_UNSPECIFIED = 0; + SYMBOL_FIDELITY_FULL = 1; +} + +message Mapping { + // Address at which the binary (or DLL) is loaded into memory. + uint64 memory_start = 1; + // The limit of the address range occupied by this mapping. + uint64 memory_limit = 2; + // Offset in the binary that corresponds to the first mapped address. + uint64 file_offset = 3; + // The object this entry is loaded from. This can be a filename on + // disk for the main binary and shared libraries, or virtual + // abstractions like "[vdso]". + uint32 filename_index = 4; // Index into string table + // A string that uniquely identifies a particular program version + // with high probability. E.g., for binaries generated by GNU tools, + // it could be the contents of the .note.gnu.build-id field. + uint32 build_id_index = 5; // Index into string table + + SymbolFidelity symbolic_info = 6; + repeated uint32 attribute_set_indices = 7; +} + +// Describes function and line table debug information. +message Location { + // The id of the corresponding profile.Mapping for this location. + // It can be unset if the mapping is unknown or not applicable for + // this profile type. + uint32 mapping_index = 1; + // The instruction address for this location, if available. It + // should be within [Mapping.memory_start...Mapping.memory_limit] + // for the corresponding mapping. A non-leaf address may be in the + // middle of a call instruction. It is up to display tools to find + // the beginning of the instruction if necessary. + uint64 address = 2; + // Multiple line indicates this location has inlined functions, + // where the last entry represents the caller into which the + // preceding entries were inlined. + // + // E.g., if memcpy() is inlined into printf: + // line[0].function_name == "memcpy" + // line[1].function_name == "printf" + repeated Line line = 3; + repeated uint32 attribute_set_indices = 5; +} + +message Line { + // The id of the corresponding profile.Function for this line. + uint32 function_index = 1; + // Line number in source code. + uint32 line = 2; +} + +message Function { + // Name of the function, in human-readable form if available. + uint32 name_index = 1; + // Name of the function, as identified by the system. + // For instance, it can be a C++ mangled name. + uint32 system_name_index = 2; + // Source file containing the function. + uint32 filename_index = 3; + // Line number in source file. + uint32 start_line = 4; } From a40299b3ac0d9b0460cc84b0486e4415ce0ca0f3 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Wed, 23 Aug 2023 21:20:27 -0700 Subject: [PATCH 19/75] more changes --- .../proto/profiles/v1/profiles.proto | 89 +++++++++++-------- 1 file changed, 51 insertions(+), 38 deletions(-) diff --git a/opentelemetry/proto/profiles/v1/profiles.proto b/opentelemetry/proto/profiles/v1/profiles.proto index a9eb666da..28cecd962 100644 --- a/opentelemetry/proto/profiles/v1/profiles.proto +++ b/opentelemetry/proto/profiles/v1/profiles.proto @@ -19,13 +19,6 @@ package opentelemetry.proto.profiles.v1; import "opentelemetry/proto/common/v1/common.proto"; import "opentelemetry/proto/resource/v1/resource.proto"; -// TODO(@petethepig): idea here is that we will eventually get rid of these -// and have just version in this (profiles.proto) file -import "opentelemetry/proto/profiles/v1/alternatives/pprof/pprof.proto"; -import "opentelemetry/proto/profiles/v1/alternatives/denormalized/denormalized.proto"; -import "opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto"; -import "opentelemetry/proto/profiles/v1/alternatives/arrays/arrays.proto"; - option csharp_namespace = "OpenTelemetry.Proto.Profiles.V1"; option java_multiple_files = true; option java_package = "io.opentelemetry.proto.profiles.v1"; @@ -82,9 +75,9 @@ message ScopeProfiles { string schema_url = 3; } -// A Profile represents a single profile generated by a profiler. This is similar to a pprof profile or a jfr profile. It has an ID and it is bound by time. Profile contains reference tables for stack traces, mappings, locations, functions, links, attribute sets, and string values. +// A Profile represents a single profile generated by a profiler. It has an ID and it has a start time and end time. Profile contains lookup tables for Stacktraces, Mappings, Locations, Functions, Links, AttributeSets, and strings. -// Profile embeds one or more ProfileType messages — this allows to represent multiple profiles of different types (e.g allocated objects and allocated bytes) in a single Profile message. +// Profile embeds one or more ProfileType messages — this allows to represent multiple profile types (e.g allocated objects and allocated bytes) in a single Profile message. message Profile { // A unique identifier for a profile. The ID is a 16-byte array. An ID with // all zeroes is considered invalid. @@ -123,36 +116,37 @@ message Profile { // attributes. If this value is 0, then no attributes were dropped. uint32 dropped_attributes_count = 5; - // This can be original pprof or jfr encoded profile. + // This is the original profile as retrieved from the profiler. For example, this can be a pprof or jfr encoded profile. The reason users might want to include these is because some formats are very generic and can not be easily converted to a more structured format. + // TODO: add a field that indicates the format of the original payload? bytes original_payload = 6; - // List of stack traces included in this profile. + // A lookup table of Stacktraces. Other messages refer to Stacktraces in this table by index. repeated Stacktrace stacktraces = 7; - // List of mappings included in this profile. + // A lookup table of Mappings. Other messages refer to Mappings in this table by index. repeated Mapping mappings = 8; - // List of locations included in this profile. + // A lookup table of Locations. Other messages refer to Locations in this table by index. repeated Location locations = 9; - // List of functions included in this profile. + // A lookup table of Functions. Other messages refer to Functions in this table by index. repeated Function functions = 10; - // List of links to trace spans associated with this profile. + // A lookup table of Links to trace spans associated with this profile. Other messages refer to Links in this table by index. The first message must be an empty Link — this represents a null Link. repeated Link links = 11; - // List of attribute sets in this profile. + // A lookup table of AttributeSets. Other messages refer to AttributeSets in this table by index. The first message must be an empty AttributeSet — this represents a null AttributeSet. repeated AttributeSet attribute_sets = 12; - // A table of string values referenced by indices. + // A lookup table of strings. Other messages refer to strings in this table by index. // The 0-th element must be an empty string (""). repeated string string_table = 13; - // List of profile types included in this profile. The first item in the list is consider to be the "default" profile type. + // List of profile types included in this profile. The first item in the list is considered to be the "default" profile type. Example profile types are allocated objects or allocated bytes. repeated ProfileType profile_types = 14; } -// A pointer from a profile to a trace span. This allows for linking between profiles and traces. +// Represents a relationship between a Sample (ephemeral structure represented by references to a Stacktrace, AttributeSet, Link + value and a timestamp) and a trace span. This allows for linking between specific Samples within a profile and traces. message Link { // A unique identifier of a trace that this linked span is part of. The ID is a // 16-byte array. @@ -161,18 +155,30 @@ message Link { bytes span_id = 2; } -// AttributeSet represents a set of attributes. It is referenced from other tables -// and is not embedded because it is common for multiple samples to have the same -// set of attributes. +// AttributeSet represents a set of attributes. Multiple Samples, Locations and Mappings may have the same attributes and that's why this is a separate message. These are stored in a lookup table in a Profile. These are referenced by index from other messages. message AttributeSet { - // Attributes associated with a specific sample. [Optional]. + // Attributes associated with a specific Sample, Location or a Mapping. + // attributes is a collection of key/value pairs. Note, global attributes + // like server name can be set using the resource API. Examples of attributes: + // + // "/http/user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36" + // "/http/server_latency": 300 + // "abc.com/myattribute": true + // "abc.com/score": 10.239 + // + // The OpenTelemetry API specification further restricts the allowed value types: + // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/common/README.md#attribute // Attribute keys MUST be unique (it is not allowed to have more than one // attribute with the same key). repeated opentelemetry.proto.common.v1.KeyValue attributes = 1; + + // dropped_attributes_count is the number of attributes that were discarded. Attributes + // can be discarded because their keys are too long or because there are too many + // attributes. If this value is 0, then no attributes were dropped. uint32 dropped_attributes_count = 2; } -// A stack trace is a sequence of locations. Order of locations is important and goes from callers to callees. Many stacktraces will point to the same locations, and that's why we use references. It does not contain any attribute sets or links or values or timestamps. This is because of the "arrays of integers" optimization. The link between stacktraces, attribute sets, links, values and timestamps is implicit and is based on the order of the elements in the corresponding tables. +// A stacktrace is a sequence of locations. Order of locations goes from callers to callees. Many stacktraces will point to the same locations. The link between stacktraces, attribute sets, links, values and timestamps is implicit and is based on the order of the elements in the corresponding tables in ProfileType message. message Stacktrace { repeated uint32 location_indices = 1; } @@ -184,7 +190,7 @@ enum AggregationTemporality { AGGREGATION_TEMPORALITY_CUMULATIVE = 2; // Cumulative aggregation over time. } -// Represents a single profile type. It implicitely creates a connection between stacktraces, links, attribute sets, values and timestamps. The connection is based on the order of the elements in the corresponding tables. This implicit connection is what is commonly refferred to as a "Sample". The length of tables must be the same. It is acceptable however for timestamps, links and attribute set tables to be empty. It is not acceptable for stacktrace or values tables to be empty. +// Represents a single profile type. It implicitely creates a connection between Stacktraces, Links, AttributeSets, values and timestamps. The connection is based on the order of the elements in the corresponding lists. This implicit connection creates an ephemeral structure called Sample. The length of reference lists must be the same. It is acceptable however for timestamps, links and attribute set lists to be empty. It is not acceptable for stacktrace or values lists to be empty. message ProfileType { // Temporality of aggregation for the sample. AggregationTemporality aggregation_temporality = 1; @@ -198,19 +204,19 @@ message ProfileType { // Index into the string table for the unit of the sample. uint32 unit_index = 4; - // List of indices referring to stack traces in the profile's stack trace table. + // List of indices referring to Stacktraces in the Profile's stacktrace table. repeated uint32 stacktrace_indices = 10; - // List of indices referring to links in the profile's link table. + // List of indices referring to Links in the Profile's link table. Each link corresponds to a Stacktrace in stacktrace_indices list. Length must match stacktrace_indices length. [Optional] repeated uint32 link_indices = 11; - // List of indices referring to attribute sets in the profile's attribute set table. + // List of indices referring to AttributeSets in the Profile's attribute set table. Each attribute set corresponds to a Stacktrace in stacktrace_indices list. Length must match stacktrace_indices length. [Optional] repeated uint32 attribute_set_indices = 12; - // List of values associated with the profile type. + // List of values. Each value corresponds to a Stacktrace in stacktrace_indices list. Length must match stacktrace_indices length. repeated int64 values = 13; - // List of timestamps associated with the profile type. + // List of timestamps. Each timestamp corresponds to a Stacktrace in stacktrace_indices list. Length must match stacktrace_indices length. repeated uint64 timestamps = 14; } @@ -219,6 +225,7 @@ enum SymbolFidelity { SYMBOL_FIDELITY_FULL = 1; } +// Describes the mapping from a binary to its original source code. These are stored in a lookup table in a Profile. These are referenced by index from other messages. message Mapping { // Address at which the binary (or DLL) is loaded into memory. uint64 memory_start = 1; @@ -228,18 +235,20 @@ message Mapping { uint64 file_offset = 3; // The object this entry is loaded from. This can be a filename on // disk for the main binary and shared libraries, or virtual - // abstractions like "[vdso]". - uint32 filename_index = 4; // Index into string table + // abstractions like "[vdso]". Index into string table + uint32 filename_index = 4; // A string that uniquely identifies a particular program version // with high probability. E.g., for binaries generated by GNU tools, - // it could be the contents of the .note.gnu.build-id field. - uint32 build_id_index = 5; // Index into string table + // it could be the contents of the .note.gnu.build-id field. Index into string table + uint32 build_id_index = 5; SymbolFidelity symbolic_info = 6; + + // Reference to an attribute set from the Profile's attribute set table. repeated uint32 attribute_set_indices = 7; } -// Describes function and line table debug information. +// Describes function and line table debug information. These are stored in a lookup table in a Profile. These are referenced by index from other messages. message Location { // The id of the corresponding profile.Mapping for this location. // It can be unset if the mapping is unknown or not applicable for @@ -259,9 +268,12 @@ message Location { // line[0].function_name == "memcpy" // line[1].function_name == "printf" repeated Line line = 3; + + // Reference to an attribute set from the Profile's attribute set table. repeated uint32 attribute_set_indices = 5; } +// Represents a line in a source file. These are embedded within a Location message. message Line { // The id of the corresponding profile.Function for this line. uint32 function_index = 1; @@ -269,13 +281,14 @@ message Line { uint32 line = 2; } +// Represents a function in a source file. These are stored in a lookup table in a Profile. These are referenced by index from other messages. message Function { - // Name of the function, in human-readable form if available. + // Name of the function, in human-readable form if available. Index into string table uint32 name_index = 1; // Name of the function, as identified by the system. - // For instance, it can be a C++ mangled name. + // For instance, it can be a C++ mangled name. Index into string table uint32 system_name_index = 2; - // Source file containing the function. + // Source file containing the function. Index into string table uint32 filename_index = 3; // Line number in source file. uint32 start_line = 4; From 10843b83e51513532b7f5a72de428b5172e23a55 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Mon, 11 Sep 2023 12:59:21 -0700 Subject: [PATCH 20/75] Updates copyright notice at the top of profiles.proto --- opentelemetry/proto/profiles/v1/profiles.proto | 3 +++ 1 file changed, 3 insertions(+) diff --git a/opentelemetry/proto/profiles/v1/profiles.proto b/opentelemetry/proto/profiles/v1/profiles.proto index 28cecd962..bf98fd884 100644 --- a/opentelemetry/proto/profiles/v1/profiles.proto +++ b/opentelemetry/proto/profiles/v1/profiles.proto @@ -1,4 +1,5 @@ // Copyright 2023, OpenTelemetry Authors +// Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -225,6 +226,8 @@ enum SymbolFidelity { SYMBOL_FIDELITY_FULL = 1; } +// The following messages were forked from the pprof proto definition and modified to fit the OTLP profile schema: + // Describes the mapping from a binary to its original source code. These are stored in a lookup table in a Profile. These are referenced by index from other messages. message Mapping { // Address at which the binary (or DLL) is loaded into memory. From 3630eec010501bab3521658828be50c02d48513c Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Wed, 4 Oct 2023 11:21:22 -0700 Subject: [PATCH 21/75] adds a new "pprofextended" alternative version --- .../pprofextended/pprofextended.proto | 221 ++++++++++++++++++ 1 file changed, 221 insertions(+) create mode 100644 opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto diff --git a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto new file mode 100644 index 000000000..330c4c08b --- /dev/null +++ b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto @@ -0,0 +1,221 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Profile is a common stacktrace profile format. +// +// Measurements represented with this format should follow the +// following conventions: +// +// - Consumers should treat unset optional fields as if they had been +// set with their default value. +// +// - When possible, measurements should be stored in "unsampled" form +// that is most useful to humans. There should be enough +// information present to determine the original sampled values. +// +// - On-disk, the serialized proto must be gzip-compressed. +// +// - The profile is represented as a set of samples, where each sample +// references a sequence of locations, and where each location belongs +// to a mapping. +// - There is a N->1 relationship from sample.location_id entries to +// locations. For every sample.location_id entry there must be a +// unique Location with that id. +// - There is an optional N->1 relationship from locations to +// mappings. For every nonzero Location.mapping_id there must be a +// unique Mapping with that id. + +syntax = "proto3"; + +package opentelemetry.proto.profiles.v1.alternatives.pprofextended; + +option go_package = "go.opentelemetry.io/proto/otlp/profiles/v1/alternatives/pprofextended"; + +message Profile { + // A description of the samples associated with each Sample.value. + // For a cpu profile this might be: + // [["cpu","nanoseconds"]] or [["wall","seconds"]] or [["syscall","count"]] + // For a heap profile, this might be: + // [["allocations","count"], ["space","bytes"]], + // If one of the values represents the number of events represented + // by the sample, by convention it should be at index 0 and use + // sample_type.unit == "count". + repeated ValueType sample_type = 1; + // The set of samples recorded in this profile. + repeated Sample sample = 2; + // Mapping from address ranges to the image/binary/library mapped + // into that address range. mapping[0] will be the main binary. + repeated Mapping mapping = 3; + // Locations referenced by samples. + repeated Location location = 4; + // Functions referenced by locations. + repeated Function function = 5; + // A common table for strings referenced by various messages. + // string_table[0] must always be "". + repeated string string_table = 6; + // frames with Function.function_name fully matching the following + // regexp will be dropped from the samples, along with their successors. + int64 drop_frames = 7; // Index into string table. + // frames with Function.function_name fully matching the following + // regexp will be kept, even if it matches drop_frames. + int64 keep_frames = 8; // Index into string table. + + // The following fields are informational, do not affect + // interpretation of results. + + // Time of collection (UTC) represented as nanoseconds past the epoch. + int64 time_nanos = 9; + // Duration of the profile, if a duration makes sense. + int64 duration_nanos = 10; + // The kind of events between sampled occurrences. + // e.g [ "cpu","cycles" ] or [ "heap","bytes" ] + ValueType period_type = 11; + // The number of events between sampled occurrences. + int64 period = 12; + // Free-form text associated with the profile. The text is displayed as is + // to the user by the tools that read profiles (e.g. by pprof). This field + // should not be used to store any machine-readable information, it is only + // for human-friendly content. The profile must stay functional if this field + // is cleaned. + repeated int64 comment = 13; // Indices into string table. + // Index into the string table of the type of the preferred sample + // value. If unset, clients should default to the last sample value. + int64 default_sample_type = 14; +} + +// ValueType describes the semantics and measurement units of a value. +message ValueType { + int64 type = 1; // Index into string table. + int64 unit = 2; // Index into string table. +} + +// Each Sample records values encountered in some program +// context. The program context is typically a stack trace, perhaps +// augmented with auxiliary information like the thread-id, some +// indicator of a higher level request being handled etc. +message Sample { + // The ids recorded here correspond to a Profile.location.id. + // The leaf is at location_id[0]. + repeated uint64 location_id = 1; + // The type and unit of each value is defined by the corresponding + // entry in Profile.sample_type. All samples must have the same + // number of values, the same as the length of Profile.sample_type. + // When aggregating multiple samples into a single sample, the + // result has a list of values that is the element-wise sum of the + // lists of the originals. + repeated int64 value = 2; + // label includes additional context for this sample. It can include + // things like a thread id, allocation size, etc. + // + // NOTE: While possible, having multiple values for the same label key is + // strongly discouraged and should never be used. Most tools (e.g. pprof) do + // not have good (or any) support for multi-value labels. And an even more + // discouraged case is having a string label and a numeric label of the same + // name on a sample. Again, possible to express, but should not be used. + repeated Label label = 3; +} + +message Label { + int64 key = 1; // Index into string table + + // At most one of the following must be present + int64 str = 2; // Index into string table + int64 num = 3; + + // Should only be present when num is present. + // Specifies the units of num. + // Use arbitrary string (for example, "requests") as a custom count unit. + // If no unit is specified, consumer may apply heuristic to deduce the unit. + // Consumers may also interpret units like "bytes" and "kilobytes" as memory + // units and units like "seconds" and "nanoseconds" as time units, + // and apply appropriate unit conversions to these. + int64 num_unit = 4; // Index into string table +} + +message Mapping { + // Unique nonzero id for the mapping. + uint64 id = 1; + // Address at which the binary (or DLL) is loaded into memory. + uint64 memory_start = 2; + // The limit of the address range occupied by this mapping. + uint64 memory_limit = 3; + // Offset in the binary that corresponds to the first mapped address. + uint64 file_offset = 4; + // The object this entry is loaded from. This can be a filename on + // disk for the main binary and shared libraries, or virtual + // abstractions like "[vdso]". + int64 filename = 5; // Index into string table + // A string that uniquely identifies a particular program version + // with high probability. E.g., for binaries generated by GNU tools, + // it could be the contents of the .note.gnu.build-id field. + int64 build_id = 6; // Index into string table + + // The following fields indicate the resolution of symbolic info. + bool has_functions = 7; + bool has_filenames = 8; + bool has_line_numbers = 9; + bool has_inline_frames = 10; +} + +// Describes function and line table debug information. +message Location { + // Unique nonzero id for the location. A profile could use + // instruction addresses or any integer sequence as ids. + uint64 id = 1; + // The id of the corresponding profile.Mapping for this location. + // It can be unset if the mapping is unknown or not applicable for + // this profile type. + uint64 mapping_id = 2; + // The instruction address for this location, if available. It + // should be within [Mapping.memory_start...Mapping.memory_limit] + // for the corresponding mapping. A non-leaf address may be in the + // middle of a call instruction. It is up to display tools to find + // the beginning of the instruction if necessary. + uint64 address = 3; + // Multiple line indicates this location has inlined functions, + // where the last entry represents the caller into which the + // preceding entries were inlined. + // + // E.g., if memcpy() is inlined into printf: + // line[0].function_name == "memcpy" + // line[1].function_name == "printf" + repeated Line line = 4; + // Provides an indication that multiple symbols map to this location's + // address, for example due to identical code folding by the linker. In that + // case the line information above represents one of the multiple + // symbols. This field must be recomputed when the symbolization state of the + // profile changes. + bool is_folded = 5; +} + +message Line { + // The id of the corresponding profile.Function for this line. + uint64 function_id = 1; + // Line number in source code. + int64 line = 2; +} + +message Function { + // Unique nonzero id for the function. + uint64 id = 1; + // Name of the function, in human-readable form if available. + int64 name = 2; // Index into string table + // Name of the function, as identified by the system. + // For instance, it can be a C++ mangled name. + int64 system_name = 3; // Index into string table + // Source file containing the function. + int64 filename = 4; // Index into string table + // Line number in source file. + int64 start_line = 5; +} From 2cf711b3cfcc1edd4e46f9b82d19d016d6d0aa2a Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Wed, 4 Oct 2023 11:49:53 -0700 Subject: [PATCH 22/75] referencing by index instead of id --- .../pprofextended/pprofextended.proto | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto index 330c4c08b..67815d456 100644 --- a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto +++ b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto @@ -105,9 +105,9 @@ message ValueType { // augmented with auxiliary information like the thread-id, some // indicator of a higher level request being handled etc. message Sample { - // The ids recorded here correspond to a Profile.location.id. - // The leaf is at location_id[0]. - repeated uint64 location_id = 1; + // The indices recorded here correspond to locations in Profile.location. + // The leaf is at location_index[0]. + repeated uint64 location_index = 1; // The type and unit of each value is defined by the corresponding // entry in Profile.sample_type. All samples must have the same // number of values, the same as the length of Profile.sample_type. @@ -144,7 +144,7 @@ message Label { } message Mapping { - // Unique nonzero id for the mapping. + // Unique nonzero id for the mapping. [deprecated] uint64 id = 1; // Address at which the binary (or DLL) is loaded into memory. uint64 memory_start = 2; @@ -171,12 +171,12 @@ message Mapping { // Describes function and line table debug information. message Location { // Unique nonzero id for the location. A profile could use - // instruction addresses or any integer sequence as ids. + // instruction addresses or any integer sequence as ids. [deprecated] uint64 id = 1; - // The id of the corresponding profile.Mapping for this location. + // The index of the corresponding profile.Mapping for this location. // It can be unset if the mapping is unknown or not applicable for // this profile type. - uint64 mapping_id = 2; + uint64 mapping_index = 2; // The instruction address for this location, if available. It // should be within [Mapping.memory_start...Mapping.memory_limit] // for the corresponding mapping. A non-leaf address may be in the @@ -200,14 +200,14 @@ message Location { } message Line { - // The id of the corresponding profile.Function for this line. - uint64 function_id = 1; + // The index of the corresponding profile.Function for this line. + uint64 function_index = 1; // Line number in source code. int64 line = 2; } message Function { - // Unique nonzero id for the function. + // Unique nonzero id for the function. [deprecated] uint64 id = 1; // Name of the function, in human-readable form if available. int64 name = 2; // Index into string table From 706154cef788bbc5e5c71854b70a19fdc259e3a9 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Wed, 4 Oct 2023 12:39:34 -0700 Subject: [PATCH 23/75] feat: sharing of the call stacks between samples --- .../pprofextended/pprofextended.proto | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto index 67815d456..39ee79b0c 100644 --- a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto +++ b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto @@ -57,7 +57,7 @@ message Profile { // Mapping from address ranges to the image/binary/library mapped // into that address range. mapping[0] will be the main binary. repeated Mapping mapping = 3; - // Locations referenced by samples. + // Locations referenced by samples. [deprecated, superseded by stacktrace] repeated Location location = 4; // Functions referenced by locations. repeated Function function = 5; @@ -92,6 +92,8 @@ message Profile { // Index into the string table of the type of the preferred sample // value. If unset, clients should default to the last sample value. int64 default_sample_type = 14; + // The set of Stacktraces recorded in this profile. + repeated Stacktrace stacktrace = 15; } // ValueType describes the semantics and measurement units of a value. @@ -106,7 +108,7 @@ message ValueType { // indicator of a higher level request being handled etc. message Sample { // The indices recorded here correspond to locations in Profile.location. - // The leaf is at location_index[0]. + // The leaf is at location_index[0]. [deprecated] repeated uint64 location_index = 1; // The type and unit of each value is defined by the corresponding // entry in Profile.sample_type. All samples must have the same @@ -124,6 +126,17 @@ message Sample { // discouraged case is having a string label and a numeric label of the same // name on a sample. Again, possible to express, but should not be used. repeated Label label = 3; + + // This is a reference to a stacktrace in Profile.stacktrace. + // Supersedes location_index. + uint64 stacktrace_index = 4; +} + +// Stacktrace is a list of locations that represent a stack trace. +message Stacktrace { + // The indices recorded here correspond to locations in Profile.location. + // The leaf is at location_index[0]. + repeated uint64 location_index = 1; } message Label { From 3e2e211594a6869cab67ee37d190a58773e3885a Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Wed, 4 Oct 2023 13:58:03 -0700 Subject: [PATCH 24/75] feat: Reuse of OpenTelemetry attributes. --- opentelemetry/proto/common/v1/common.proto | 46 ++++++++++++++++++- .../pprofextended/pprofextended.proto | 9 ++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/opentelemetry/proto/common/v1/common.proto b/opentelemetry/proto/common/v1/common.proto index ff8a21a1f..4af3dec17 100644 --- a/opentelemetry/proto/common/v1/common.proto +++ b/opentelemetry/proto/common/v1/common.proto @@ -67,7 +67,7 @@ message KeyValue { } // InstrumentationScope is a message representing the instrumentation scope information -// such as the fully qualified name and version. +// such as the fully qualified name and version. message InstrumentationScope { // An empty instrumentation scope name means the name is unknown. string name = 1; @@ -79,3 +79,47 @@ message InstrumentationScope { repeated KeyValue attributes = 3; uint32 dropped_attributes_count = 4; } + +// AnyValueInterned is used to represent any type of attribute value. AnyValueInterned may contain a +// primitive value such as a string or integer or it may contain an arbitrary nested +// object containing arrays, key-value lists and primitives. +message AnyValueInterned { + // The value is one of the listed fields. It is valid for all values to be unspecified + // in which case this AnyValueInterned is considered to be "empty". + oneof value { + int64 string_value = 1; // Index into the string_table. + bool bool_value = 2; + int64 int_value = 3; + double double_value = 4; + ArrayValueInterned array_value = 5; + KeyValueListInterned kvlist_value = 6; + int64 bytes_value = 7; // Index into the bytes_table. + } +} + +// ArrayValue is a list of AnyValueInterned messages. We need ArrayValue as a message +// since oneof in AnyValueInterned does not allow repeated fields. +message ArrayValueInterned { + // Array of values. The array may be empty (contain 0 elements). + repeated AnyValueInterned values = 1; +} + +// KeyValueListInterned is a list of KeyValueInterned messages. We need KeyValueListInterned as a message +// since `oneof` in AnyValueInterned does not allow repeated fields. Everywhere else where we need +// a list of KeyValueInterned messages (e.g. in Span) we use `repeated KeyValueInterned` directly to +// avoid unnecessary extra wrapping (which slows down the protocol). The 2 approaches +// are semantically equivalent. +message KeyValueListInterned { + // A collection of key/value pairs of key-value pairs. The list may be empty (may + // contain 0 elements). + // The keys MUST be unique (it is not allowed to have more than one + // value with the same key). + repeated KeyValueInterned values = 1; +} + +// KeyValueInterned is a key-value pair that is used to store Span attributes, Link +// attributes, etc. +message KeyValueInterned { + int64 key = 1; // Index into the string_table. + AnyValueInterned value = 2; +} diff --git a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto index 39ee79b0c..ce4399994 100644 --- a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto +++ b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto @@ -40,6 +40,8 @@ syntax = "proto3"; package opentelemetry.proto.profiles.v1.alternatives.pprofextended; +import "opentelemetry/proto/common/v1/common.proto"; + option go_package = "go.opentelemetry.io/proto/otlp/profiles/v1/alternatives/pprofextended"; message Profile { @@ -94,6 +96,9 @@ message Profile { int64 default_sample_type = 14; // The set of Stacktraces recorded in this profile. repeated Stacktrace stacktrace = 15; + // A common table for bytes referenced by various messages. + // bytes_table[0] must always be "". + repeated bytes bytes_table = 16; } // ValueType describes the semantics and measurement units of a value. @@ -125,11 +130,15 @@ message Sample { // not have good (or any) support for multi-value labels. And an even more // discouraged case is having a string label and a numeric label of the same // name on a sample. Again, possible to express, but should not be used. + // [deprecated, superseded by attributes] repeated Label label = 3; // This is a reference to a stacktrace in Profile.stacktrace. // Supersedes location_index. uint64 stacktrace_index = 4; + + repeated opentelemetry.proto.common.v1.KeyValueInterned attributes = 11; + uint32 dropped_attributes_count = 12; } // Stacktrace is a list of locations that represent a stack trace. From 9ec863c5b5a404382af9148aadf431d9f64a97ed Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Wed, 4 Oct 2023 19:57:00 -0700 Subject: [PATCH 25/75] adds unit to KeyValueInterned --- opentelemetry/proto/common/v1/common.proto | 1 + 1 file changed, 1 insertion(+) diff --git a/opentelemetry/proto/common/v1/common.proto b/opentelemetry/proto/common/v1/common.proto index 4af3dec17..74e649cfc 100644 --- a/opentelemetry/proto/common/v1/common.proto +++ b/opentelemetry/proto/common/v1/common.proto @@ -122,4 +122,5 @@ message KeyValueListInterned { message KeyValueInterned { int64 key = 1; // Index into the string_table. AnyValueInterned value = 2; + int64 unit = 3; // Index into the string_table. } From f10e577f6b72629846cd3588e2c6c2878b351d10 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Wed, 4 Oct 2023 19:59:31 -0700 Subject: [PATCH 26/75] fixes proto numbers --- .../v1/alternatives/pprofextended/pprofextended.proto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto index ce4399994..85e8579d9 100644 --- a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto +++ b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto @@ -137,8 +137,8 @@ message Sample { // Supersedes location_index. uint64 stacktrace_index = 4; - repeated opentelemetry.proto.common.v1.KeyValueInterned attributes = 11; - uint32 dropped_attributes_count = 12; + repeated opentelemetry.proto.common.v1.KeyValueInterned attributes = 5; + uint32 dropped_attributes_count = 6; } // Stacktrace is a list of locations that represent a stack trace. From 850e05c5bca957d2e2cdc7c9ce0d697e882eeea7 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Wed, 4 Oct 2023 20:06:22 -0700 Subject: [PATCH 27/75] feat: timestamps --- .../profiles/v1/alternatives/pprofextended/pprofextended.proto | 2 ++ 1 file changed, 2 insertions(+) diff --git a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto index 85e8579d9..7280fe1a9 100644 --- a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto +++ b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto @@ -139,6 +139,8 @@ message Sample { repeated opentelemetry.proto.common.v1.KeyValueInterned attributes = 5; uint32 dropped_attributes_count = 6; + + repeated uint64 timestamps = 7; } // Stacktrace is a list of locations that represent a stack trace. From b6997ba41bfde274b0c32bae7295a5914a96e8ad Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Wed, 4 Oct 2023 20:06:31 -0700 Subject: [PATCH 28/75] feat: aggregation temporality --- .../v1/alternatives/pprofextended/pprofextended.proto | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto index 7280fe1a9..0bc0d36ad 100644 --- a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto +++ b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto @@ -105,6 +105,14 @@ message Profile { message ValueType { int64 type = 1; // Index into string table. int64 unit = 2; // Index into string table. + + enum AggregationTemporality { + AGGREGATION_TEMPORALITY_UNSPECIFIED = 0; + AGGREGATION_TEMPORALITY_DELTA = 1; + AGGREGATION_TEMPORALITY_CUMULATIVE = 2; + } + + AggregationTemporality aggregation_temporality = 3; } // Each Sample records values encountered in some program From 013f7affaeff7b3e9a8757283a188a85d012ef0b Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Wed, 4 Oct 2023 22:48:40 -0700 Subject: [PATCH 29/75] adds pprofextended to the main profiles.proto file --- opentelemetry/proto/profiles/v1/profiles.proto | 2 ++ 1 file changed, 2 insertions(+) diff --git a/opentelemetry/proto/profiles/v1/profiles.proto b/opentelemetry/proto/profiles/v1/profiles.proto index 2f902d5ec..af841113d 100644 --- a/opentelemetry/proto/profiles/v1/profiles.proto +++ b/opentelemetry/proto/profiles/v1/profiles.proto @@ -25,6 +25,7 @@ import "opentelemetry/proto/profiles/v1/alternatives/pprof/pprof.proto"; import "opentelemetry/proto/profiles/v1/alternatives/denormalized/denormalized.proto"; import "opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto"; import "opentelemetry/proto/profiles/v1/alternatives/arrays/arrays.proto"; +import "opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto"; option csharp_namespace = "OpenTelemetry.Proto.Profiles.V1"; option java_multiple_files = true; @@ -131,5 +132,6 @@ message Profile { opentelemetry.proto.profiles.v1.alternatives.normalized.Profile normalized = 9; opentelemetry.proto.profiles.v1.alternatives.arrays.Profile arrays = 10; opentelemetry.proto.profiles.v1.alternatives.denormalized.Profile denormalized = 11; + opentelemetry.proto.profiles.v1.alternatives.pprofextended.Profile pprofextended = 12; } } From 2299b716b4bf784b839bca861d4636d4ae8ef551 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Fri, 13 Oct 2023 15:25:05 -0700 Subject: [PATCH 30/75] better location_id handling --- .../alternatives/pprofextended/pprofextended.proto | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto index 0bc0d36ad..4ee9f7cf6 100644 --- a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto +++ b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto @@ -94,8 +94,8 @@ message Profile { // Index into the string table of the type of the preferred sample // value. If unset, clients should default to the last sample value. int64 default_sample_type = 14; - // The set of Stacktraces recorded in this profile. - repeated Stacktrace stacktrace = 15; + // Array of locations referenced by samples. + repeated int64 location_indices = 15; // A common table for bytes referenced by various messages. // bytes_table[0] must always be "". repeated bytes bytes_table = 16; @@ -143,12 +143,13 @@ message Sample { // This is a reference to a stacktrace in Profile.stacktrace. // Supersedes location_index. - uint64 stacktrace_index = 4; + uint64 locations_start_index = 4; + uint64 locations_end_index = 5; - repeated opentelemetry.proto.common.v1.KeyValueInterned attributes = 5; - uint32 dropped_attributes_count = 6; + repeated opentelemetry.proto.common.v1.KeyValueInterned attributes = 6; + uint32 dropped_attributes_count = 7; - repeated uint64 timestamps = 7; + repeated uint64 timestamps = 8; } // Stacktrace is a list of locations that represent a stack trace. From a80227bbb015e7439a1324d2c5031c4315d57981 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Wed, 18 Oct 2023 17:32:47 -0700 Subject: [PATCH 31/75] attribute experiments --- .../v1/alternatives/pprofextended/pprofextended.proto | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto index 4ee9f7cf6..eed9a8703 100644 --- a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto +++ b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto @@ -99,6 +99,8 @@ message Profile { // A common table for bytes referenced by various messages. // bytes_table[0] must always be "". repeated bytes bytes_table = 16; + + repeated opentelemetry.proto.common.v1.KeyValue attributes3 = 17; } // ValueType describes the semantics and measurement units of a value. @@ -150,6 +152,9 @@ message Sample { uint32 dropped_attributes_count = 7; repeated uint64 timestamps = 8; + + repeated opentelemetry.proto.common.v1.KeyValue attributes2 = 9; + repeated uint64 attributes3 = 10; } // Stacktrace is a list of locations that represent a stack trace. From d938d1fedc518fd87c9b6cb895b8644dce66a00b Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Wed, 18 Oct 2023 17:33:16 -0700 Subject: [PATCH 32/75] Suggestions from Christos (see https://github.com/open-telemetry/opentelemetry-proto-profile/pull/8) --- .../v1/alternatives/pprofextended/pprofextended.proto | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto index eed9a8703..137e11843 100644 --- a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto +++ b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto @@ -204,6 +204,11 @@ message Mapping { bool has_filenames = 8; bool has_line_numbers = 9; bool has_inline_frames = 10; + + // A string that uniquely identifies a particular binary with high probability. + // It is meant to work around deficiencies in the GNU Build ID that may compromise + // uniqueness / correlatability. Index into string table. + uint32 file_id_index = 11; } // Describes function and line table debug information. @@ -235,6 +240,9 @@ message Location { // symbols. This field must be recomputed when the symbolization state of the // profile changes. bool is_folded = 5; + + // Type of frame (e.g. kernel, native, python, hotspot, php). Index into string table. + uint32 type_index = 6; } message Line { From 460581752e4fb636d8660a9e3e27a63f2c92c419 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Wed, 1 Nov 2023 10:28:06 -0700 Subject: [PATCH 33/75] cleanups --- .../pprofextended/pprofextended.proto | 13 +------------ opentelemetry/proto/profiles/v1/profiles.proto | 18 ++++++++++++------ 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto index 137e11843..3170744db 100644 --- a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto +++ b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto @@ -96,11 +96,7 @@ message Profile { int64 default_sample_type = 14; // Array of locations referenced by samples. repeated int64 location_indices = 15; - // A common table for bytes referenced by various messages. - // bytes_table[0] must always be "". - repeated bytes bytes_table = 16; - - repeated opentelemetry.proto.common.v1.KeyValue attributes3 = 17; + repeated opentelemetry.proto.common.v1.KeyValue attribute_table = 17; } // ValueType describes the semantics and measurement units of a value. @@ -157,13 +153,6 @@ message Sample { repeated uint64 attributes3 = 10; } -// Stacktrace is a list of locations that represent a stack trace. -message Stacktrace { - // The indices recorded here correspond to locations in Profile.location. - // The leaf is at location_index[0]. - repeated uint64 location_index = 1; -} - message Label { int64 key = 1; // Index into string table diff --git a/opentelemetry/proto/profiles/v1/profiles.proto b/opentelemetry/proto/profiles/v1/profiles.proto index af841113d..6719ac1b2 100644 --- a/opentelemetry/proto/profiles/v1/profiles.proto +++ b/opentelemetry/proto/profiles/v1/profiles.proto @@ -122,16 +122,22 @@ message Profile { // attributes. If this value is 0, then no attributes were dropped. uint32 dropped_attributes_count = 5; - // This can be original pprof or jfr encoded profile. + // Specifies format of the original payload. Index to the string table. [optional, required if original_payload is present] + uint32 original_payload_format = 6; + // Original payload is stored in this field. It is typically a pprof or jfr file. [optional] bytes original_payload = 7; + // TODO: better name for this? + opentelemetry.proto.profiles.v1.alternatives.pprofextended.Profile profile_body = 8; + + // THIS IS TEMPORARY AND WILL BE REMOVED. ONLY USED FOR BENCHMARKS // this is temporary until we pick one of these formats // fields from one of these will be embedded in Profile message: oneof alternative_profile { - opentelemetry.proto.profiles.v1.alternatives.pprof.Profile pprof = 8; - opentelemetry.proto.profiles.v1.alternatives.normalized.Profile normalized = 9; - opentelemetry.proto.profiles.v1.alternatives.arrays.Profile arrays = 10; - opentelemetry.proto.profiles.v1.alternatives.denormalized.Profile denormalized = 11; - opentelemetry.proto.profiles.v1.alternatives.pprofextended.Profile pprofextended = 12; + opentelemetry.proto.profiles.v1.alternatives.pprof.Profile pprof = 101; + opentelemetry.proto.profiles.v1.alternatives.normalized.Profile normalized = 102; + opentelemetry.proto.profiles.v1.alternatives.arrays.Profile arrays = 103; + opentelemetry.proto.profiles.v1.alternatives.denormalized.Profile denormalized = 104; + opentelemetry.proto.profiles.v1.alternatives.pprofextended.Profile pprofextended = 105; } } From 6310a5b2a4535cbf5c4bf8208abccfa504122442 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Wed, 1 Nov 2023 19:17:40 -0700 Subject: [PATCH 34/75] improvements --- .../alternatives/pprofextended/pprofextended.proto | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto index 3170744db..be4a8282a 100644 --- a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto +++ b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto @@ -59,7 +59,7 @@ message Profile { // Mapping from address ranges to the image/binary/library mapped // into that address range. mapping[0] will be the main binary. repeated Mapping mapping = 3; - // Locations referenced by samples. [deprecated, superseded by stacktrace] + // Locations referenced by samples via location_indices. repeated Location location = 4; // Functions referenced by locations. repeated Function function = 5; @@ -96,7 +96,7 @@ message Profile { int64 default_sample_type = 14; // Array of locations referenced by samples. repeated int64 location_indices = 15; - repeated opentelemetry.proto.common.v1.KeyValue attribute_table = 17; + repeated opentelemetry.proto.common.v1.KeyValue attribute_table = 16; } // ValueType describes the semantics and measurement units of a value. @@ -144,13 +144,11 @@ message Sample { uint64 locations_start_index = 4; uint64 locations_end_index = 5; - repeated opentelemetry.proto.common.v1.KeyValueInterned attributes = 6; - uint32 dropped_attributes_count = 7; + // Timestamps associated with Sample represented in ms. These timestamps are expected to fall within the Profile's time range. [optional] + repeated uint64 timestamps = 6; - repeated uint64 timestamps = 8; - - repeated opentelemetry.proto.common.v1.KeyValue attributes2 = 9; - repeated uint64 attributes3 = 10; + // References to attributes in Profile.attribute_table. [optional] + repeated uint64 attributes = 7; } message Label { From 1f95c9f83c452f77c40ae4082ded62974ce1e534 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Wed, 1 Nov 2023 19:43:21 -0700 Subject: [PATCH 35/75] more changes --- .../pprofextended/pprofextended.proto | 7 +++ .../proto/profiles/v1/profiles.proto | 63 +++++++++++++++++-- 2 files changed, 66 insertions(+), 4 deletions(-) diff --git a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto index be4a8282a..9bb2eff84 100644 --- a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto +++ b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto @@ -96,6 +96,7 @@ message Profile { int64 default_sample_type = 14; // Array of locations referenced by samples. repeated int64 location_indices = 15; + // Lookup table for attributes. repeated opentelemetry.proto.common.v1.KeyValue attribute_table = 16; } @@ -196,6 +197,9 @@ message Mapping { // It is meant to work around deficiencies in the GNU Build ID that may compromise // uniqueness / correlatability. Index into string table. uint32 file_id_index = 11; + + // References to attributes in Profile.attribute_table. [optional] + repeated uint64 attributes = 12; } // Describes function and line table debug information. @@ -230,6 +234,9 @@ message Location { // Type of frame (e.g. kernel, native, python, hotspot, php). Index into string table. uint32 type_index = 6; + + // References to attributes in Profile.attribute_table. [optional] + repeated uint64 attributes = 7; } message Line { diff --git a/opentelemetry/proto/profiles/v1/profiles.proto b/opentelemetry/proto/profiles/v1/profiles.proto index 6719ac1b2..0c17f07d5 100644 --- a/opentelemetry/proto/profiles/v1/profiles.proto +++ b/opentelemetry/proto/profiles/v1/profiles.proto @@ -33,6 +33,61 @@ option java_package = "io.opentelemetry.proto.profiles.v1"; option java_outer_classname = "ProfilesProto"; option go_package = "go.opentelemetry.io/proto/otlp/profiles/v1"; +// Relationships Diagram +// +// ┌──────────────────┐ LEGEND +// │ ProfilesData │ +// └──────────────────┘ ─────▶ embedded +// │ +// │ 1-n ─────▷ referenced by index +// ▼ +// ┌──────────────────┐ +// │ ResourceProfiles │ +// └──────────────────┘ +// │ +// │ 1-n +// ▼ +// ┌──────────────────┐ +// │ ScopeProfiles │ +// └──────────────────┘ +// │ +// │ 1-n +// ▼ +// ┌──────────────────┐ +// │ ProfileContainer │ +// └──────────────────┘ +// │ +// │ 1-1 +// ▼ +// ┌──────────────────┐ +// │ Profile │ +// └──────────────────┘ +// │ +// │ 1-n +// ▼ +// ┌──────────────────┐ 1-n ┌──────────────┐ +// │ Sample │ ──────▷ │ KeyValue │ +// └──────────────────┘ └──────────────┘ +// │ 1-n △ △ +// │ 1-n ┌─────────────────┘ │ 1-n +// ▽ │ │ +// ┌──────────────────┐ 1-1 ┌──────────────┐ +// │ Location │ ──────▷ │ Mapping │ +// └──────────────────┘ └──────────────┘ +// │ +// │ 1-n +// ▼ +// ┌──────────────────┐ +// │ Line │ +// └──────────────────┘ +// │ +// │ 1-1 +// ▽ +// ┌──────────────────┐ +// │ Function │ +// └──────────────────┘ +// + // ProfilesData represents the profiles data that can be stored in a persistent storage, // OR can be embedded by other protocols that transfer OTLP profiles data but do not // implement the OTLP protocol. @@ -83,8 +138,8 @@ message ScopeProfiles { string schema_url = 3; } -// A Profile represents a single profile. -message Profile { +// A ProfileContainer represents a single profile. It wraps pprof profile with open-telemetry specific metadata. +message ProfileContainer { // A unique identifier for a profile. The ID is a 16-byte array. An ID with // all zeroes is considered invalid. // @@ -127,8 +182,8 @@ message Profile { // Original payload is stored in this field. It is typically a pprof or jfr file. [optional] bytes original_payload = 7; - // TODO: better name for this? - opentelemetry.proto.profiles.v1.alternatives.pprofextended.Profile profile_body = 8; + // This is a reference to a pprof profile. + opentelemetry.proto.profiles.v1.alternatives.pprofextended.Profile profile = 8; // THIS IS TEMPORARY AND WILL BE REMOVED. ONLY USED FOR BENCHMARKS // this is temporary until we pick one of these formats From dc7dfcb25759f03d55390452c12468cd2974102d Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Wed, 1 Nov 2023 20:01:13 -0700 Subject: [PATCH 36/75] better spelling --- opentelemetry/proto/profiles/v1/profiles.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opentelemetry/proto/profiles/v1/profiles.proto b/opentelemetry/proto/profiles/v1/profiles.proto index 0c17f07d5..ad5739e5c 100644 --- a/opentelemetry/proto/profiles/v1/profiles.proto +++ b/opentelemetry/proto/profiles/v1/profiles.proto @@ -138,7 +138,7 @@ message ScopeProfiles { string schema_url = 3; } -// A ProfileContainer represents a single profile. It wraps pprof profile with open-telemetry specific metadata. +// A ProfileContainer represents a single profile. It wraps pprof profile with OpenTelemetry specific metadata. message ProfileContainer { // A unique identifier for a profile. The ID is a 16-byte array. An ID with // all zeroes is considered invalid. From 56b272b0146019c8b3292cbb66424e9d527f56d1 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Wed, 1 Nov 2023 20:11:52 -0700 Subject: [PATCH 37/75] fix --- opentelemetry/proto/profiles/v1/profiles.proto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opentelemetry/proto/profiles/v1/profiles.proto b/opentelemetry/proto/profiles/v1/profiles.proto index ad5739e5c..97ab70bf6 100644 --- a/opentelemetry/proto/profiles/v1/profiles.proto +++ b/opentelemetry/proto/profiles/v1/profiles.proto @@ -131,8 +131,8 @@ message ScopeProfiles { // an empty instrumentation scope name (unknown). opentelemetry.proto.common.v1.InstrumentationScope scope = 1; - // A list of Profiles that originate from an instrumentation scope. - repeated Profile profiles = 2; + // A list of ProfileContainers that originate from an instrumentation scope. + repeated ProfileContainer profiles = 2; // This schema_url applies to all profiles and profile events in the "profiles" field. string schema_url = 3; From d78bc9aa963ad827c2e5e493df77df49148c2c88 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Wed, 1 Nov 2023 21:54:45 -0700 Subject: [PATCH 38/75] changes --- .../pprofextended/pprofextended.proto | 72 +++++++++++++++++-- .../proto/profiles/v1/profiles.proto | 9 +-- 2 files changed, 71 insertions(+), 10 deletions(-) diff --git a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto index 9bb2eff84..b3481196f 100644 --- a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto +++ b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto @@ -100,17 +100,77 @@ message Profile { repeated opentelemetry.proto.common.v1.KeyValue attribute_table = 16; } +enum AggregationTemporality { + /* UNSPECIFIED is the default AggregationTemporality, it MUST not be used. */ + AGGREGATION_TEMPORALITY_UNSPECIFIED = 0; + + /** DELTA is an AggregationTemporality for a profiler which reports + changes since last report time. Successive metrics contain aggregation of + values from continuous and non-overlapping intervals. + + The values for a DELTA metric are based only on the time interval + associated with one measurement cycle. There is no dependency on + previous measurements like is the case for CUMULATIVE metrics. + + For example, consider a system measuring the number of requests that + it receives and reports the sum of these requests every second as a + DELTA metric: + + 1. The system starts receiving at time=t_0. + 2. A request is received, the system measures 1 request. + 3. A request is received, the system measures 1 request. + 4. A request is received, the system measures 1 request. + 5. The 1 second collection cycle ends. A metric is exported for the + number of requests received over the interval of time t_0 to + t_0+1 with a value of 3. + 6. A request is received, the system measures 1 request. + 7. A request is received, the system measures 1 request. + 8. The 1 second collection cycle ends. A metric is exported for the + number of requests received over the interval of time t_0+1 to + t_0+2 with a value of 2. */ + AGGREGATION_TEMPORALITY_DELTA = 1; + + /** CUMULATIVE is an AggregationTemporality for a profiler which + reports changes since a fixed start time. This means that current values + of a CUMULATIVE metric depend on all previous measurements since the + start time. Because of this, the sender is required to retain this state + in some form. If this state is lost or invalidated, the CUMULATIVE metric + values MUST be reset and a new fixed start time following the last + reported measurement time sent MUST be used. + + For example, consider a system measuring the number of requests that + it receives and reports the sum of these requests every second as a + CUMULATIVE metric: + + 1. The system starts receiving at time=t_0. + 2. A request is received, the system measures 1 request. + 3. A request is received, the system measures 1 request. + 4. A request is received, the system measures 1 request. + 5. The 1 second collection cycle ends. A metric is exported for the + number of requests received over the interval of time t_0 to + t_0+1 with a value of 3. + 6. A request is received, the system measures 1 request. + 7. A request is received, the system measures 1 request. + 8. The 1 second collection cycle ends. A metric is exported for the + number of requests received over the interval of time t_0 to + t_0+2 with a value of 5. + 9. The system experiences a fault and loses state. + 10. The system recovers and resumes receiving at time=t_1. + 11. A request is received, the system measures 1 request. + 12. The 1 second collection cycle ends. A metric is exported for the + number of requests received over the interval of time t_1 to + t_0+1 with a value of 1. + + Note: Even though, when reporting changes since last report time, using + CUMULATIVE is valid, it is not recommended. */ + AGGREGATION_TEMPORALITY_CUMULATIVE = 2; +} + // ValueType describes the semantics and measurement units of a value. message ValueType { int64 type = 1; // Index into string table. int64 unit = 2; // Index into string table. - enum AggregationTemporality { - AGGREGATION_TEMPORALITY_UNSPECIFIED = 0; - AGGREGATION_TEMPORALITY_DELTA = 1; - AGGREGATION_TEMPORALITY_CUMULATIVE = 2; - } - AggregationTemporality aggregation_temporality = 3; } diff --git a/opentelemetry/proto/profiles/v1/profiles.proto b/opentelemetry/proto/profiles/v1/profiles.proto index 97ab70bf6..8088b4300 100644 --- a/opentelemetry/proto/profiles/v1/profiles.proto +++ b/opentelemetry/proto/profiles/v1/profiles.proto @@ -177,12 +177,13 @@ message ProfileContainer { // attributes. If this value is 0, then no attributes were dropped. uint32 dropped_attributes_count = 5; - // Specifies format of the original payload. Index to the string table. [optional, required if original_payload is present] - uint32 original_payload_format = 6; - // Original payload is stored in this field. It is typically a pprof or jfr file. [optional] + // Specifies format of the original payload. Common values are defined in semantic conventions. [required if original_payload is present] + string original_payload_format = 6; + // Original payload is stored in this field. It is typically a pprof or jfr file. + // The field is optional, however if it is present `profile` MUST be present as well. bytes original_payload = 7; - // This is a reference to a pprof profile. + // This is a reference to a pprof profile. Required, even when original_payload is present. opentelemetry.proto.profiles.v1.alternatives.pprofextended.Profile profile = 8; // THIS IS TEMPORARY AND WILL BE REMOVED. ONLY USED FOR BENCHMARKS From 621a03e6dd253b74afca2654c938e7e7c3df065e Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Wed, 1 Nov 2023 22:03:17 -0700 Subject: [PATCH 39/75] more changes --- opentelemetry/proto/profiles/v1/profiles.proto | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/opentelemetry/proto/profiles/v1/profiles.proto b/opentelemetry/proto/profiles/v1/profiles.proto index 8088b4300..fc3b5ab3d 100644 --- a/opentelemetry/proto/profiles/v1/profiles.proto +++ b/opentelemetry/proto/profiles/v1/profiles.proto @@ -179,8 +179,12 @@ message ProfileContainer { // Specifies format of the original payload. Common values are defined in semantic conventions. [required if original_payload is present] string original_payload_format = 6; - // Original payload is stored in this field. It is typically a pprof or jfr file. - // The field is optional, however if it is present `profile` MUST be present as well. + + // Original payload can be stored in this field. This can be useful for users who want to get the original payload. + // Formats such as JFR are highly extensible and can contain more information than what is defined in this spec. + // Inclusion of original payload should be configurable by the user. Default behavior should be to not include the original payload. + // If the original payload is in pprof format, it SHOULD not be included in this field. + // The field is optional, however if it is present `profile` MUST be present and contain the same profiling information. bytes original_payload = 7; // This is a reference to a pprof profile. Required, even when original_payload is present. From f4cab6254b64e378f50b638416a202a89dba29cf Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Wed, 1 Nov 2023 22:27:42 -0700 Subject: [PATCH 40/75] more changes --- .../v1/alternatives/pprofextended/pprofextended.proto | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto index b3481196f..7281c13a5 100644 --- a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto +++ b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto @@ -200,10 +200,11 @@ message Sample { // [deprecated, superseded by attributes] repeated Label label = 3; - // This is a reference to a stacktrace in Profile.stacktrace. - // Supersedes location_index. + // locations_start_index along with locations_length refers to to a slice of locations in Profile.location. Supersedes location_index. uint64 locations_start_index = 4; - uint64 locations_end_index = 5; + + // locations_length along with locations_start_index refers to a slice of locations in Profile.location. Supersedes location_index. + uint64 locations_length = 5; // Timestamps associated with Sample represented in ms. These timestamps are expected to fall within the Profile's time range. [optional] repeated uint64 timestamps = 6; From a11e73643d475e13ccba291a737e6ebbbbaabedb Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Mon, 11 Dec 2023 21:07:53 -0800 Subject: [PATCH 41/75] adds the last suggestion from https://github.com/open-telemetry/opentelemetry-proto-profile/pull/8 --- .../profiles/v1/alternatives/pprofextended/pprofextended.proto | 3 +++ 1 file changed, 3 insertions(+) diff --git a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto index 7281c13a5..d15d7da09 100644 --- a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto +++ b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto @@ -211,6 +211,9 @@ message Sample { // References to attributes in Profile.attribute_table. [optional] repeated uint64 attributes = 7; + + // A 128bit id that uniquely identifies this stacktrace, globally. Index into string table. [optional] + uint32 stacktrace_id_index = 2; } message Label { From b3cbe490e02d287e788e7ff7494081486aecf6f9 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Thu, 11 Jan 2024 09:35:11 -0800 Subject: [PATCH 42/75] adds clarifications --- opentelemetry/proto/common/v1/common.proto | 2 ++ 1 file changed, 2 insertions(+) diff --git a/opentelemetry/proto/common/v1/common.proto b/opentelemetry/proto/common/v1/common.proto index 74e649cfc..58963bba1 100644 --- a/opentelemetry/proto/common/v1/common.proto +++ b/opentelemetry/proto/common/v1/common.proto @@ -80,6 +80,8 @@ message InstrumentationScope { uint32 dropped_attributes_count = 4; } +// THE FOLLOWING "Interned" MESSAGES ARE TEMPORARY AND WILL BE REMOVED. ONLY USED FOR BENCHMARKS + // AnyValueInterned is used to represent any type of attribute value. AnyValueInterned may contain a // primitive value such as a string or integer or it may contain an arbitrary nested // object containing arrays, key-value lists and primitives. From 210837b7a57de0236639fbc639625599daae0df7 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Thu, 11 Jan 2024 09:57:00 -0800 Subject: [PATCH 43/75] introduces Link - pointer from a profile Sample to a trace Span --- .../pprofextended/pprofextended.proto | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto index d15d7da09..6cf68960a 100644 --- a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto +++ b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto @@ -98,6 +98,18 @@ message Profile { repeated int64 location_indices = 15; // Lookup table for attributes. repeated opentelemetry.proto.common.v1.KeyValue attribute_table = 16; + // Lookup table for links. + repeated Link link_table = 17; +} + +// A pointer from a profile Sample to a trace Span. +message Link { + // A unique identifier of a trace that this linked span is part of. The ID is a + // 16-byte array. + bytes trace_id = 1; + + // A unique identifier for the linked span. The ID is an 8-byte array. + bytes span_id = 2; } enum AggregationTemporality { @@ -212,8 +224,11 @@ message Sample { // References to attributes in Profile.attribute_table. [optional] repeated uint64 attributes = 7; + // References to links in Profile.link_table. [optional] + repeated uint64 links = 8; + // A 128bit id that uniquely identifies this stacktrace, globally. Index into string table. [optional] - uint32 stacktrace_id_index = 2; + uint32 stacktrace_id_index = 9; } message Label { From 61ee41964c52ebf8c401a238a5ab450b02f006fe Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Thu, 11 Jan 2024 10:05:47 -0800 Subject: [PATCH 44/75] Replaces file_id with build_id+enum --- .../alternatives/pprofextended/pprofextended.proto | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto index 6cf68960a..fa22e4a33 100644 --- a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto +++ b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto @@ -248,6 +248,13 @@ message Label { int64 num_unit = 4; // Index into string table } +enum BuildIdKind { + // BUILD_ID_LINKER indicates that the build id is linker-provided (usually GNU build ID) + BUILD_ID_LINKER = 0; + // BUILD_ID_BINARY_HASH indicates that the build id is a hash of the binary. + BUILD_ID_BINARY_HASH = 1; +} + message Mapping { // Unique nonzero id for the mapping. [deprecated] uint64 id = 1; @@ -272,10 +279,8 @@ message Mapping { bool has_line_numbers = 9; bool has_inline_frames = 10; - // A string that uniquely identifies a particular binary with high probability. - // It is meant to work around deficiencies in the GNU Build ID that may compromise - // uniqueness / correlatability. Index into string table. - uint32 file_id_index = 11; + // Specifies the kind of build id. See BuildIdKind enum for more detaiks [optional] + BuildIdKind build_id_kind = 11; // References to attributes in Profile.attribute_table. [optional] repeated uint64 attributes = 12; From f02d550362aee977f1084c40a3f7bfae5c613906 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Thu, 11 Jan 2024 10:09:34 -0800 Subject: [PATCH 45/75] Adds comments to each message and enum --- .../alternatives/pprofextended/pprofextended.proto | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto index fa22e4a33..1a6d60e07 100644 --- a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto +++ b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto @@ -44,6 +44,7 @@ import "opentelemetry/proto/common/v1/common.proto"; option go_package = "go.opentelemetry.io/proto/otlp/profiles/v1/alternatives/pprofextended"; +// Represents a complete profile, including sample types, samples, mappings to binaries, locations, functions, string table, and additional metadata. message Profile { // A description of the samples associated with each Sample.value. // For a cpu profile this might be: @@ -103,6 +104,7 @@ message Profile { } // A pointer from a profile Sample to a trace Span. +// Connects a profile sample to a trace span, identified by unique trace and span IDs. message Link { // A unique identifier of a trace that this linked span is part of. The ID is a // 16-byte array. @@ -112,6 +114,8 @@ message Link { bytes span_id = 2; } +// Specifies the method of aggregating metric values, either DELTA (change since last report) +// or CUMULATIVE (total since a fixed start time). enum AggregationTemporality { /* UNSPECIFIED is the default AggregationTemporality, it MUST not be used. */ AGGREGATION_TEMPORALITY_UNSPECIFIED = 0; @@ -178,7 +182,7 @@ enum AggregationTemporality { AGGREGATION_TEMPORALITY_CUMULATIVE = 2; } -// ValueType describes the semantics and measurement units of a value. +// ValueType describes the type and units of a value, with an optional aggregation temporality. message ValueType { int64 type = 1; // Index into string table. int64 unit = 2; // Index into string table. @@ -231,6 +235,8 @@ message Sample { uint32 stacktrace_id_index = 9; } +// Provides additional context for a sample, +// such as thread ID or allocation size, with optional units. [deprecated] message Label { int64 key = 1; // Index into string table @@ -248,6 +254,7 @@ message Label { int64 num_unit = 4; // Index into string table } +// Indicates the type of build ID, either from the linker or a binary hash. enum BuildIdKind { // BUILD_ID_LINKER indicates that the build id is linker-provided (usually GNU build ID) BUILD_ID_LINKER = 0; @@ -255,6 +262,8 @@ enum BuildIdKind { BUILD_ID_BINARY_HASH = 1; } +// Describes the mapping of a binary in memory, including its address range, +// file offset, and metadata like build ID message Mapping { // Unique nonzero id for the mapping. [deprecated] uint64 id = 1; @@ -323,6 +332,7 @@ message Location { repeated uint64 attributes = 7; } +// Details a specific line in a source code, linked to a function. message Line { // The index of the corresponding profile.Function for this line. uint64 function_index = 1; @@ -330,6 +340,8 @@ message Line { int64 line = 2; } +// Describes a function, including its human-readable name, system name, +// source file, and starting line number in the source. message Function { // Unique nonzero id for the function. [deprecated] uint64 id = 1; From 2fb4135ee661344e457bc09b951a7a6d635fc634 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Thu, 11 Jan 2024 10:09:47 -0800 Subject: [PATCH 46/75] new line fixes --- .../profiles/v1/alternatives/pprofextended/pprofextended.proto | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto index 1a6d60e07..032b4fe50 100644 --- a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto +++ b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto @@ -44,7 +44,8 @@ import "opentelemetry/proto/common/v1/common.proto"; option go_package = "go.opentelemetry.io/proto/otlp/profiles/v1/alternatives/pprofextended"; -// Represents a complete profile, including sample types, samples, mappings to binaries, locations, functions, string table, and additional metadata. +// Represents a complete profile, including sample types, samples, +// mappings to binaries, locations, functions, string table, and additional metadata. message Profile { // A description of the samples associated with each Sample.value. // For a cpu profile this might be: From b6cf37b3aef643f414adf5d43949f1be7899d272 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Thu, 11 Jan 2024 12:40:57 -0800 Subject: [PATCH 47/75] updates diagram --- opentelemetry/proto/profiles/v1/profiles.proto | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/opentelemetry/proto/profiles/v1/profiles.proto b/opentelemetry/proto/profiles/v1/profiles.proto index fc3b5ab3d..b8a22f481 100644 --- a/opentelemetry/proto/profiles/v1/profiles.proto +++ b/opentelemetry/proto/profiles/v1/profiles.proto @@ -62,12 +62,12 @@ option go_package = "go.opentelemetry.io/proto/otlp/profiles/v1"; // ┌──────────────────┐ // │ Profile │ // └──────────────────┘ -// │ -// │ 1-n -// ▼ -// ┌──────────────────┐ 1-n ┌──────────────┐ -// │ Sample │ ──────▷ │ KeyValue │ -// └──────────────────┘ └──────────────┘ +// │ 1-n +// │ 1-n ┌───────────────────────────────────────┐ +// ▼ │ ▽ +// ┌──────────────────┐ 1-n ┌──────────────┐ ┌──────────┐ +// │ Sample │ ──────▷ │ KeyValue │ │ Link │ +// └──────────────────┘ └──────────────┘ └──────────┘ // │ 1-n △ △ // │ 1-n ┌─────────────────┘ │ 1-n // ▽ │ │ From 9fd6d421aeddcb3e478d7a78cba571254ea76878 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Thu, 11 Jan 2024 14:25:16 -0800 Subject: [PATCH 48/75] support for attribute units --- .../v1/alternatives/pprofextended/pprofextended.proto | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto index 032b4fe50..988073a85 100644 --- a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto +++ b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto @@ -229,11 +229,14 @@ message Sample { // References to attributes in Profile.attribute_table. [optional] repeated uint64 attributes = 7; + // References to attribute units in Profile.string_table. Each number in this array MUST have a matching attribute reference in attributes array. [optional] + repeated uint64 attribute_units = 8; + // References to links in Profile.link_table. [optional] - repeated uint64 links = 8; + repeated uint64 links = 9; // A 128bit id that uniquely identifies this stacktrace, globally. Index into string table. [optional] - uint32 stacktrace_id_index = 9; + uint32 stacktrace_id_index = 10; } // Provides additional context for a sample, From ed6a03672d9768f90ad4f87086e3e145780ab8b7 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Tue, 16 Jan 2024 17:21:27 -0800 Subject: [PATCH 49/75] resolves Alexey's comments: BuildIDKind comments --- .../v1/alternatives/pprofextended/pprofextended.proto | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto index 988073a85..5fcd907e9 100644 --- a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto +++ b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto @@ -258,11 +258,14 @@ message Label { int64 num_unit = 4; // Index into string table } -// Indicates the type of build ID, either from the linker or a binary hash. +// Indicates the semantics of the build_id field. enum BuildIdKind { - // BUILD_ID_LINKER indicates that the build id is linker-provided (usually GNU build ID) + // Linker-generated build ID, stored in the ELF binary notes. BUILD_ID_LINKER = 0; - // BUILD_ID_BINARY_HASH indicates that the build id is a hash of the binary. + // Build ID based on the content hash of the binary. Currently no particular + // hashing approach is standardized, so a given producer needs to define it + // themselves and thus unlike BUILD_ID_LINKER this kind of hash is producer-specific. + // We may choose to provide a standardized stable hash recommendation later. BUILD_ID_BINARY_HASH = 1; } From 69f4b4370750b6bc808873f8ecc42bf4126a3f0a Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Tue, 16 Jan 2024 17:22:00 -0800 Subject: [PATCH 50/75] resolves Alexey's comments: BuildIdKind placement --- .../v1/alternatives/pprofextended/pprofextended.proto | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto index 5fcd907e9..7062d64e2 100644 --- a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto +++ b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto @@ -288,16 +288,13 @@ message Mapping { // with high probability. E.g., for binaries generated by GNU tools, // it could be the contents of the .note.gnu.build-id field. int64 build_id = 6; // Index into string table - + // Specifies the kind of build id. See BuildIdKind enum for more details [optional] + BuildIdKind build_id_kind = 11; // The following fields indicate the resolution of symbolic info. bool has_functions = 7; bool has_filenames = 8; bool has_line_numbers = 9; bool has_inline_frames = 10; - - // Specifies the kind of build id. See BuildIdKind enum for more detaiks [optional] - BuildIdKind build_id_kind = 11; - // References to attributes in Profile.attribute_table. [optional] repeated uint64 attributes = 12; } From d3b0908200d6f5ba9793bcc1e5ca8dd0dbdd1dd6 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Tue, 16 Jan 2024 17:29:09 -0800 Subject: [PATCH 51/75] addresses Alexey's comments - organizes fields logically rather than new ones to the bottom of each struct --- .../pprofextended/pprofextended.proto | 47 +++++++++---------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto index 7062d64e2..47c8370e5 100644 --- a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto +++ b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto @@ -63,10 +63,16 @@ message Profile { repeated Mapping mapping = 3; // Locations referenced by samples via location_indices. repeated Location location = 4; + // Array of locations referenced by samples. + repeated int64 location_indices = 15; // Functions referenced by locations. repeated Function function = 5; // A common table for strings referenced by various messages. // string_table[0] must always be "". + // Lookup table for attributes. + repeated opentelemetry.proto.common.v1.KeyValue attribute_table = 16; + // Lookup table for links. + repeated Link link_table = 17; repeated string string_table = 6; // frames with Function.function_name fully matching the following // regexp will be dropped from the samples, along with their successors. @@ -96,12 +102,6 @@ message Profile { // Index into the string table of the type of the preferred sample // value. If unset, clients should default to the last sample value. int64 default_sample_type = 14; - // Array of locations referenced by samples. - repeated int64 location_indices = 15; - // Lookup table for attributes. - repeated opentelemetry.proto.common.v1.KeyValue attribute_table = 16; - // Lookup table for links. - repeated Link link_table = 17; } // A pointer from a profile Sample to a trace Span. @@ -197,8 +197,16 @@ message ValueType { // indicator of a higher level request being handled etc. message Sample { // The indices recorded here correspond to locations in Profile.location. - // The leaf is at location_index[0]. [deprecated] + // The leaf is at location_index[0]. [deprecated, superseded by locations_start_index / locations_length] repeated uint64 location_index = 1; + // locations_start_index along with locations_length refers to to a slice of locations in Profile.location. + // Supersedes location_index. + uint64 locations_start_index = 7; + // locations_length along with locations_start_index refers to a slice of locations in Profile.location. + // Supersedes location_index. + uint64 locations_length = 8; + // A 128bit id that uniquely identifies this stacktrace, globally. Index into string table. [optional] + uint32 stacktrace_id_index = 9; // The type and unit of each value is defined by the corresponding // entry in Profile.sample_type. All samples must have the same // number of values, the same as the length of Profile.sample_type. @@ -216,27 +224,18 @@ message Sample { // name on a sample. Again, possible to express, but should not be used. // [deprecated, superseded by attributes] repeated Label label = 3; - - // locations_start_index along with locations_length refers to to a slice of locations in Profile.location. Supersedes location_index. - uint64 locations_start_index = 4; - - // locations_length along with locations_start_index refers to a slice of locations in Profile.location. Supersedes location_index. - uint64 locations_length = 5; - - // Timestamps associated with Sample represented in ms. These timestamps are expected to fall within the Profile's time range. [optional] - repeated uint64 timestamps = 6; - // References to attributes in Profile.attribute_table. [optional] - repeated uint64 attributes = 7; + repeated uint64 attributes = 10; // References to attribute units in Profile.string_table. Each number in this array MUST have a matching attribute reference in attributes array. [optional] - repeated uint64 attribute_units = 8; + repeated uint64 attribute_units = 11; // References to links in Profile.link_table. [optional] - repeated uint64 links = 9; + repeated uint64 links = 12; - // A 128bit id that uniquely identifies this stacktrace, globally. Index into string table. [optional] - uint32 stacktrace_id_index = 10; + // Timestamps associated with Sample represented in ms. These timestamps are expected + // to fall within the Profile's time range. [optional] + repeated uint64 timestamps = 13; } // Provides additional context for a sample, @@ -290,13 +289,13 @@ message Mapping { int64 build_id = 6; // Index into string table // Specifies the kind of build id. See BuildIdKind enum for more details [optional] BuildIdKind build_id_kind = 11; + // References to attributes in Profile.attribute_table. [optional] + repeated uint64 attributes = 12; // The following fields indicate the resolution of symbolic info. bool has_functions = 7; bool has_filenames = 8; bool has_line_numbers = 9; bool has_inline_frames = 10; - // References to attributes in Profile.attribute_table. [optional] - repeated uint64 attributes = 12; } // Describes function and line table debug information. From 40f36cb3a950a0c6376ce8f491abe8e188bfc042 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Tue, 16 Jan 2024 18:31:16 -0800 Subject: [PATCH 52/75] Addresses Alexey's comments --- .../v1/alternatives/pprofextended/pprofextended.proto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto index 47c8370e5..c5dba3e1c 100644 --- a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto +++ b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto @@ -230,8 +230,8 @@ message Sample { // References to attribute units in Profile.string_table. Each number in this array MUST have a matching attribute reference in attributes array. [optional] repeated uint64 attribute_units = 11; - // References to links in Profile.link_table. [optional] - repeated uint64 links = 12; + // Reference to link in Profile.link_table. [optional] + uint64 link = 12; // Timestamps associated with Sample represented in ms. These timestamps are expected // to fall within the Profile's time range. [optional] From 99bc8fefaf8695aca84064700edba1c22fad0476 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Tue, 16 Jan 2024 18:36:51 -0800 Subject: [PATCH 53/75] Addresses Alexey's comments RE attribute units --- .../pprofextended/pprofextended.proto | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto index c5dba3e1c..6513784be 100644 --- a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto +++ b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto @@ -44,6 +44,14 @@ import "opentelemetry/proto/common/v1/common.proto"; option go_package = "go.opentelemetry.io/proto/otlp/profiles/v1/alternatives/pprofextended"; +// Represents a mapping between Attribute Keys and Units. +message AttributeUnit { + // Index into string table. + int64 attribute_key = 1; + // Index into string table. + int64 unit = 2; +} + // Represents a complete profile, including sample types, samples, // mappings to binaries, locations, functions, string table, and additional metadata. message Profile { @@ -71,8 +79,10 @@ message Profile { // string_table[0] must always be "". // Lookup table for attributes. repeated opentelemetry.proto.common.v1.KeyValue attribute_table = 16; + // Represents a mapping between Attribute Keys and Units. + repeated AttributeUnit attribute_units = 17; // Lookup table for links. - repeated Link link_table = 17; + repeated Link link_table = 18; repeated string string_table = 6; // frames with Function.function_name fully matching the following // regexp will be dropped from the samples, along with their successors. @@ -227,9 +237,6 @@ message Sample { // References to attributes in Profile.attribute_table. [optional] repeated uint64 attributes = 10; - // References to attribute units in Profile.string_table. Each number in this array MUST have a matching attribute reference in attributes array. [optional] - repeated uint64 attribute_units = 11; - // Reference to link in Profile.link_table. [optional] uint64 link = 12; From c1f281361ff1aa25b6b04c598f70367b75c829f4 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Tue, 16 Jan 2024 19:10:01 -0800 Subject: [PATCH 54/75] addresses comments from @Kielek --- .../profiles/v1/alternatives/pprofextended/pprofextended.proto | 1 + 1 file changed, 1 insertion(+) diff --git a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto index 6513784be..b896f5e0c 100644 --- a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto +++ b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto @@ -42,6 +42,7 @@ package opentelemetry.proto.profiles.v1.alternatives.pprofextended; import "opentelemetry/proto/common/v1/common.proto"; +option csharp_namespace = "OpenTelemetry.Proto.Profiles.V1.Alternatives.PprofExtended"; option go_package = "go.opentelemetry.io/proto/otlp/profiles/v1/alternatives/pprofextended"; // Represents a mapping between Attribute Keys and Units. From 7ec7a8d3b82be530e10a884e828968b404f2133d Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Mon, 22 Jan 2024 18:36:03 -0800 Subject: [PATCH 55/75] ports changes from original pprof: https://github.com/google/pprof/pull/818 --- .../profiles/v1/alternatives/pprofextended/pprofextended.proto | 2 ++ 1 file changed, 2 insertions(+) diff --git a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto index b896f5e0c..013a4ddc2 100644 --- a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto +++ b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto @@ -349,6 +349,8 @@ message Line { uint64 function_index = 1; // Line number in source code. int64 line = 2; + // Column number in source code. + int64 column = 3; } // Describes a function, including its human-readable name, system name, From e94d88854a964a70f822ca59293d88838891e5b9 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Mon, 22 Jan 2024 18:36:25 -0800 Subject: [PATCH 56/75] reorders structs better --- .../pprofextended/pprofextended.proto | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto index 013a4ddc2..787c0e165 100644 --- a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto +++ b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto @@ -45,14 +45,6 @@ import "opentelemetry/proto/common/v1/common.proto"; option csharp_namespace = "OpenTelemetry.Proto.Profiles.V1.Alternatives.PprofExtended"; option go_package = "go.opentelemetry.io/proto/otlp/profiles/v1/alternatives/pprofextended"; -// Represents a mapping between Attribute Keys and Units. -message AttributeUnit { - // Index into string table. - int64 attribute_key = 1; - // Index into string table. - int64 unit = 2; -} - // Represents a complete profile, including sample types, samples, // mappings to binaries, locations, functions, string table, and additional metadata. message Profile { @@ -115,6 +107,14 @@ message Profile { int64 default_sample_type = 14; } +// Represents a mapping between Attribute Keys and Units. +message AttributeUnit { + // Index into string table. + int64 attribute_key = 1; + // Index into string table. + int64 unit = 2; +} + // A pointer from a profile Sample to a trace Span. // Connects a profile sample to a trace span, identified by unique trace and span IDs. message Link { From 33c35687c322fcca2bd621c7b9e082178af09789 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Tue, 23 Jan 2024 09:40:18 -0800 Subject: [PATCH 57/75] fixes a comment for string_table --- .../v1/alternatives/pprofextended/pprofextended.proto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto index 787c0e165..d46502664 100644 --- a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto +++ b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto @@ -68,14 +68,14 @@ message Profile { repeated int64 location_indices = 15; // Functions referenced by locations. repeated Function function = 5; - // A common table for strings referenced by various messages. - // string_table[0] must always be "". // Lookup table for attributes. repeated opentelemetry.proto.common.v1.KeyValue attribute_table = 16; // Represents a mapping between Attribute Keys and Units. repeated AttributeUnit attribute_units = 17; // Lookup table for links. repeated Link link_table = 18; + // A common table for strings referenced by various messages. + // string_table[0] must always be "". repeated string string_table = 6; // frames with Function.function_name fully matching the following // regexp will be dropped from the samples, along with their successors. From 3fc34d8a90aa3051d5b0cc589ce6580bd9403cf5 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Wed, 24 Jan 2024 18:00:35 -0800 Subject: [PATCH 58/75] fixes relationship in diagram --- opentelemetry/proto/profiles/v1/profiles.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opentelemetry/proto/profiles/v1/profiles.proto b/opentelemetry/proto/profiles/v1/profiles.proto index b8a22f481..b02925287 100644 --- a/opentelemetry/proto/profiles/v1/profiles.proto +++ b/opentelemetry/proto/profiles/v1/profiles.proto @@ -71,7 +71,7 @@ option go_package = "go.opentelemetry.io/proto/otlp/profiles/v1"; // │ 1-n △ △ // │ 1-n ┌─────────────────┘ │ 1-n // ▽ │ │ -// ┌──────────────────┐ 1-1 ┌──────────────┐ +// ┌──────────────────┐ n-1 ┌──────────────┐ // │ Location │ ──────▷ │ Mapping │ // └──────────────────┘ └──────────────┘ // │ From 3a3e4d73d2dc4b49d1260799d92f2b1f30ee7f7b Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Wed, 24 Jan 2024 19:28:45 -0800 Subject: [PATCH 59/75] typo --- opentelemetry/proto/profiles/v1/profiles.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opentelemetry/proto/profiles/v1/profiles.proto b/opentelemetry/proto/profiles/v1/profiles.proto index b02925287..fc092d0f0 100644 --- a/opentelemetry/proto/profiles/v1/profiles.proto +++ b/opentelemetry/proto/profiles/v1/profiles.proto @@ -88,7 +88,7 @@ option go_package = "go.opentelemetry.io/proto/otlp/profiles/v1"; // └──────────────────┘ // -// ProfilesData represents the profiles data that can be stored in a persistent storage, +// ProfilesData represents the profiles data that can be stored in persistent storage, // OR can be embedded by other protocols that transfer OTLP profiles data but do not // implement the OTLP protocol. // From d6e6ed8dd11eda65054273a70774feaaa1678490 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Mon, 5 Feb 2024 12:11:40 -0800 Subject: [PATCH 60/75] updates comment --- .../v1/alternatives/pprofextended/pprofextended.proto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto index d46502664..0cd3cef16 100644 --- a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto +++ b/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto @@ -31,10 +31,10 @@ // to a mapping. // - There is a N->1 relationship from sample.location_id entries to // locations. For every sample.location_id entry there must be a -// unique Location with that id. +// unique Location with that index. // - There is an optional N->1 relationship from locations to // mappings. For every nonzero Location.mapping_id there must be a -// unique Mapping with that id. +// unique Mapping with that index. syntax = "proto3"; From d46d02be24f4c96396944028a8813e66fc752999 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Mon, 5 Feb 2024 16:04:29 -0800 Subject: [PATCH 61/75] clarifies profile id --- opentelemetry/proto/profiles/v1/profiles.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opentelemetry/proto/profiles/v1/profiles.proto b/opentelemetry/proto/profiles/v1/profiles.proto index fc092d0f0..0117eaa20 100644 --- a/opentelemetry/proto/profiles/v1/profiles.proto +++ b/opentelemetry/proto/profiles/v1/profiles.proto @@ -140,7 +140,7 @@ message ScopeProfiles { // A ProfileContainer represents a single profile. It wraps pprof profile with OpenTelemetry specific metadata. message ProfileContainer { - // A unique identifier for a profile. The ID is a 16-byte array. An ID with + // A globally unique identifier for a profile. The ID is a 16-byte array. An ID with // all zeroes is considered invalid. // // This field is required. From 3e707317dfe25d9ff3f65ceb1ea8aaebee398cd7 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Tue, 19 Mar 2024 10:57:10 -0700 Subject: [PATCH 62/75] adds experimental profiles support --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 1bc502643..16ca733b7 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ components as indicated by the Maturity table below: | metrics/\*
collector/metrics/* | Stable | [Stable](docs/specification.md#json-protobuf-encoding) | | trace/\*
collector/trace/* | Stable | [Stable](docs/specification.md#json-protobuf-encoding) | | logs/\*
collector/logs/* | Stable | [Stable](docs/specification.md#json-protobuf-encoding) | +| profiles/\*
collector/profiles/* | Experimental | [Experimental](docs/specification.md#json-protobuf-encoding) | (See [maturity-matrix.yaml](https://github.com/open-telemetry/community/blob/47813530864b9fe5a5146f466a58bd2bb94edc72/maturity-matrix.yaml#L57) for definition of maturity levels). From e72e0e314d62955049bf13a103a80359e132abdf Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Tue, 19 Mar 2024 10:51:27 -0700 Subject: [PATCH 63/75] deletes decision log --- .../proto/profiles/v1/decision-log.md | 168 ------------------ 1 file changed, 168 deletions(-) delete mode 100644 opentelemetry/proto/profiles/v1/decision-log.md diff --git a/opentelemetry/proto/profiles/v1/decision-log.md b/opentelemetry/proto/profiles/v1/decision-log.md deleted file mode 100644 index 6c1f4fb84..000000000 --- a/opentelemetry/proto/profiles/v1/decision-log.md +++ /dev/null @@ -1,168 +0,0 @@ -# Decision #1 -Spec must include an optional field for the original payload (e.g original JFR/pprof profile) - -### Date -Jun 01 2023 - -### Context -Some formats (JFR) have advanced functinality that is too hard to generalize (e.g in JFR you can include custom events in your profiles) - -### Alternatives Considered -* try to generalize it anyway, so that OTEL format is a subset of JFR -* discard such data - -### Reasoning -* generalizing it is too hard, JFR allows for pretty much any kind of message -* discarding is not good, some percentage of users depend on custom JFR events - -### Details -[SIG Meeting Notes from June 1 2023](https://docs.google.com/document/d/19UqPPPlGE83N37MhS93uRlxsP1_wGxQ33Qv6CDHaEp0/edit?usp=sharing) - - - - - -# Decision #2 -Use normalized representation of profiling data - -### Date -Jun 29 2023 - -### Context -Profiling data is different than other signals in a way that it has heavy structures with many links between them. - -### Alternatives Considered -Use denormalized represenation of profiling data - -### Reasoning -Because profiling data has many heavy structures with many links between them, denormalized representation takes 10x more space compared to normalized version where each object references other objects by index - -### Details -[Github Conversation](https://github.com/open-telemetry/opentelemetry-proto/pull/488#discussion_r1246163710) -[Github Conversation](https://github.com/open-telemetry/opentelemetry-proto/pull/488#pullrequestreview-1505265763) - - - - - -# Decision #3 -When referring to other structs use array indexes for references instead of ids - -### Date -Jun 29 2023 - -### Context -in pprof references happen using ids, e.g {function_id=123}. this leads to confusion, as these ids almost always match array indexes. e.g "function" with id=1 will usually be in functions array at index 1. - -### Alternatives Considered -Using ids instead of indexes - -### Reasoning -* reduces confusion -* reduces serialized size -* improves performance since array lookup is faster than search (O(1) vs O(n) - -### Details -[Github Conversation](https://github.com/open-telemetry/opentelemetry-proto/pull/488#discussion_r1248116971) - - - - - -# Decision #4 -Spec must include an optional Mapping struct from pprof - -### Date -Jun 29 2023 - -### Context -pprof includes Mapping struct that describes relationships between functions and where those functions are located in the binary. This struct is rarely used in client implementations - -### Alternatives Considered -Not include it to simplify spec - -### Reasoning -It is useful when symbol information is supplied separately, e.g very common in linux - -### Details -[SIG Meeting Notes from Jun 29 2023](https://docs.google.com/document/d/19UqPPPlGE83N37MhS93uRlxsP1_wGxQ33Qv6CDHaEp0/edit?usp=sharing) - - - - - -# Decision #5 -Internal references should be uint32 - -### Date -Jun 29 2023 - -### Context -there's many internal reference fields, e.g: -* location_id -* function_id -if they are > 32 bits then the payload is going to be too huge anyway - -### Alternatives Considered -* use uint64 - - -### Reasoning -* uint32 is big enough -* but it's better than uint64 because it will reduce struct size in go -* this will reduct collector memory footprint - -### Details -[Github Conversation](https://github.com/open-telemetry/opentelemetry-proto/pull/488/files#r1246155907) - - - - - -# Decision #6 -Arrays of integers should be used instead of arrays of structs - -### Date -Aug 23 2023 - -### Context - -Samples are a collection of references to other messages plus a value. The standard way of representing those is to put each `Sample` into a separate message, and link from `Sample` to other messages. Parsing / generating such payloads creates many individual objects that runtime has to track. - -An "arrays of integers" representation puts values of the same kind into separate arrays. This reduces the size of the resulting protobuf payload and the number of objects that need to be allocated to parse / generate such payload. - -Here's pseudocode that illustrates the approach. Note that this example is simplified for the sake of clarity: - -``` -// normalized -"samples": [ - { - "stacktrace_id": 1, - "value": 100 - }, { - "stacktrace_id": 2, - "value": 200 - } -], - -// arrays -"stacktrace_ids": [1, 2], -"values": [100, 200] -``` - -Benchmarking shows that this approach is more efficient in terms of CPU utilization, memory consumption and size of the resulting protobuf payload. - -### Alternatives Considered - -`normalized` represenation where we don't use arrays of integers, but arrays of structures instead - - -### Reasoning - -* `arrays` representation is the most efficient in terms of CPU utilization, memory consumption and size of the resulting protobuf payload. It is however cognitively more complex to implement and understand. -* as presented in [Design Goals](#design-goals) section, the performance characteristics of the format are very important for the profiling signal -* the format is not intended to be used directly by the end users, but rather by the developers of profiling systems that are used to and are expected to be able to handle the complexity. It is not more complex than other existing formats - -### Details - -* [Benchmarking Results](https://docs.google.com/spreadsheets/d/1Q-6MlegV8xLYdz5WD5iPxQU2tsfodX1-CDV1WeGzyQ0/edit?usp=sharing) From ab21da785cf1aef326d14059574d86ccba24c55a Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Tue, 19 Mar 2024 10:49:01 -0700 Subject: [PATCH 64/75] rolls back README changes --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 16ca733b7..b2bc40361 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ and the corresponding Language Independent Interface Types ([.proto files](opent ## Language Independent Interface Types -The `.proto` files can be consumed as GIT submodules or copied and built directly into the consumer project. +The proto files can be consumed as GIT submodules or copied and built directly in the consumer project. The compiled files are published to central repositories (Maven, ...) from OpenTelemetry client libraries. @@ -15,11 +15,11 @@ See [contribution guidelines](CONTRIBUTING.md) if you would like to make any cha ## OTLP/JSON -For additional requirements regarding the OTLP/JSON wire representation, refer to [OTLP/JSON wire representation here](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/otlp.md#json-protobuf-encoding). +See additional requirements for [OTLP/JSON wire representation here](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/otlp.md#json-protobuf-encoding). ## Generate gRPC Client Libraries -To generate the raw gRPC client libraries, use the command `make gen-${LANGUAGE}`. The currently supported languages are: +To generate the raw gRPC client libraries, use `make gen-${LANGUAGE}`. Currently supported languages are: * cpp * csharp @@ -33,8 +33,8 @@ To generate the raw gRPC client libraries, use the command `make gen-${LANGUAGE} ## Maturity Level -Releases 1.0.0 and newer releases from this repository may contain unstable (alpha or beta) -components as indicated by the Maturity table below: +1.0.0 and newer releases from this repository may contain unstable (alpha or beta) +components as indicated by the Maturity table below. | Component | Binary Protobuf Maturity | JSON Maturity | | --------- |--------------- | ------------- | @@ -57,7 +57,7 @@ Components marked `Stable` provide the following guarantees: - Service method names will not change. [from 1.0.0] - Service method parameter names will not change. [from 1.0.0] - Service method parameter types and return types will not change. [from 1.0.0] -- Service method kind (unary vs. streaming) will not change. +- Service method kind (unary vs streaming) will not change. - Names of messages and enums will not change. [from 1.0.0] - Numbers assigned to enum choices will not change. - Names of enum choices will not change. [from 1.0.0] @@ -67,7 +67,7 @@ Components marked `Stable` provide the following guarantees: - `optional` and `repeated` declarators of existing fields will not change. [from 1.0.0] - No existing symbol will be deleted. [from 1.0.0] -Note: The guarantees marked as [from 1.0.0] will go into effect when this repository is tagged +Note: guarantees marked [from 1.0.0] will go into effect when this repository is tagged with version number 1.0.0. The following additive changes are allowed: @@ -88,7 +88,7 @@ is generated from the .proto files by any particular code generator. ## Experiments -In some cases, we are trying to experiment with different features. In this case, +In some cases we are trying to experiment with different features. In this case, we recommend using an "experimental" sub-directory instead of adding them to any protocol version. These protocols should not be used, except for development/testing purposes. From 17592eaf37646f6dccc7824dd867ea65a2b288f1 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Tue, 19 Mar 2024 11:00:31 -0700 Subject: [PATCH 65/75] resets common.proto to original version --- opentelemetry/proto/common/v1/common.proto | 49 +--------------------- 1 file changed, 1 insertion(+), 48 deletions(-) diff --git a/opentelemetry/proto/common/v1/common.proto b/opentelemetry/proto/common/v1/common.proto index 58963bba1..ff8a21a1f 100644 --- a/opentelemetry/proto/common/v1/common.proto +++ b/opentelemetry/proto/common/v1/common.proto @@ -67,7 +67,7 @@ message KeyValue { } // InstrumentationScope is a message representing the instrumentation scope information -// such as the fully qualified name and version. +// such as the fully qualified name and version. message InstrumentationScope { // An empty instrumentation scope name means the name is unknown. string name = 1; @@ -79,50 +79,3 @@ message InstrumentationScope { repeated KeyValue attributes = 3; uint32 dropped_attributes_count = 4; } - -// THE FOLLOWING "Interned" MESSAGES ARE TEMPORARY AND WILL BE REMOVED. ONLY USED FOR BENCHMARKS - -// AnyValueInterned is used to represent any type of attribute value. AnyValueInterned may contain a -// primitive value such as a string or integer or it may contain an arbitrary nested -// object containing arrays, key-value lists and primitives. -message AnyValueInterned { - // The value is one of the listed fields. It is valid for all values to be unspecified - // in which case this AnyValueInterned is considered to be "empty". - oneof value { - int64 string_value = 1; // Index into the string_table. - bool bool_value = 2; - int64 int_value = 3; - double double_value = 4; - ArrayValueInterned array_value = 5; - KeyValueListInterned kvlist_value = 6; - int64 bytes_value = 7; // Index into the bytes_table. - } -} - -// ArrayValue is a list of AnyValueInterned messages. We need ArrayValue as a message -// since oneof in AnyValueInterned does not allow repeated fields. -message ArrayValueInterned { - // Array of values. The array may be empty (contain 0 elements). - repeated AnyValueInterned values = 1; -} - -// KeyValueListInterned is a list of KeyValueInterned messages. We need KeyValueListInterned as a message -// since `oneof` in AnyValueInterned does not allow repeated fields. Everywhere else where we need -// a list of KeyValueInterned messages (e.g. in Span) we use `repeated KeyValueInterned` directly to -// avoid unnecessary extra wrapping (which slows down the protocol). The 2 approaches -// are semantically equivalent. -message KeyValueListInterned { - // A collection of key/value pairs of key-value pairs. The list may be empty (may - // contain 0 elements). - // The keys MUST be unique (it is not allowed to have more than one - // value with the same key). - repeated KeyValueInterned values = 1; -} - -// KeyValueInterned is a key-value pair that is used to store Span attributes, Link -// attributes, etc. -message KeyValueInterned { - int64 key = 1; // Index into the string_table. - AnyValueInterned value = 2; - int64 unit = 3; // Index into the string_table. -} From f4de0462413adda4b7ece0d46e7b224340c5b23d Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Tue, 19 Mar 2024 11:17:30 -0700 Subject: [PATCH 66/75] cleans up copyrights and file paths --- .../pprofextended => }/pprofextended.proto | 18 +++++++++++++++++- opentelemetry/proto/profiles/v1/profiles.proto | 14 +------------- 2 files changed, 18 insertions(+), 14 deletions(-) rename opentelemetry/proto/profiles/v1/{alternatives/pprofextended => }/pprofextended.proto (95%) diff --git a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto b/opentelemetry/proto/profiles/v1/pprofextended.proto similarity index 95% rename from opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto rename to opentelemetry/proto/profiles/v1/pprofextended.proto index 0cd3cef16..1fb9458ff 100644 --- a/opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto +++ b/opentelemetry/proto/profiles/v1/pprofextended.proto @@ -1,3 +1,19 @@ +// Copyright 2023, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// This file includes work covered by the following copyright and permission notices: +// // Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -38,7 +54,7 @@ syntax = "proto3"; -package opentelemetry.proto.profiles.v1.alternatives.pprofextended; +package opentelemetry.proto.profiles.v1.pprofextended; import "opentelemetry/proto/common/v1/common.proto"; diff --git a/opentelemetry/proto/profiles/v1/profiles.proto b/opentelemetry/proto/profiles/v1/profiles.proto index 8636002f5..b476cacd2 100644 --- a/opentelemetry/proto/profiles/v1/profiles.proto +++ b/opentelemetry/proto/profiles/v1/profiles.proto @@ -1,5 +1,4 @@ // Copyright 2023, OpenTelemetry Authors -// Copyright 2016 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -189,16 +188,5 @@ message ProfileContainer { bytes original_payload = 7; // This is a reference to a pprof profile. Required, even when original_payload is present. - opentelemetry.proto.profiles.v1.alternatives.pprofextended.Profile profile = 8; - - // THIS IS TEMPORARY AND WILL BE REMOVED. ONLY USED FOR BENCHMARKS - // this is temporary until we pick one of these formats - // fields from one of these will be embedded in Profile message: - oneof alternative_profile { - opentelemetry.proto.profiles.v1.alternatives.pprof.Profile pprof = 101; - opentelemetry.proto.profiles.v1.alternatives.normalized.Profile normalized = 102; - opentelemetry.proto.profiles.v1.alternatives.arrays.Profile arrays = 103; - opentelemetry.proto.profiles.v1.alternatives.denormalized.Profile denormalized = 104; - opentelemetry.proto.profiles.v1.alternatives.pprofextended.Profile pprofextended = 105; - } + opentelemetry.proto.profiles.v1.pprofextended.Profile profile = 8; } From 845b17a7adae6255fb47be79291230735242ded6 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Tue, 19 Mar 2024 11:49:40 -0700 Subject: [PATCH 67/75] updates to paths and such --- opentelemetry/proto/profiles/v1/pprofextended.proto | 6 +++--- opentelemetry/proto/profiles/v1/profiles.proto | 11 ++--------- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/opentelemetry/proto/profiles/v1/pprofextended.proto b/opentelemetry/proto/profiles/v1/pprofextended.proto index 1fb9458ff..79128c492 100644 --- a/opentelemetry/proto/profiles/v1/pprofextended.proto +++ b/opentelemetry/proto/profiles/v1/pprofextended.proto @@ -54,12 +54,12 @@ syntax = "proto3"; -package opentelemetry.proto.profiles.v1.pprofextended; +package opentelemetry.proto.profiles.v1; import "opentelemetry/proto/common/v1/common.proto"; -option csharp_namespace = "OpenTelemetry.Proto.Profiles.V1.Alternatives.PprofExtended"; -option go_package = "go.opentelemetry.io/proto/otlp/profiles/v1/alternatives/pprofextended"; +option csharp_namespace = "OpenTelemetry.Proto.Profiles.V1.PprofExtended"; +option go_package = "go.opentelemetry.io/proto/otlp/profiles/v1/pprofextended"; // Represents a complete profile, including sample types, samples, // mappings to binaries, locations, functions, string table, and additional metadata. diff --git a/opentelemetry/proto/profiles/v1/profiles.proto b/opentelemetry/proto/profiles/v1/profiles.proto index b476cacd2..9c2d52145 100644 --- a/opentelemetry/proto/profiles/v1/profiles.proto +++ b/opentelemetry/proto/profiles/v1/profiles.proto @@ -18,14 +18,7 @@ package opentelemetry.proto.profiles.v1; import "opentelemetry/proto/common/v1/common.proto"; import "opentelemetry/proto/resource/v1/resource.proto"; - -// TODO(@petethepig): idea here is that we will eventually get rid of these -// and have just version in this (profiles.proto) file -import "opentelemetry/proto/profiles/v1/alternatives/pprof/pprof.proto"; -import "opentelemetry/proto/profiles/v1/alternatives/denormalized/denormalized.proto"; -import "opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto"; -import "opentelemetry/proto/profiles/v1/alternatives/arrays/arrays.proto"; -import "opentelemetry/proto/profiles/v1/alternatives/pprofextended/pprofextended.proto"; +import "opentelemetry/proto/profiles/v1/pprofextended.proto"; option csharp_namespace = "OpenTelemetry.Proto.Profiles.V1"; option java_multiple_files = true; @@ -188,5 +181,5 @@ message ProfileContainer { bytes original_payload = 7; // This is a reference to a pprof profile. Required, even when original_payload is present. - opentelemetry.proto.profiles.v1.pprofextended.Profile profile = 8; + opentelemetry.proto.profiles.v1.Profile profile = 8; } From 876093e564577e4ec7c9a91f3ae949cc31bc4875 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Tue, 26 Mar 2024 19:22:06 -0700 Subject: [PATCH 68/75] v1 -> v1experimental --- .../{v1 => v1experimental}/profiles_service.proto | 6 +++--- .../profiles_service_http.yaml | 2 +- .../{v1 => v1experimental}/pprofextended.proto | 6 +++--- .../profiles/{v1 => v1experimental}/profiles.proto | 12 ++++++------ 4 files changed, 13 insertions(+), 13 deletions(-) rename opentelemetry/proto/collector/profiles/{v1 => v1experimental}/profiles_service.proto (94%) rename opentelemetry/proto/collector/profiles/{v1 => v1experimental}/profiles_service_http.yaml (89%) rename opentelemetry/proto/profiles/{v1 => v1experimental}/pprofextended.proto (98%) rename opentelemetry/proto/profiles/{v1 => v1experimental}/profiles.proto (95%) diff --git a/opentelemetry/proto/collector/profiles/v1/profiles_service.proto b/opentelemetry/proto/collector/profiles/v1experimental/profiles_service.proto similarity index 94% rename from opentelemetry/proto/collector/profiles/v1/profiles_service.proto rename to opentelemetry/proto/collector/profiles/v1experimental/profiles_service.proto index bfbd0aa29..f5316ef66 100644 --- a/opentelemetry/proto/collector/profiles/v1/profiles_service.proto +++ b/opentelemetry/proto/collector/profiles/v1experimental/profiles_service.proto @@ -1,4 +1,4 @@ -// Copyright 2019, OpenTelemetry Authors +// Copyright 2023, OpenTelemetry Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ syntax = "proto3"; package opentelemetry.proto.collector.profiles.v1; -import "opentelemetry/proto/profiles/v1/profiles.proto"; +import "opentelemetry/proto/profiles/v1experimental/profiles.proto"; option csharp_namespace = "OpenTelemetry.Proto.Collector.Profiles.V1"; option java_multiple_files = true; @@ -39,7 +39,7 @@ message ExportProfilesServiceRequest { // element. Intermediary nodes (such as OpenTelemetry Collector) that receive // data from multiple origins typically batch the data before forwarding further and // in that case this array will contain multiple elements. - repeated opentelemetry.proto.profiles.v1.ResourceProfiles resource_profiles = 1; + repeated opentelemetry.proto.profiles.v1experimental.ResourceProfiles resource_profiles = 1; } message ExportProfilesServiceResponse { diff --git a/opentelemetry/proto/collector/profiles/v1/profiles_service_http.yaml b/opentelemetry/proto/collector/profiles/v1experimental/profiles_service_http.yaml similarity index 89% rename from opentelemetry/proto/collector/profiles/v1/profiles_service_http.yaml rename to opentelemetry/proto/collector/profiles/v1experimental/profiles_service_http.yaml index eed628920..6bb7cf740 100644 --- a/opentelemetry/proto/collector/profiles/v1/profiles_service_http.yaml +++ b/opentelemetry/proto/collector/profiles/v1experimental/profiles_service_http.yaml @@ -5,5 +5,5 @@ config_version: 3 http: rules: - selector: opentelemetry.proto.collector.profiles.v1.ProfilesService.Export - post: /v1/profiles + post: /v1experimental/profiles body: "*" diff --git a/opentelemetry/proto/profiles/v1/pprofextended.proto b/opentelemetry/proto/profiles/v1experimental/pprofextended.proto similarity index 98% rename from opentelemetry/proto/profiles/v1/pprofextended.proto rename to opentelemetry/proto/profiles/v1experimental/pprofextended.proto index 79128c492..3ff8ab59d 100644 --- a/opentelemetry/proto/profiles/v1/pprofextended.proto +++ b/opentelemetry/proto/profiles/v1experimental/pprofextended.proto @@ -54,12 +54,12 @@ syntax = "proto3"; -package opentelemetry.proto.profiles.v1; +package opentelemetry.proto.profiles.v1experimental; import "opentelemetry/proto/common/v1/common.proto"; -option csharp_namespace = "OpenTelemetry.Proto.Profiles.V1.PprofExtended"; -option go_package = "go.opentelemetry.io/proto/otlp/profiles/v1/pprofextended"; +option csharp_namespace = "OpenTelemetry.Proto.Profiles.V1Experimental"; +option go_package = "go.opentelemetry.io/proto/otlp/profiles/v1experimental"; // Represents a complete profile, including sample types, samples, // mappings to binaries, locations, functions, string table, and additional metadata. diff --git a/opentelemetry/proto/profiles/v1/profiles.proto b/opentelemetry/proto/profiles/v1experimental/profiles.proto similarity index 95% rename from opentelemetry/proto/profiles/v1/profiles.proto rename to opentelemetry/proto/profiles/v1experimental/profiles.proto index 9c2d52145..69dc41aa3 100644 --- a/opentelemetry/proto/profiles/v1/profiles.proto +++ b/opentelemetry/proto/profiles/v1experimental/profiles.proto @@ -14,17 +14,17 @@ syntax = "proto3"; -package opentelemetry.proto.profiles.v1; +package opentelemetry.proto.profiles.v1experimental; import "opentelemetry/proto/common/v1/common.proto"; import "opentelemetry/proto/resource/v1/resource.proto"; -import "opentelemetry/proto/profiles/v1/pprofextended.proto"; +import "opentelemetry/proto/profiles/v1experimental/pprofextended.proto"; -option csharp_namespace = "OpenTelemetry.Proto.Profiles.V1"; +option csharp_namespace = "OpenTelemetry.Proto.Profiles.V1Experimental"; option java_multiple_files = true; -option java_package = "io.opentelemetry.proto.profiles.v1"; +option java_package = "io.opentelemetry.proto.profiles.v1experimental"; option java_outer_classname = "ProfilesProto"; -option go_package = "go.opentelemetry.io/proto/otlp/profiles/v1"; +option go_package = "go.opentelemetry.io/proto/otlp/profiles/v1experimental"; // Relationships Diagram // @@ -181,5 +181,5 @@ message ProfileContainer { bytes original_payload = 7; // This is a reference to a pprof profile. Required, even when original_payload is present. - opentelemetry.proto.profiles.v1.Profile profile = 8; + opentelemetry.proto.profiles.v1experimental.Profile profile = 8; } From e1d0492b6bab9e38d7e4513412cbb9797738cec1 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Tue, 26 Mar 2024 19:24:51 -0700 Subject: [PATCH 69/75] more v1 -> v1experimental --- .../profiles/v1experimental/profiles_service.proto | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/opentelemetry/proto/collector/profiles/v1experimental/profiles_service.proto b/opentelemetry/proto/collector/profiles/v1experimental/profiles_service.proto index f5316ef66..a4db43045 100644 --- a/opentelemetry/proto/collector/profiles/v1experimental/profiles_service.proto +++ b/opentelemetry/proto/collector/profiles/v1experimental/profiles_service.proto @@ -14,15 +14,15 @@ syntax = "proto3"; -package opentelemetry.proto.collector.profiles.v1; +package opentelemetry.proto.collector.profiles.v1experimental; import "opentelemetry/proto/profiles/v1experimental/profiles.proto"; -option csharp_namespace = "OpenTelemetry.Proto.Collector.Profiles.V1"; +option csharp_namespace = "OpenTelemetry.Proto.Collector.Profiles.V1Experimental"; option java_multiple_files = true; -option java_package = "io.opentelemetry.proto.collector.profiles.v1"; +option java_package = "io.opentelemetry.proto.collector.profiles.v1experimental"; option java_outer_classname = "ProfilesServiceProto"; -option go_package = "go.opentelemetry.io/proto/otlp/collector/profiles/v1"; +option go_package = "go.opentelemetry.io/proto/otlp/collector/profiles/v1experimental"; // Service that can be used to push profiles between one Application instrumented with // OpenTelemetry and a collector, or between a collector and a central collector (in this From 5c04661af1545220c969a78caf8f9b5033fe7de5 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Wed, 17 Apr 2024 20:53:00 -0700 Subject: [PATCH 70/75] Update opentelemetry/proto/collector/profiles/v1experimental/profiles_service.proto Co-authored-by: jack-berg <34418638+jack-berg@users.noreply.github.com> --- .../collector/profiles/v1experimental/profiles_service.proto | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/opentelemetry/proto/collector/profiles/v1experimental/profiles_service.proto b/opentelemetry/proto/collector/profiles/v1experimental/profiles_service.proto index a4db43045..d0e7894b2 100644 --- a/opentelemetry/proto/collector/profiles/v1experimental/profiles_service.proto +++ b/opentelemetry/proto/collector/profiles/v1experimental/profiles_service.proto @@ -25,8 +25,7 @@ option java_outer_classname = "ProfilesServiceProto"; option go_package = "go.opentelemetry.io/proto/otlp/collector/profiles/v1experimental"; // Service that can be used to push profiles between one Application instrumented with -// OpenTelemetry and a collector, or between a collector and a central collector (in this -// case spans are sent/received to/from multiple Applications). +// OpenTelemetry and a collector, or between a collector and a central collector. service ProfilesService { // For performance reasons, it is recommended to keep this RPC // alive for the entire life of the application. From b69a9b35d3f0a5944bbce2f17f20bd5a085dbc1f Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Wed, 17 Apr 2024 20:53:28 -0700 Subject: [PATCH 71/75] Update opentelemetry/proto/profiles/v1experimental/profiles.proto Co-authored-by: jack-berg <34418638+jack-berg@users.noreply.github.com> --- opentelemetry/proto/profiles/v1experimental/profiles.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opentelemetry/proto/profiles/v1experimental/profiles.proto b/opentelemetry/proto/profiles/v1experimental/profiles.proto index 69dc41aa3..d5ba57b42 100644 --- a/opentelemetry/proto/profiles/v1experimental/profiles.proto +++ b/opentelemetry/proto/profiles/v1experimental/profiles.proto @@ -117,7 +117,7 @@ message ResourceProfiles { string schema_url = 3; } -// A collection of Profiles produced by an InstrumentationScope. +// A collection of ProfileContainers produced by an InstrumentationScope. message ScopeProfiles { // The instrumentation scope information for the profiles in this message. // Semantically when InstrumentationScope isn't set, it is equivalent with From 97cb64e82dd00c437fa30945a2303916d8db50ea Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Wed, 17 Apr 2024 21:33:40 -0700 Subject: [PATCH 72/75] Add schema URLs for resource and scope profiles --- .../proto/profiles/v1experimental/profiles.proto | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/opentelemetry/proto/profiles/v1experimental/profiles.proto b/opentelemetry/proto/profiles/v1experimental/profiles.proto index d5ba57b42..bbc2b2931 100644 --- a/opentelemetry/proto/profiles/v1experimental/profiles.proto +++ b/opentelemetry/proto/profiles/v1experimental/profiles.proto @@ -112,6 +112,9 @@ message ResourceProfiles { // A list of ScopeProfiles that originate from a resource. repeated ScopeProfiles scope_profiles = 2; + // The Schema URL, if known. This is the identifier of the Schema that the resource data + // is recorded in. To learn more about Schema URL see + // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url // This schema_url applies to the data in the "resource" field. It does not apply // to the data in the "scope_profiles" field which have their own schema_url field. string schema_url = 3; @@ -127,7 +130,10 @@ message ScopeProfiles { // A list of ProfileContainers that originate from an instrumentation scope. repeated ProfileContainer profiles = 2; - // This schema_url applies to all profiles and profile events in the "profiles" field. + // The Schema URL, if known. This is the identifier of the Schema that the metric data + // is recorded in. To learn more about Schema URL see + // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url + // This schema_url applies to all profiles in the "profiles" field. string schema_url = 3; } From f5b589717f79ddb5d143c9a78976f315e637cf04 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Wed, 17 Apr 2024 22:13:00 -0700 Subject: [PATCH 73/75] Update timestamps to nanoseconds in Sample message --- opentelemetry/proto/profiles/v1experimental/pprofextended.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opentelemetry/proto/profiles/v1experimental/pprofextended.proto b/opentelemetry/proto/profiles/v1experimental/pprofextended.proto index 3ff8ab59d..17b3896d7 100644 --- a/opentelemetry/proto/profiles/v1experimental/pprofextended.proto +++ b/opentelemetry/proto/profiles/v1experimental/pprofextended.proto @@ -257,7 +257,7 @@ message Sample { // Reference to link in Profile.link_table. [optional] uint64 link = 12; - // Timestamps associated with Sample represented in ms. These timestamps are expected + // Timestamps associated with Sample represented in nanoseconds. These timestamps are expected // to fall within the Profile's time range. [optional] repeated uint64 timestamps = 13; } From 8acbb5ab478e68757e4a9d82122e9509b1ece593 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Thu, 18 Apr 2024 17:58:27 -0700 Subject: [PATCH 74/75] Update opentelemetry/proto/profiles/v1experimental/pprofextended.proto Co-authored-by: jack-berg <34418638+jack-berg@users.noreply.github.com> --- opentelemetry/proto/profiles/v1experimental/pprofextended.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opentelemetry/proto/profiles/v1experimental/pprofextended.proto b/opentelemetry/proto/profiles/v1experimental/pprofextended.proto index 17b3896d7..d5e7c86bc 100644 --- a/opentelemetry/proto/profiles/v1experimental/pprofextended.proto +++ b/opentelemetry/proto/profiles/v1experimental/pprofextended.proto @@ -259,7 +259,7 @@ message Sample { // Timestamps associated with Sample represented in nanoseconds. These timestamps are expected // to fall within the Profile's time range. [optional] - repeated uint64 timestamps = 13; + repeated uint64 unix_nano_times = 13; } // Provides additional context for a sample, From ff7d7e04cee6d9734b6ca0bb4a732fbb8dbfeecd Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Thu, 18 Apr 2024 18:02:12 -0700 Subject: [PATCH 75/75] Update opentelemetry/proto/profiles/v1experimental/pprofextended.proto --- opentelemetry/proto/profiles/v1experimental/pprofextended.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opentelemetry/proto/profiles/v1experimental/pprofextended.proto b/opentelemetry/proto/profiles/v1experimental/pprofextended.proto index d5e7c86bc..bd3008355 100644 --- a/opentelemetry/proto/profiles/v1experimental/pprofextended.proto +++ b/opentelemetry/proto/profiles/v1experimental/pprofextended.proto @@ -259,7 +259,7 @@ message Sample { // Timestamps associated with Sample represented in nanoseconds. These timestamps are expected // to fall within the Profile's time range. [optional] - repeated uint64 unix_nano_times = 13; + repeated uint64 timestamps_unix_nano = 13; } // Provides additional context for a sample,