From 0c3571c98c238780b553128131b220672956d2a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Geisend=C3=B6rfer?= Date: Thu, 26 Jan 2023 18:35:27 +0100 Subject: [PATCH 1/7] WIP: add profiling signal --- .../profiling/v1/profiling_service.proto | 65 ++++++++++ .../profiling/v1/trace_service_http.yaml | 9 ++ .../proto/profiling/v1/profiling.proto | 120 ++++++++++++++++++ 3 files changed, 194 insertions(+) create mode 100644 opentelemetry/proto/collector/profiling/v1/profiling_service.proto create mode 100644 opentelemetry/proto/collector/profiling/v1/trace_service_http.yaml create mode 100644 opentelemetry/proto/profiling/v1/profiling.proto diff --git a/opentelemetry/proto/collector/profiling/v1/profiling_service.proto b/opentelemetry/proto/collector/profiling/v1/profiling_service.proto new file mode 100644 index 000000000..d074cfac9 --- /dev/null +++ b/opentelemetry/proto/collector/profiling/v1/profiling_service.proto @@ -0,0 +1,65 @@ +// Copyright 2019, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package opentelemetry.proto.collector.profiling.v1; + +import "opentelemetry/proto/profiling/v1/profiling.proto"; + +option csharp_namespace = "OpenTelemetry.Proto.Collector.Profiling.V1"; +option java_multiple_files = true; +option java_package = "io.opentelemetry.proto.collector.profiling.v1"; +option java_outer_classname = "ProfilingServiceProto"; +option go_package = "go.opentelemetry.io/proto/otlp/collector/profiling/v1"; + +// Service that can be used to push profiles between one Application instrumented with +// OpenTelemetry and a collector, or between a collector and a central collector (in this +// case spans are sent/received to/from multiple Applications). +service ProfilingService { + // For performance reasons, it is recommended to keep this RPC + // alive for the entire life of the application. + rpc Export(ExportProfilingServiceRequest) returns (ExportProfilingServiceResponse) {} +} + +message ExportProfilingServiceRequest { + // An array of ResourceProfiles. + // For data coming from a single resource this array will typically contain one + // element. Intermediary nodes (such as OpenTelemetry Collector) that receive + // data from multiple origins typically batch the data before forwarding further and + // in that case this array will contain multiple elements. + repeated opentelemetry.proto.profiling.v1.ResourceProfiles resource_profiles = 1; +} + +message ExportProfilingServiceResponse { + // The details of a partially successful export request. + // + // If the request is only partially accepted + // (i.e. when the server accepts only parts of the data and rejects the rest) + // the server MUST initialize the `partial_success` field and MUST + // set the `rejected_` with the number of items it rejected. + // + // Servers MAY also make use of the `partial_success` field to convey + // warnings/suggestions to senders even when the request was fully accepted. + // In such cases, the `rejected_` MUST have a value of `0` and + // the `error_message` MUST be non-empty. + // + // A `partial_success` message with an empty value (rejected_ = 0 and + // `error_message` = "") is equivalent to it not being set/present. Senders + // SHOULD interpret it the same way as in the full success case. + ExportProfilingPartialSuccess partial_success = 1; +} + +message ExportProfilingPartialSuccess { +} diff --git a/opentelemetry/proto/collector/profiling/v1/trace_service_http.yaml b/opentelemetry/proto/collector/profiling/v1/trace_service_http.yaml new file mode 100644 index 000000000..287473597 --- /dev/null +++ b/opentelemetry/proto/collector/profiling/v1/trace_service_http.yaml @@ -0,0 +1,9 @@ +# This is an API configuration to generate an HTTP/JSON -> gRPC gateway for the +# OpenTelemetry service using github.com/grpc-ecosystem/grpc-gateway. +type: google.api.Service +config_version: 3 +http: + rules: + - selector: opentelemetry.proto.collector.trace.v1.TraceService.Export + post: /v1/trace + body: "*" \ No newline at end of file diff --git a/opentelemetry/proto/profiling/v1/profiling.proto b/opentelemetry/proto/profiling/v1/profiling.proto new file mode 100644 index 000000000..ed665d4bf --- /dev/null +++ b/opentelemetry/proto/profiling/v1/profiling.proto @@ -0,0 +1,120 @@ +// Copyright 2019, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package opentelemetry.proto.profiling.v1; + +import "opentelemetry/proto/common/v1/common.proto"; +import "opentelemetry/proto/resource/v1/resource.proto"; + +option csharp_namespace = "OpenTelemetry.Proto.Profiling.V1"; +option java_multiple_files = true; +option java_package = "io.opentelemetry.proto.profiling.v1"; +option java_outer_classname = "ProfilingProto"; +option go_package = "go.opentelemetry.io/proto/otlp/profiling/v1"; + + +// A collection of ScopeProfiles from a Resource. +message ResourceProfiles { + reserved 1000; + + // The resource for the profiles in this message. + // If this field is not set then no resource info is known. + opentelemetry.proto.resource.v1.Resource resource = 1; + + // A list of ScopeProfiles that originate from a resource. + repeated ScopeProfiles scope_profiles = 2; + + // This schema_url applies to the data in the "resource" field. It does not apply + // to the data in the "scope_profiles" field which have their own schema_url field. + string schema_url = 3; +} + +// A collection of Profiles produced by an InstrumentationScope. +message ScopeProfiles { + // The instrumentation scope information for the profiles in this message. + // Semantically when InstrumentationScope isn't set, it is equivalent with + // an empty instrumentation scope name (unknown). + opentelemetry.proto.common.v1.InstrumentationScope scope = 1; + + // A list of Profiles that originate from an instrumentation scope. + repeated Profile profiles = 2; + + // This schema_url applies to all profiles and profile events in the "profiles" field. + string schema_url = 3; +} + +// A Profile represents a single profile. +message Profile { + // A unique identifier for a profile. The ID is a 16-byte array. An ID with + // all zeroes is considered invalid. + // + // This field is required. + bytes profile_id = 1; + + // start_time_unix_nano is the start time of the profile. + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. + // + // This field is semantically required and it is expected that end_time >= start_time. + fixed64 start_time_unix_nano = 2; + + // end_time_unix_nano is the end time of the span. + // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. + // + // This field is semantically required and it is expected that end_time >= start_time. + fixed64 end_time_unix_nano = 3; + + // attributes is a collection of key/value pairs. Note, global attributes + // like server name can be set using the resource API. Examples of attributes: + // + // "/http/user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36" + // "/http/server_latency": 300 + // "abc.com/myattribute": true + // "abc.com/score": 10.239 + // + // The OpenTelemetry API specification further restricts the allowed value types: + // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/common/README.md#attribute + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + repeated opentelemetry.proto.common.v1.KeyValue attributes = 4; + + // dropped_attributes_count is the number of attributes that were discarded. Attributes + // can be discarded because their keys are too long or because there are too many + // attributes. If this value is 0, then no attributes were dropped. + uint32 dropped_attributes_count = 5; + + reserved 6 to 15; + + // PPROF contains a pprof profile. + message PPROF { + // payload contains a pprof file. + bytes payload = 1; + } + + // JFR contains a JFR file. + message JFR { + // payload contains a JFR file. + bytes payload = 1; + } + + // OPROF contains a batch of oprof events (details TBD) + message OPROF {} + + oneof profile_kind { + PPROF pprof = 16; + JFR jfr = 17; + OPROF oprof = 18; + } +} From 912021b15b8d1a2c305df0f20977a9ebadafb518 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Wed, 14 Jun 2023 14:13:22 -0700 Subject: [PATCH 2/7] snapshot --- .../v1/profiles_service.proto} | 26 +++++++++---------- .../v1/profiles_service_http.yaml} | 6 ++--- .../v1/profiles.proto} | 10 +++---- 3 files changed, 21 insertions(+), 21 deletions(-) rename opentelemetry/proto/collector/{profiling/v1/profiling_service.proto => profiles/v1/profiles_service.proto} (76%) rename opentelemetry/proto/collector/{profiling/v1/trace_service_http.yaml => profiles/v1/profiles_service_http.yaml} (65%) rename opentelemetry/proto/{profiling/v1/profiling.proto => profiles/v1/profiles.proto} (93%) diff --git a/opentelemetry/proto/collector/profiling/v1/profiling_service.proto b/opentelemetry/proto/collector/profiles/v1/profiles_service.proto similarity index 76% rename from opentelemetry/proto/collector/profiling/v1/profiling_service.proto rename to opentelemetry/proto/collector/profiles/v1/profiles_service.proto index d074cfac9..538d6f47b 100644 --- a/opentelemetry/proto/collector/profiling/v1/profiling_service.proto +++ b/opentelemetry/proto/collector/profiles/v1/profiles_service.proto @@ -14,35 +14,35 @@ syntax = "proto3"; -package opentelemetry.proto.collector.profiling.v1; +package opentelemetry.proto.collector.profiles.v1; -import "opentelemetry/proto/profiling/v1/profiling.proto"; +import "opentelemetry/proto/profiles/v1/profiles.proto"; -option csharp_namespace = "OpenTelemetry.Proto.Collector.Profiling.V1"; +option csharp_namespace = "OpenTelemetry.Proto.Collector.Profiles.V1"; option java_multiple_files = true; -option java_package = "io.opentelemetry.proto.collector.profiling.v1"; -option java_outer_classname = "ProfilingServiceProto"; -option go_package = "go.opentelemetry.io/proto/otlp/collector/profiling/v1"; +option java_package = "io.opentelemetry.proto.collector.profiles.v1"; +option java_outer_classname = "ProfilesServiceProto"; +option go_package = "go.opentelemetry.io/proto/otlp/collector/profiles/v1"; // Service that can be used to push profiles between one Application instrumented with // OpenTelemetry and a collector, or between a collector and a central collector (in this // case spans are sent/received to/from multiple Applications). -service ProfilingService { +service ProfilesService { // For performance reasons, it is recommended to keep this RPC // alive for the entire life of the application. - rpc Export(ExportProfilingServiceRequest) returns (ExportProfilingServiceResponse) {} + rpc Export(ExportProfilesServiceRequest) returns (ExportProfilesServiceResponse) {} } -message ExportProfilingServiceRequest { +message ExportProfilesServiceRequest { // An array of ResourceProfiles. // For data coming from a single resource this array will typically contain one // element. Intermediary nodes (such as OpenTelemetry Collector) that receive // data from multiple origins typically batch the data before forwarding further and // in that case this array will contain multiple elements. - repeated opentelemetry.proto.profiling.v1.ResourceProfiles resource_profiles = 1; + repeated opentelemetry.proto.profiles.v1.ResourceProfiles resource_profiles = 1; } -message ExportProfilingServiceResponse { +message ExportProfilesServiceResponse { // The details of a partially successful export request. // // If the request is only partially accepted @@ -58,8 +58,8 @@ message ExportProfilingServiceResponse { // A `partial_success` message with an empty value (rejected_ = 0 and // `error_message` = "") is equivalent to it not being set/present. Senders // SHOULD interpret it the same way as in the full success case. - ExportProfilingPartialSuccess partial_success = 1; + ExportProfilesPartialSuccess partial_success = 1; } -message ExportProfilingPartialSuccess { +message ExportProfilesPartialSuccess { } diff --git a/opentelemetry/proto/collector/profiling/v1/trace_service_http.yaml b/opentelemetry/proto/collector/profiles/v1/profiles_service_http.yaml similarity index 65% rename from opentelemetry/proto/collector/profiling/v1/trace_service_http.yaml rename to opentelemetry/proto/collector/profiles/v1/profiles_service_http.yaml index 287473597..64a314f1c 100644 --- a/opentelemetry/proto/collector/profiling/v1/trace_service_http.yaml +++ b/opentelemetry/proto/collector/profiles/v1/profiles_service_http.yaml @@ -4,6 +4,6 @@ type: google.api.Service config_version: 3 http: rules: - - selector: opentelemetry.proto.collector.trace.v1.TraceService.Export - post: /v1/trace - body: "*" \ No newline at end of file + - selector: opentelemetry.proto.collector.profiles.v1.TraceService.Export + post: /v1/profiles + body: "*" diff --git a/opentelemetry/proto/profiling/v1/profiling.proto b/opentelemetry/proto/profiles/v1/profiles.proto similarity index 93% rename from opentelemetry/proto/profiling/v1/profiling.proto rename to opentelemetry/proto/profiles/v1/profiles.proto index ed665d4bf..c1afc8d20 100644 --- a/opentelemetry/proto/profiling/v1/profiling.proto +++ b/opentelemetry/proto/profiles/v1/profiles.proto @@ -14,16 +14,16 @@ syntax = "proto3"; -package opentelemetry.proto.profiling.v1; +package opentelemetry.proto.profiles.v1; import "opentelemetry/proto/common/v1/common.proto"; import "opentelemetry/proto/resource/v1/resource.proto"; -option csharp_namespace = "OpenTelemetry.Proto.Profiling.V1"; +option csharp_namespace = "OpenTelemetry.Proto.Profiles.V1"; option java_multiple_files = true; -option java_package = "io.opentelemetry.proto.profiling.v1"; -option java_outer_classname = "ProfilingProto"; -option go_package = "go.opentelemetry.io/proto/otlp/profiling/v1"; +option java_package = "io.opentelemetry.proto.profiles.v1"; +option java_outer_classname = "ProfilesProto"; +option go_package = "go.opentelemetry.io/proto/otlp/profiles/v1"; // A collection of ScopeProfiles from a Resource. From 72ddb55bbab2a048d2fda0664124eb488fa68476 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Wed, 14 Jun 2023 14:14:52 -0700 Subject: [PATCH 3/7] snapshot --- .../proto/collector/profiles/v1/profiles_service_http.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opentelemetry/proto/collector/profiles/v1/profiles_service_http.yaml b/opentelemetry/proto/collector/profiles/v1/profiles_service_http.yaml index 64a314f1c..eed628920 100644 --- a/opentelemetry/proto/collector/profiles/v1/profiles_service_http.yaml +++ b/opentelemetry/proto/collector/profiles/v1/profiles_service_http.yaml @@ -4,6 +4,6 @@ type: google.api.Service config_version: 3 http: rules: - - selector: opentelemetry.proto.collector.profiles.v1.TraceService.Export + - selector: opentelemetry.proto.collector.profiles.v1.ProfilesService.Export post: /v1/profiles body: "*" From 05ecb3c0e54b534b606189f5d2371bbd309d3cf7 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Wed, 14 Jun 2023 15:16:52 -0700 Subject: [PATCH 4/7] empty profile proto --- .../profiles/v1/profiles_service.proto | 14 +++++ .../proto/profiles/v1/profiles.proto | 55 +++++++++++++------ 2 files changed, 51 insertions(+), 18 deletions(-) diff --git a/opentelemetry/proto/collector/profiles/v1/profiles_service.proto b/opentelemetry/proto/collector/profiles/v1/profiles_service.proto index 538d6f47b..aaaefdef7 100644 --- a/opentelemetry/proto/collector/profiles/v1/profiles_service.proto +++ b/opentelemetry/proto/collector/profiles/v1/profiles_service.proto @@ -62,4 +62,18 @@ message ExportProfilesServiceResponse { } message ExportProfilesPartialSuccess { + // The number of rejected profile records. + // + // A `rejected_` field holding a `0` value indicates that the + // request was fully accepted. + int64 rejected_profile_records = 1; + + // A developer-facing human-readable message in English. It should be used + // either to explain why the server rejected parts of the data during a partial + // success or to convey warnings/suggestions during a full success. The message + // should offer guidance on how users can address such issues. + // + // error_message is an optional field. An error_message with an empty value + // is equivalent to it not being set. + string error_message = 2; } diff --git a/opentelemetry/proto/profiles/v1/profiles.proto b/opentelemetry/proto/profiles/v1/profiles.proto index c1afc8d20..08dfdffc3 100644 --- a/opentelemetry/proto/profiles/v1/profiles.proto +++ b/opentelemetry/proto/profiles/v1/profiles.proto @@ -25,6 +25,25 @@ option java_package = "io.opentelemetry.proto.profiles.v1"; option java_outer_classname = "ProfilesProto"; option go_package = "go.opentelemetry.io/proto/otlp/profiles/v1"; +// ProfilesData represents the profiles data that can be stored in a persistent storage, +// OR can be embedded by other protocols that transfer OTLP profiles data but do not +// implement the OTLP protocol. +// +// The main difference between this message and collector protocol is that +// in this message there will not be any "control" or "metadata" specific to +// OTLP protocol. +// +// When new fields are added into this message, the OTLP request MUST be updated +// as well. +message ProfilesData { + // An array of ResourceProfiles. + // For data coming from a single resource this array will typically contain + // one element. Intermediary nodes that receive data from multiple origins + // typically batch the data before forwarding further and in that case this + // array will contain multiple elements. + repeated ResourceProfiles resource_profiles = 1; +} + // A collection of ScopeProfiles from a Resource. message ResourceProfiles { @@ -95,26 +114,26 @@ message Profile { // attributes. If this value is 0, then no attributes were dropped. uint32 dropped_attributes_count = 5; - reserved 6 to 15; + // reserved 6 to 15; - // PPROF contains a pprof profile. - message PPROF { - // payload contains a pprof file. - bytes payload = 1; - } + // // PPROF contains a pprof profile. + // message PPROF { + // // payload contains a pprof file. + // bytes payload = 1; + // } - // JFR contains a JFR file. - message JFR { - // payload contains a JFR file. - bytes payload = 1; - } + // // JFR contains a JFR file. + // message JFR { + // // payload contains a JFR file. + // bytes payload = 1; + // } - // OPROF contains a batch of oprof events (details TBD) - message OPROF {} + // // OPROF contains a batch of oprof events (details TBD) + // message OPROF {} - oneof profile_kind { - PPROF pprof = 16; - JFR jfr = 17; - OPROF oprof = 18; - } + // oneof profile_kind { + // PPROF pprof = 16; + // JFR jfr = 17; + // OPROF oprof = 18; + // } } From 1ebf7eb59ed5dd82ed5a1144c5fdaac0013ba879 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Tue, 20 Jun 2023 11:08:47 -0700 Subject: [PATCH 5/7] improved naming (profile_records came from log_records) --- .../proto/collector/profiles/v1/profiles_service.proto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opentelemetry/proto/collector/profiles/v1/profiles_service.proto b/opentelemetry/proto/collector/profiles/v1/profiles_service.proto index aaaefdef7..bfbd0aa29 100644 --- a/opentelemetry/proto/collector/profiles/v1/profiles_service.proto +++ b/opentelemetry/proto/collector/profiles/v1/profiles_service.proto @@ -62,11 +62,11 @@ message ExportProfilesServiceResponse { } message ExportProfilesPartialSuccess { - // The number of rejected profile records. + // The number of rejected profiles. // // A `rejected_` field holding a `0` value indicates that the // request was fully accepted. - int64 rejected_profile_records = 1; + int64 rejected_profiles = 1; // A developer-facing human-readable message in English. It should be used // either to explain why the server rejected parts of the data during a partial From fb6e2152bc0ff20943686d4ff27110cb4848d4c0 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Mon, 26 Jun 2023 22:49:17 -0700 Subject: [PATCH 6/7] adds alternative variations of profiles proto --- .../denormalized/denormalized.proto | 131 +++++++++++ .../alternatives/normalized/normalized.proto | 142 +++++++++++ .../v1/alternatives/pprof/pprof.proto | 221 ++++++++++++++++++ .../proto/profiles/v1/profiles.proto | 38 ++- 4 files changed, 510 insertions(+), 22 deletions(-) create mode 100644 opentelemetry/proto/profiles/v1/alternatives/denormalized/denormalized.proto create mode 100644 opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto create mode 100644 opentelemetry/proto/profiles/v1/alternatives/pprof/pprof.proto diff --git a/opentelemetry/proto/profiles/v1/alternatives/denormalized/denormalized.proto b/opentelemetry/proto/profiles/v1/alternatives/denormalized/denormalized.proto new file mode 100644 index 000000000..605c913e3 --- /dev/null +++ b/opentelemetry/proto/profiles/v1/alternatives/denormalized/denormalized.proto @@ -0,0 +1,131 @@ +syntax = "proto3"; + +package opentelemetry.proto.profiles.v1.alternatives.denormalized; + +import "opentelemetry/proto/common/v1/common.proto"; +import "opentelemetry/proto/resource/v1/resource.proto"; + +option go_package = "go.opentelemetry.io/proto/otlp/profiles/v1/alternatives/denormalized"; + + +// A pointer from a profile to a trace span. +message Link { + // A unique identifier of a trace that this linked span is part of. The ID is a + // 16-byte array. + bytes trace_id = 1; + // A unique identifier for the linked span. The ID is an 8-byte array. + bytes span_id = 2; +} + +message Sample { + repeated Location locations = 1; + repeated Link links = 2; + repeated opentelemetry.proto.common.v1.KeyValue attributes = 3; + uint32 dropped_attributes_count = 4; + + // this one is repeated because there can be multiple profile kinds in one profile. Typical example is memory profiles in go that contain: + // * alloc_objects + // * alloc_bytes + // * inuse_objects + // * inuse_bytes + repeated fixed64 values = 5; + + // optional + fixed64 timestamp_unix_nano = 6; +} + +// borrowed from metrics proto +enum AggregationTemporality { + AGGREGATION_TEMPORALITY_UNSPECIFIED = 0; + AGGREGATION_TEMPORALITY_DELTA = 1; + AGGREGATION_TEMPORALITY_CUMULATIVE = 2; +} + +message SampleType { + AggregationTemporality aggregation_temporality = 1; + uint64 sample_size = 2; + + // CPU / memory /etc + string type = 3; + string unit = 4; +} + +message Profile { + repeated SampleType sample_types = 1; + repeated Sample samples = 2; +} + +// borrowed from pprof proto +message Mapping { + // Address at which the binary (or DLL) is loaded into memory. + uint64 memory_start = 1; + // The limit of the address range occupied by this mapping. + uint64 memory_limit = 2; + // Offset in the binary that corresponds to the first mapped address. + uint64 file_offset = 3; + // The object this entry is loaded from. This can be a filename on + // disk for the main binary and shared libraries, or virtual + // abstractions like "[vdso]". + string filename = 4; + // A string that uniquely identifies a particular program version + // with high probability. E.g., for binaries generated by GNU tools, + // it could be the contents of the .note.gnu.build-id field. + string build_id = 5; + + // The following fields indicate the resolution of symbolic info. + bool has_functions = 6; + bool has_filenames = 7; + bool has_line_numbers = 8; + bool has_inline_frames = 9; +} + +// borrowed from pprof proto +// Describes function and line table debug information. +message Location { + // profile.Mapping for this location. + // It can be unset if the mapping is unknown or not applicable for + // this profile type. + Mapping mapping = 1; + + // The instruction address for this location, if available. It + // should be within [Mapping.memory_start...Mapping.memory_limit] + // for the corresponding mapping. A non-leaf address may be in the + // middle of a call instruction. It is up to display tools to find + // the beginning of the instruction if necessary. + uint64 address = 2; + // Multiple line indicates this location has inlined functions, + // where the last entry represents the caller into which the + // preceding entries were inlined. + // + // E.g., if memcpy() is inlined into printf: + // line[0].function_name == "memcpy" + // line[1].function_name == "printf" + repeated Line line = 3; + // Provides an indication that multiple symbols map to this location's + // address, for example due to identical code folding by the linker. In that + // case the line information above represents one of the multiple + // symbols. This field must be recomputed when the symbolization state of the + // profile changes. + bool is_folded = 4; +} + +// borrowed from pprof proto +message Line { + // Function for this line. + Function function = 1; + // Line number in source code. + int64 line = 2; +} + +// borrowed from pprof proto +message Function { + // Name of the function, in human-readable form if available. + string name = 1; + // Name of the function, as identified by the system. + // For instance, it can be a C++ mangled name. + string system_name = 2; + // Source file containing the function. + string filename = 3; + // Line number in source file. + string start_line = 4; +} diff --git a/opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto b/opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto new file mode 100644 index 000000000..7fd156f0e --- /dev/null +++ b/opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto @@ -0,0 +1,142 @@ +syntax = "proto3"; + +package opentelemetry.proto.profiles.v1.alternatives.normalized; + +import "opentelemetry/proto/common/v1/common.proto"; +import "opentelemetry/proto/resource/v1/resource.proto"; + +option go_package = "go.opentelemetry.io/proto/otlp/profiles/v1/normalized"; + + +// A pointer from a profile to a trace span. +message Link { + // A unique identifier of a trace that this linked span is part of. The ID is a + // 16-byte array. + bytes trace_id = 1; + // A unique identifier for the linked span. The ID is an 8-byte array. + bytes span_id = 2; +} + +// we link to attribute sets because +message AttributeSet { + repeated opentelemetry.proto.common.v1.KeyValue attributes = 1; + uint32 dropped_attributes_count = 2; +} + +message Sample { + repeated uint64 location_ids = 1; + repeated uint64 link_ids = 2; + repeated uint64 attribute_set_ids = 3; + + // this one is repeated because there can be multiple profile kinds in one profile. Typical example is memory profiles in go that contain: + // * alloc_objects + // * alloc_bytes + // * inuse_objects + // * inuse_bytes + repeated fixed64 values = 4; + + // optional + fixed64 timestamp_unix_nano = 5; +} + +// borrowed from metrics proto +enum AggregationTemporality { + AGGREGATION_TEMPORALITY_UNSPECIFIED = 0; + AGGREGATION_TEMPORALITY_DELTA = 1; + AGGREGATION_TEMPORALITY_CUMULATIVE = 2; +} + +message SampleType { + AggregationTemporality aggregation_temporality = 1; + uint64 sample_size = 2; + + // CPU / memory /etc + int64 type = 3; // Index into string table. + int64 unit = 4; // Index into string table. +} + +message Profile { + repeated SampleType sample_types = 1; + repeated Sample samples = 2; + repeated Mapping mappings = 3; + repeated Location locations = 4; + repeated Function functions = 5; + repeated Link links = 6; + repeated AttributeSet attribute_sets = 7; + repeated string string_table = 8; +} + + +// borrowed from pprof proto +message Mapping { + // Address at which the binary (or DLL) is loaded into memory. + uint64 memory_start = 1; + // The limit of the address range occupied by this mapping. + uint64 memory_limit = 2; + // Offset in the binary that corresponds to the first mapped address. + uint64 file_offset = 3; + // The object this entry is loaded from. This can be a filename on + // disk for the main binary and shared libraries, or virtual + // abstractions like "[vdso]". + int64 filename = 4; // Index into string table + // A string that uniquely identifies a particular program version + // with high probability. E.g., for binaries generated by GNU tools, + // it could be the contents of the .note.gnu.build-id field. + int64 build_id = 5; // Index into string table + + // The following fields indicate the resolution of symbolic info. + bool has_functions = 6; + bool has_filenames = 7; + bool has_line_numbers = 8; + bool has_inline_frames = 9; +} + +// borrowed from pprof proto +// Describes function and line table debug information. +message Location { + // The id of the corresponding profile.Mapping for this location. + // It can be unset if the mapping is unknown or not applicable for + // this profile type. + uint64 mapping_id = 1; + // The instruction address for this location, if available. It + // should be within [Mapping.memory_start...Mapping.memory_limit] + // for the corresponding mapping. A non-leaf address may be in the + // middle of a call instruction. It is up to display tools to find + // the beginning of the instruction if necessary. + uint64 address = 2; + // Multiple line indicates this location has inlined functions, + // where the last entry represents the caller into which the + // preceding entries were inlined. + // + // E.g., if memcpy() is inlined into printf: + // line[0].function_name == "memcpy" + // line[1].function_name == "printf" + repeated Line line = 3; + // Provides an indication that multiple symbols map to this location's + // address, for example due to identical code folding by the linker. In that + // case the line information above represents one of the multiple + // symbols. This field must be recomputed when the symbolization state of the + // profile changes. + bool is_folded = 4; +} + +// borrowed from pprof proto +message Line { + // The id of the corresponding profile.Function for this line. + uint64 function_id = 1; + // Line number in source code. + int64 line = 2; +} + +// borrowed from pprof proto +message Function { + // Name of the function, in human-readable form if available. + int64 name = 1; // Index into string table + // Name of the function, as identified by the system. + // For instance, it can be a C++ mangled name. + int64 system_name = 2; // Index into string table + // Source file containing the function. + int64 filename = 3; // Index into string table + // Line number in source file. + int64 start_line = 4; +} diff --git a/opentelemetry/proto/profiles/v1/alternatives/pprof/pprof.proto b/opentelemetry/proto/profiles/v1/alternatives/pprof/pprof.proto new file mode 100644 index 000000000..ec55bc207 --- /dev/null +++ b/opentelemetry/proto/profiles/v1/alternatives/pprof/pprof.proto @@ -0,0 +1,221 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Profile is a common stacktrace profile format. +// +// Measurements represented with this format should follow the +// following conventions: +// +// - Consumers should treat unset optional fields as if they had been +// set with their default value. +// +// - When possible, measurements should be stored in "unsampled" form +// that is most useful to humans. There should be enough +// information present to determine the original sampled values. +// +// - On-disk, the serialized proto must be gzip-compressed. +// +// - The profile is represented as a set of samples, where each sample +// references a sequence of locations, and where each location belongs +// to a mapping. +// - There is a N->1 relationship from sample.location_id entries to +// locations. For every sample.location_id entry there must be a +// unique Location with that id. +// - There is an optional N->1 relationship from locations to +// mappings. For every nonzero Location.mapping_id there must be a +// unique Mapping with that id. + +syntax = "proto3"; + +package opentelemetry.proto.profiles.v1.alternatives.pprof; + +option go_package = "go.opentelemetry.io/proto/otlp/profiles/v1/alternatives/pprof"; + +message Profile { + // A description of the samples associated with each Sample.value. + // For a cpu profile this might be: + // [["cpu","nanoseconds"]] or [["wall","seconds"]] or [["syscall","count"]] + // For a heap profile, this might be: + // [["allocations","count"], ["space","bytes"]], + // If one of the values represents the number of events represented + // by the sample, by convention it should be at index 0 and use + // sample_type.unit == "count". + repeated ValueType sample_type = 1; + // The set of samples recorded in this profile. + repeated Sample sample = 2; + // Mapping from address ranges to the image/binary/library mapped + // into that address range. mapping[0] will be the main binary. + repeated Mapping mapping = 3; + // Locations referenced by samples. + repeated Location location = 4; + // Functions referenced by locations. + repeated Function function = 5; + // A common table for strings referenced by various messages. + // string_table[0] must always be "". + repeated string string_table = 6; + // frames with Function.function_name fully matching the following + // regexp will be dropped from the samples, along with their successors. + int64 drop_frames = 7; // Index into string table. + // frames with Function.function_name fully matching the following + // regexp will be kept, even if it matches drop_frames. + int64 keep_frames = 8; // Index into string table. + + // The following fields are informational, do not affect + // interpretation of results. + + // Time of collection (UTC) represented as nanoseconds past the epoch. + int64 time_nanos = 9; + // Duration of the profile, if a duration makes sense. + int64 duration_nanos = 10; + // The kind of events between sampled occurrences. + // e.g [ "cpu","cycles" ] or [ "heap","bytes" ] + ValueType period_type = 11; + // The number of events between sampled occurrences. + int64 period = 12; + // Free-form text associated with the profile. The text is displayed as is + // to the user by the tools that read profiles (e.g. by pprof). This field + // should not be used to store any machine-readable information, it is only + // for human-friendly content. The profile must stay functional if this field + // is cleaned. + repeated int64 comment = 13; // Indices into string table. + // Index into the string table of the type of the preferred sample + // value. If unset, clients should default to the last sample value. + int64 default_sample_type = 14; +} + +// ValueType describes the semantics and measurement units of a value. +message ValueType { + int64 type = 1; // Index into string table. + int64 unit = 2; // Index into string table. +} + +// Each Sample records values encountered in some program +// context. The program context is typically a stack trace, perhaps +// augmented with auxiliary information like the thread-id, some +// indicator of a higher level request being handled etc. +message Sample { + // The ids recorded here correspond to a Profile.location.id. + // The leaf is at location_id[0]. + repeated uint64 location_id = 1; + // The type and unit of each value is defined by the corresponding + // entry in Profile.sample_type. All samples must have the same + // number of values, the same as the length of Profile.sample_type. + // When aggregating multiple samples into a single sample, the + // result has a list of values that is the element-wise sum of the + // lists of the originals. + repeated int64 value = 2; + // label includes additional context for this sample. It can include + // things like a thread id, allocation size, etc. + // + // NOTE: While possible, having multiple values for the same label key is + // strongly discouraged and should never be used. Most tools (e.g. pprof) do + // not have good (or any) support for multi-value labels. And an even more + // discouraged case is having a string label and a numeric label of the same + // name on a sample. Again, possible to express, but should not be used. + repeated Label label = 3; +} + +message Label { + int64 key = 1; // Index into string table + + // At most one of the following must be present + int64 str = 2; // Index into string table + int64 num = 3; + + // Should only be present when num is present. + // Specifies the units of num. + // Use arbitrary string (for example, "requests") as a custom count unit. + // If no unit is specified, consumer may apply heuristic to deduce the unit. + // Consumers may also interpret units like "bytes" and "kilobytes" as memory + // units and units like "seconds" and "nanoseconds" as time units, + // and apply appropriate unit conversions to these. + int64 num_unit = 4; // Index into string table +} + +message Mapping { + // Unique nonzero id for the mapping. + uint64 id = 1; + // Address at which the binary (or DLL) is loaded into memory. + uint64 memory_start = 2; + // The limit of the address range occupied by this mapping. + uint64 memory_limit = 3; + // Offset in the binary that corresponds to the first mapped address. + uint64 file_offset = 4; + // The object this entry is loaded from. This can be a filename on + // disk for the main binary and shared libraries, or virtual + // abstractions like "[vdso]". + int64 filename = 5; // Index into string table + // A string that uniquely identifies a particular program version + // with high probability. E.g., for binaries generated by GNU tools, + // it could be the contents of the .note.gnu.build-id field. + int64 build_id = 6; // Index into string table + + // The following fields indicate the resolution of symbolic info. + bool has_functions = 7; + bool has_filenames = 8; + bool has_line_numbers = 9; + bool has_inline_frames = 10; +} + +// Describes function and line table debug information. +message Location { + // Unique nonzero id for the location. A profile could use + // instruction addresses or any integer sequence as ids. + uint64 id = 1; + // The id of the corresponding profile.Mapping for this location. + // It can be unset if the mapping is unknown or not applicable for + // this profile type. + uint64 mapping_id = 2; + // The instruction address for this location, if available. It + // should be within [Mapping.memory_start...Mapping.memory_limit] + // for the corresponding mapping. A non-leaf address may be in the + // middle of a call instruction. It is up to display tools to find + // the beginning of the instruction if necessary. + uint64 address = 3; + // Multiple line indicates this location has inlined functions, + // where the last entry represents the caller into which the + // preceding entries were inlined. + // + // E.g., if memcpy() is inlined into printf: + // line[0].function_name == "memcpy" + // line[1].function_name == "printf" + repeated Line line = 4; + // Provides an indication that multiple symbols map to this location's + // address, for example due to identical code folding by the linker. In that + // case the line information above represents one of the multiple + // symbols. This field must be recomputed when the symbolization state of the + // profile changes. + bool is_folded = 5; +} + +message Line { + // The id of the corresponding profile.Function for this line. + uint64 function_id = 1; + // Line number in source code. + int64 line = 2; +} + +message Function { + // Unique nonzero id for the function. + uint64 id = 1; + // Name of the function, in human-readable form if available. + int64 name = 2; // Index into string table + // Name of the function, as identified by the system. + // For instance, it can be a C++ mangled name. + int64 system_name = 3; // Index into string table + // Source file containing the function. + int64 filename = 4; // Index into string table + // Line number in source file. + int64 start_line = 5; +} diff --git a/opentelemetry/proto/profiles/v1/profiles.proto b/opentelemetry/proto/profiles/v1/profiles.proto index 08dfdffc3..1adae179f 100644 --- a/opentelemetry/proto/profiles/v1/profiles.proto +++ b/opentelemetry/proto/profiles/v1/profiles.proto @@ -19,6 +19,12 @@ package opentelemetry.proto.profiles.v1; import "opentelemetry/proto/common/v1/common.proto"; import "opentelemetry/proto/resource/v1/resource.proto"; +// TODO(@petethepig): idea here is that we will eventually get rid of these +// and have just version in this (profiles.proto) file +import "opentelemetry/proto/profiles/v1/alternatives/pprof/pprof.proto"; +import "opentelemetry/proto/profiles/v1/alternatives/denormalized/denormalized.proto"; +import "opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto"; + option csharp_namespace = "OpenTelemetry.Proto.Profiles.V1"; option java_multiple_files = true; option java_package = "io.opentelemetry.proto.profiles.v1"; @@ -89,7 +95,7 @@ message Profile { // This field is semantically required and it is expected that end_time >= start_time. fixed64 start_time_unix_nano = 2; - // end_time_unix_nano is the end time of the span. + // end_time_unix_nano is the end time of the profile. // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. // // This field is semantically required and it is expected that end_time >= start_time. @@ -114,26 +120,14 @@ message Profile { // attributes. If this value is 0, then no attributes were dropped. uint32 dropped_attributes_count = 5; - // reserved 6 to 15; - - // // PPROF contains a pprof profile. - // message PPROF { - // // payload contains a pprof file. - // bytes payload = 1; - // } - - // // JFR contains a JFR file. - // message JFR { - // // payload contains a JFR file. - // bytes payload = 1; - // } - - // // OPROF contains a batch of oprof events (details TBD) - // message OPROF {} + // This can be original pprof or jfr encoded profile. + bytes original_payload = 7; - // oneof profile_kind { - // PPROF pprof = 16; - // JFR jfr = 17; - // OPROF oprof = 18; - // } + // this is temporary until we pick one of these formats + // fields from one of these will be embedded in Profile message: + oneof alternative_profile { + opentelemetry.proto.profiles.v1.alternatives.pprof.Profile pprof = 8; + opentelemetry.proto.profiles.v1.alternatives.normalized.Profile normalized = 9; + opentelemetry.proto.profiles.v1.alternatives.denormalized.Profile denormalized = 10; + } } From 25dce2ebcc39508e4e29b9eb5e92e606f3d142b7 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Mon, 26 Jun 2023 23:03:56 -0700 Subject: [PATCH 7/7] fix --- .../proto/profiles/v1/alternatives/normalized/normalized.proto | 1 - 1 file changed, 1 deletion(-) diff --git a/opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto b/opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto index 7fd156f0e..ab69254e2 100644 --- a/opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto +++ b/opentelemetry/proto/profiles/v1/alternatives/normalized/normalized.proto @@ -17,7 +17,6 @@ message Link { bytes span_id = 2; } -// we link to attribute sets because message AttributeSet { repeated opentelemetry.proto.common.v1.KeyValue attributes = 1; uint32 dropped_attributes_count = 2;