-
Notifications
You must be signed in to change notification settings - Fork 272
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add profiles proto #488
Add profiles proto #488
Changes from all commits
0c3571c
3decfa0
912021b
72ddb55
05ecb3c
1ebf7eb
fb6e215
25dce2e
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,79 @@ | ||
// Copyright 2019, OpenTelemetry Authors | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
syntax = "proto3"; | ||
|
||
package opentelemetry.proto.collector.profiles.v1; | ||
|
||
import "opentelemetry/proto/profiles/v1/profiles.proto"; | ||
|
||
option csharp_namespace = "OpenTelemetry.Proto.Collector.Profiles.V1"; | ||
option java_multiple_files = true; | ||
option java_package = "io.opentelemetry.proto.collector.profiles.v1"; | ||
option java_outer_classname = "ProfilesServiceProto"; | ||
option go_package = "go.opentelemetry.io/proto/otlp/collector/profiles/v1"; | ||
|
||
// Service that can be used to push profiles between one Application instrumented with | ||
// OpenTelemetry and a collector, or between a collector and a central collector (in this | ||
// case spans are sent/received to/from multiple Applications). | ||
service ProfilesService { | ||
// For performance reasons, it is recommended to keep this RPC | ||
// alive for the entire life of the application. | ||
rpc Export(ExportProfilesServiceRequest) returns (ExportProfilesServiceResponse) {} | ||
} | ||
|
||
message ExportProfilesServiceRequest { | ||
// An array of ResourceProfiles. | ||
// For data coming from a single resource this array will typically contain one | ||
// element. Intermediary nodes (such as OpenTelemetry Collector) that receive | ||
// data from multiple origins typically batch the data before forwarding further and | ||
// in that case this array will contain multiple elements. | ||
repeated opentelemetry.proto.profiles.v1.ResourceProfiles resource_profiles = 1; | ||
} | ||
|
||
message ExportProfilesServiceResponse { | ||
// The details of a partially successful export request. | ||
// | ||
// If the request is only partially accepted | ||
// (i.e. when the server accepts only parts of the data and rejects the rest) | ||
// the server MUST initialize the `partial_success` field and MUST | ||
// set the `rejected_<signal>` with the number of items it rejected. | ||
// | ||
// Servers MAY also make use of the `partial_success` field to convey | ||
// warnings/suggestions to senders even when the request was fully accepted. | ||
// In such cases, the `rejected_<signal>` MUST have a value of `0` and | ||
// the `error_message` MUST be non-empty. | ||
// | ||
// A `partial_success` message with an empty value (rejected_<signal> = 0 and | ||
// `error_message` = "") is equivalent to it not being set/present. Senders | ||
// SHOULD interpret it the same way as in the full success case. | ||
ExportProfilesPartialSuccess partial_success = 1; | ||
} | ||
|
||
message ExportProfilesPartialSuccess { | ||
// The number of rejected profiles. | ||
// | ||
// A `rejected_<signal>` field holding a `0` value indicates that the | ||
// request was fully accepted. | ||
int64 rejected_profiles = 1; | ||
|
||
// A developer-facing human-readable message in English. It should be used | ||
// either to explain why the server rejected parts of the data during a partial | ||
// success or to convey warnings/suggestions during a full success. The message | ||
// should offer guidance on how users can address such issues. | ||
// | ||
// error_message is an optional field. An error_message with an empty value | ||
// is equivalent to it not being set. | ||
string error_message = 2; | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,9 @@ | ||
# This is an API configuration to generate an HTTP/JSON -> gRPC gateway for the | ||
# OpenTelemetry service using github.com/grpc-ecosystem/grpc-gateway. | ||
type: google.api.Service | ||
config_version: 3 | ||
http: | ||
rules: | ||
- selector: opentelemetry.proto.collector.profiles.v1.ProfilesService.Export | ||
post: /v1/profiles | ||
body: "*" |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,131 @@ | ||
syntax = "proto3"; | ||
|
||
package opentelemetry.proto.profiles.v1.alternatives.denormalized; | ||
|
||
import "opentelemetry/proto/common/v1/common.proto"; | ||
import "opentelemetry/proto/resource/v1/resource.proto"; | ||
|
||
option go_package = "go.opentelemetry.io/proto/otlp/profiles/v1/alternatives/denormalized"; | ||
|
||
|
||
// A pointer from a profile to a trace span. | ||
message Link { | ||
// A unique identifier of a trace that this linked span is part of. The ID is a | ||
// 16-byte array. | ||
bytes trace_id = 1; | ||
// A unique identifier for the linked span. The ID is an 8-byte array. | ||
bytes span_id = 2; | ||
} | ||
|
||
message Sample { | ||
repeated Location locations = 1; | ||
repeated Link links = 2; | ||
repeated opentelemetry.proto.common.v1.KeyValue attributes = 3; | ||
uint32 dropped_attributes_count = 4; | ||
|
||
// this one is repeated because there can be multiple profile kinds in one profile. Typical example is memory profiles in go that contain: | ||
// * alloc_objects | ||
// * alloc_bytes | ||
// * inuse_objects | ||
// * inuse_bytes | ||
repeated fixed64 values = 5; | ||
|
||
// optional | ||
fixed64 timestamp_unix_nano = 6; | ||
} | ||
|
||
// borrowed from metrics proto | ||
enum AggregationTemporality { | ||
AGGREGATION_TEMPORALITY_UNSPECIFIED = 0; | ||
AGGREGATION_TEMPORALITY_DELTA = 1; | ||
AGGREGATION_TEMPORALITY_CUMULATIVE = 2; | ||
} | ||
|
||
message SampleType { | ||
AggregationTemporality aggregation_temporality = 1; | ||
uint64 sample_size = 2; | ||
|
||
// CPU / memory /etc | ||
string type = 3; | ||
string unit = 4; | ||
} | ||
|
||
message Profile { | ||
repeated SampleType sample_types = 1; | ||
repeated Sample samples = 2; | ||
} | ||
|
||
// borrowed from pprof proto | ||
message Mapping { | ||
// Address at which the binary (or DLL) is loaded into memory. | ||
uint64 memory_start = 1; | ||
// The limit of the address range occupied by this mapping. | ||
uint64 memory_limit = 2; | ||
// Offset in the binary that corresponds to the first mapped address. | ||
uint64 file_offset = 3; | ||
// The object this entry is loaded from. This can be a filename on | ||
// disk for the main binary and shared libraries, or virtual | ||
// abstractions like "[vdso]". | ||
string filename = 4; | ||
// A string that uniquely identifies a particular program version | ||
// with high probability. E.g., for binaries generated by GNU tools, | ||
// it could be the contents of the .note.gnu.build-id field. | ||
string build_id = 5; | ||
|
||
// The following fields indicate the resolution of symbolic info. | ||
bool has_functions = 6; | ||
bool has_filenames = 7; | ||
bool has_line_numbers = 8; | ||
bool has_inline_frames = 9; | ||
} | ||
|
||
// borrowed from pprof proto | ||
// Describes function and line table debug information. | ||
message Location { | ||
// profile.Mapping for this location. | ||
// It can be unset if the mapping is unknown or not applicable for | ||
// this profile type. | ||
Mapping mapping = 1; | ||
|
||
// The instruction address for this location, if available. It | ||
// should be within [Mapping.memory_start...Mapping.memory_limit] | ||
// for the corresponding mapping. A non-leaf address may be in the | ||
// middle of a call instruction. It is up to display tools to find | ||
// the beginning of the instruction if necessary. | ||
uint64 address = 2; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We need to specify exactly how deep into the call instruction the address is allowed to be, display tools do not have access to the assembly bytes and hence cannot do more than simple arithmetic to find the beginning of the instruction. |
||
// Multiple line indicates this location has inlined functions, | ||
// where the last entry represents the caller into which the | ||
// preceding entries were inlined. | ||
// | ||
// E.g., if memcpy() is inlined into printf: | ||
// line[0].function_name == "memcpy" | ||
// line[1].function_name == "printf" | ||
repeated Line line = 3; | ||
// Provides an indication that multiple symbols map to this location's | ||
// address, for example due to identical code folding by the linker. In that | ||
// case the line information above represents one of the multiple | ||
// symbols. This field must be recomputed when the symbolization state of the | ||
// profile changes. | ||
bool is_folded = 4; | ||
} | ||
|
||
// borrowed from pprof proto | ||
message Line { | ||
// Function for this line. | ||
Function function = 1; | ||
// Line number in source code. | ||
int64 line = 2; | ||
} | ||
|
||
// borrowed from pprof proto | ||
message Function { | ||
// Name of the function, in human-readable form if available. | ||
string name = 1; | ||
// Name of the function, as identified by the system. | ||
// For instance, it can be a C++ mangled name. | ||
string system_name = 2; | ||
// Source file containing the function. | ||
string filename = 3; | ||
// Line number in source file. | ||
string start_line = 4; | ||
} |
Original file line number | Diff line number | Diff line change | |||||||||||||||||||||||||
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
@@ -0,0 +1,141 @@ | |||||||||||||||||||||||||||
syntax = "proto3"; | |||||||||||||||||||||||||||
|
|||||||||||||||||||||||||||
package opentelemetry.proto.profiles.v1.alternatives.normalized; | |||||||||||||||||||||||||||
|
|||||||||||||||||||||||||||
import "opentelemetry/proto/common/v1/common.proto"; | |||||||||||||||||||||||||||
import "opentelemetry/proto/resource/v1/resource.proto"; | |||||||||||||||||||||||||||
|
|||||||||||||||||||||||||||
option go_package = "go.opentelemetry.io/proto/otlp/profiles/v1/normalized"; | |||||||||||||||||||||||||||
|
|||||||||||||||||||||||||||
|
|||||||||||||||||||||||||||
// A pointer from a profile to a trace span. | |||||||||||||||||||||||||||
message Link { | |||||||||||||||||||||||||||
// A unique identifier of a trace that this linked span is part of. The ID is a | |||||||||||||||||||||||||||
// 16-byte array. | |||||||||||||||||||||||||||
bytes trace_id = 1; | |||||||||||||||||||||||||||
// A unique identifier for the linked span. The ID is an 8-byte array. | |||||||||||||||||||||||||||
bytes span_id = 2; | |||||||||||||||||||||||||||
} | |||||||||||||||||||||||||||
|
|||||||||||||||||||||||||||
message AttributeSet { | |||||||||||||||||||||||||||
repeated opentelemetry.proto.common.v1.KeyValue attributes = 1; | |||||||||||||||||||||||||||
uint32 dropped_attributes_count = 2; | |||||||||||||||||||||||||||
} | |||||||||||||||||||||||||||
|
|||||||||||||||||||||||||||
message Sample { | |||||||||||||||||||||||||||
repeated uint64 location_ids = 1; | |||||||||||||||||||||||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I wonder if those should be int32/uint32 I don't think we'll overflow this specially since they refer slice/array indexes. This is not super important for the wire format since proto will varint encode it but for the collector this will allow it to buffer more as the memory footprint will be lower per profiles. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Experiment with storing these as a tree (encoded as 2d array of integers) |
|||||||||||||||||||||||||||
repeated uint64 link_ids = 2; | |||||||||||||||||||||||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. embed these in attributes? |
|||||||||||||||||||||||||||
repeated uint64 attribute_set_ids = 3; | |||||||||||||||||||||||||||
|
|||||||||||||||||||||||||||
// this one is repeated because there can be multiple profile kinds in one profile. Typical example is memory profiles in go that contain: | |||||||||||||||||||||||||||
// * alloc_objects | |||||||||||||||||||||||||||
// * alloc_bytes | |||||||||||||||||||||||||||
// * inuse_objects | |||||||||||||||||||||||||||
// * inuse_bytes | |||||||||||||||||||||||||||
repeated fixed64 values = 4; | |||||||||||||||||||||||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. byte-seconds might overflow when aggregated, consider using double floats There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. should not be fixed64 |
|||||||||||||||||||||||||||
|
|||||||||||||||||||||||||||
// optional | |||||||||||||||||||||||||||
fixed64 timestamp_unix_nano = 5; | |||||||||||||||||||||||||||
} | |||||||||||||||||||||||||||
|
|||||||||||||||||||||||||||
// borrowed from metrics proto | |||||||||||||||||||||||||||
enum AggregationTemporality { | |||||||||||||||||||||||||||
AGGREGATION_TEMPORALITY_UNSPECIFIED = 0; | |||||||||||||||||||||||||||
AGGREGATION_TEMPORALITY_DELTA = 1; | |||||||||||||||||||||||||||
AGGREGATION_TEMPORALITY_CUMULATIVE = 2; | |||||||||||||||||||||||||||
} | |||||||||||||||||||||||||||
|
|||||||||||||||||||||||||||
message SampleType { | |||||||||||||||||||||||||||
AggregationTemporality aggregation_temporality = 1; | |||||||||||||||||||||||||||
uint64 sample_size = 2; | |||||||||||||||||||||||||||
|
|||||||||||||||||||||||||||
// CPU / memory /etc | |||||||||||||||||||||||||||
int64 type = 3; // Index into string table. | |||||||||||||||||||||||||||
int64 unit = 4; // Index into string table. | |||||||||||||||||||||||||||
} | |||||||||||||||||||||||||||
|
|||||||||||||||||||||||||||
message Profile { | |||||||||||||||||||||||||||
repeated SampleType sample_types = 1; | |||||||||||||||||||||||||||
repeated Sample samples = 2; | |||||||||||||||||||||||||||
repeated Mapping mappings = 3; | |||||||||||||||||||||||||||
repeated Location locations = 4; | |||||||||||||||||||||||||||
repeated Function functions = 5; | |||||||||||||||||||||||||||
repeated Link links = 6; | |||||||||||||||||||||||||||
repeated AttributeSet attribute_sets = 7; | |||||||||||||||||||||||||||
repeated string string_table = 8; | |||||||||||||||||||||||||||
} | |||||||||||||||||||||||||||
|
|||||||||||||||||||||||||||
|
|||||||||||||||||||||||||||
// borrowed from pprof proto | |||||||||||||||||||||||||||
message Mapping { | |||||||||||||||||||||||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Florian: should stay for profiles that don't have symbols |
|||||||||||||||||||||||||||
// Address at which the binary (or DLL) is loaded into memory. | |||||||||||||||||||||||||||
uint64 memory_start = 1; | |||||||||||||||||||||||||||
// The limit of the address range occupied by this mapping. | |||||||||||||||||||||||||||
uint64 memory_limit = 2; | |||||||||||||||||||||||||||
// Offset in the binary that corresponds to the first mapped address. | |||||||||||||||||||||||||||
uint64 file_offset = 3; | |||||||||||||||||||||||||||
// The object this entry is loaded from. This can be a filename on | |||||||||||||||||||||||||||
// disk for the main binary and shared libraries, or virtual | |||||||||||||||||||||||||||
// abstractions like "[vdso]". | |||||||||||||||||||||||||||
int64 filename = 4; // Index into string table | |||||||||||||||||||||||||||
// A string that uniquely identifies a particular program version | |||||||||||||||||||||||||||
// with high probability. E.g., for binaries generated by GNU tools, | |||||||||||||||||||||||||||
// it could be the contents of the .note.gnu.build-id field. | |||||||||||||||||||||||||||
int64 build_id = 5; // Index into string table | |||||||||||||||||||||||||||
|
|||||||||||||||||||||||||||
// The following fields indicate the resolution of symbolic info. | |||||||||||||||||||||||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. maybe change these to 1 enum or a bit-field. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @aalexand Can you provide the list of combinations that are valid? I remember you mentioned that not all 16 combinations are valid, but also 4-way enum won't cut it either, so maybe we can make this an enum with more than 4 but less than 16 combinations. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think the combinations encountered in practice are these - see below.
I would appreciate if someone also familiar with the subject could verify this from their perspective. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @aalexand thank you! I was trying to make an enum out of this, and here's what I have so far: enum SymbolicInfo {
SYMBOLIC_INFO_UNSPECIFIED = 0;
SYMBOLIC_INFO_FULL = 1;
SYMBOLIC_INFO_FUNCTIONS_ONLY = 2;
SYMBOLIC_INFO_NO_INLINE_FRAMES = 3;
} As I was doing this one thing I thought of is do we need these booleans / enums at all? My understanding is that these booleans are useful when combining a profile without symbols with symbols information (e.g in DWARF format). But I would assume that when you have symbols in some different format (let's say DWARF), you already know if it has functions / line numbers / etc. So it feels like maybe we don't need these booleans / enums. Unfortunately, my understanding of all of this is still pretty limited, so I would love your input on this. Btw, I checked google/pprof repo, and it looks like out of these 4 booleans, the only one that's meaningfully used is HasFunctions one, and even then there's only one mention ( cc @korniltsev |
|||||||||||||||||||||||||||
bool has_functions = 6; | |||||||||||||||||||||||||||
bool has_filenames = 7; | |||||||||||||||||||||||||||
bool has_line_numbers = 8; | |||||||||||||||||||||||||||
bool has_inline_frames = 9; | |||||||||||||||||||||||||||
} | |||||||||||||||||||||||||||
|
|||||||||||||||||||||||||||
// borrowed from pprof proto | |||||||||||||||||||||||||||
// Describes function and line table debug information. | |||||||||||||||||||||||||||
message Location { | |||||||||||||||||||||||||||
// The id of the corresponding profile.Mapping for this location. | |||||||||||||||||||||||||||
// It can be unset if the mapping is unknown or not applicable for | |||||||||||||||||||||||||||
// this profile type. | |||||||||||||||||||||||||||
uint64 mapping_id = 1; | |||||||||||||||||||||||||||
// The instruction address for this location, if available. It | |||||||||||||||||||||||||||
// should be within [Mapping.memory_start...Mapping.memory_limit] | |||||||||||||||||||||||||||
// for the corresponding mapping. A non-leaf address may be in the | |||||||||||||||||||||||||||
// middle of a call instruction. It is up to display tools to find | |||||||||||||||||||||||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Would it make sense to specify precisely what "in the middle" means? I'd prefer not to give implementers the choice between 3 different values... There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This comment comes from pprof proto. My understanding is that this comment refers to the limitation of some profilers where they are not always able to get exact instruction address for a given location. So it's less about what the implementer should put into this field and more of a word of caution for implementers of display tools. I might be totally wrong though — this is getting a little outside of my area of expertise, so I will need help from someone. |
|||||||||||||||||||||||||||
// the beginning of the instruction if necessary. | |||||||||||||||||||||||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Display tools may not have the ability to find the beginning of the instruction unless the "in the middle" above is specified precisely. |
|||||||||||||||||||||||||||
uint64 address = 2; | |||||||||||||||||||||||||||
// Multiple line indicates this location has inlined functions, | |||||||||||||||||||||||||||
// where the last entry represents the caller into which the | |||||||||||||||||||||||||||
// preceding entries were inlined. | |||||||||||||||||||||||||||
// | |||||||||||||||||||||||||||
// E.g., if memcpy() is inlined into printf: | |||||||||||||||||||||||||||
// line[0].function_name == "memcpy" | |||||||||||||||||||||||||||
// line[1].function_name == "printf" | |||||||||||||||||||||||||||
repeated Line line = 3; | |||||||||||||||||||||||||||
// Provides an indication that multiple symbols map to this location's | |||||||||||||||||||||||||||
// address, for example due to identical code folding by the linker. In that | |||||||||||||||||||||||||||
// case the line information above represents one of the multiple | |||||||||||||||||||||||||||
// symbols. This field must be recomputed when the symbolization state of the | |||||||||||||||||||||||||||
// profile changes. | |||||||||||||||||||||||||||
bool is_folded = 4; | |||||||||||||||||||||||||||
} | |||||||||||||||||||||||||||
|
|||||||||||||||||||||||||||
// borrowed from pprof proto | |||||||||||||||||||||||||||
message Line { | |||||||||||||||||||||||||||
// The id of the corresponding profile.Function for this line. | |||||||||||||||||||||||||||
uint64 function_id = 1; | |||||||||||||||||||||||||||
Comment on lines
+124
to
+125
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Is this an index into the other array? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Perhaps we should rename There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yeah, this makes a lot more sense — changed it to |
|||||||||||||||||||||||||||
// Line number in source code. | |||||||||||||||||||||||||||
int64 line = 2; | |||||||||||||||||||||||||||
} | |||||||||||||||||||||||||||
|
|||||||||||||||||||||||||||
// borrowed from pprof proto | |||||||||||||||||||||||||||
message Function { | |||||||||||||||||||||||||||
// Name of the function, in human-readable form if available. | |||||||||||||||||||||||||||
int64 name = 1; // Index into string table | |||||||||||||||||||||||||||
// Name of the function, as identified by the system. | |||||||||||||||||||||||||||
// For instance, it can be a C++ mangled name. | |||||||||||||||||||||||||||
int64 system_name = 2; // Index into string table | |||||||||||||||||||||||||||
// Source file containing the function. | |||||||||||||||||||||||||||
int64 filename = 3; // Index into string table | |||||||||||||||||||||||||||
// Line number in source file. | |||||||||||||||||||||||||||
int64 start_line = 4; | |||||||||||||||||||||||||||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
should be repeated?