diff --git a/.azure-pipelines/pipelines.yml b/.azure-pipelines/pipelines.yml index ddda20ece981..b7d2f94d3206 100644 --- a/.azure-pipelines/pipelines.yml +++ b/.azure-pipelines/pipelines.yml @@ -143,3 +143,6 @@ jobs: TMPDIR: $(Agent.TempDirectory) BAZEL_VC: "C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\Enterprise\\VC" BAZEL_SH: "C:\\Program Files\\Git\\bin\\bash.exe" + BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com + BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance + GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) diff --git a/.bazelrc b/.bazelrc index 648beb564106..028f8bf46785 100644 --- a/.bazelrc +++ b/.bazelrc @@ -158,8 +158,8 @@ build:remote-msan --config=rbe-toolchain-clang-libc++ build:remote-msan --config=rbe-toolchain-msan # Docker sandbox -# NOTE: Update this from https://github.com/envoyproxy/envoy-build-tools/blob/master/toolchains/rbe_toolchains_config.bzl#L7 -build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build-ubuntu@sha256:ebf534b8aa505e8ff5663a31eed782942a742ae4d656b54f4236b00399f17911 +# NOTE: Update this from https://github.com/envoyproxy/envoy-build-tools/blob/master/toolchains/rbe_toolchains_config.bzl#L8 +build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build-ubuntu:09a5a914c904faa39dbc641181cb43b68cabf626 build:docker-sandbox --spawn_strategy=docker build:docker-sandbox --strategy=Javac=docker build:docker-sandbox --strategy=Closure=docker diff --git a/.bazelversion b/.bazelversion index ccbccc3dc626..4a36342fcab7 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -2.2.0 +3.0.0 diff --git a/CODEOWNERS b/CODEOWNERS index 5e405f0c833d..37e376e77e79 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -32,6 +32,8 @@ extensions/filters/common/original_src @snowp @klarose /*/extensions/transport_sockets/tls @PiotrSikora @lizan # sni_cluster extension /*/extensions/filters/network/sni_cluster @rshriram @lizan +# sni_dynamic_forward_proxy extension +/*/extensions/filters/network/sni_dynamic_forward_proxy @rshriram @lizan # tracers.datadog extension /*/extensions/tracers/datadog @cgilmour @palazzem @mattklein123 # tracers.xray extension diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 026e370aab0c..ffd804f25b81 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -52,7 +52,7 @@ versioning guidelines: cause a configuration load failure, unless the feature in question is explicitly overridden in [runtime](https://www.envoyproxy.io/docs/envoy/latest/configuration/operations/runtime#using-runtime-overrides-for-deprecated-features) - config ([example](configs/using_deprecated_config.v2.yaml)). Finally, following the deprecation + config ([example](configs/using_deprecated_config.v2.yaml)). Finally, following the deprecation of the API major version where the field was first marked deprecated, the entire implementation code will be removed from the Envoy implementation. * This policy means that organizations deploying master should have some time to get ready for @@ -63,7 +63,7 @@ versioning guidelines: deprecation window. Within this window, a warning of deprecation should be carefully logged (some features might need rate limiting for logging this). We make no guarantees about code or deployments that rely on undocumented behavior. -* All deprecations/breaking changes will be clearly listed in the [deprecated log](docs/root/intro/deprecated.rst). +* All deprecations/breaking changes will be clearly listed in the [version history](docs/root/version_history/). * High risk deprecations/breaking changes may be announced to the [envoy-announce](https://groups.google.com/forum/#!forum/envoy-announce) email list but by default it is expected the multi-phase warn-by-default/fail-by-default is sufficient to warn users to move @@ -109,8 +109,8 @@ versioning guidelines: changes for 7 days. Obviously PRs that are closed due to lack of activity can be reopened later. Closing stale PRs helps us to keep on top of all of the work currently in flight. * If a commit deprecates a feature, the commit message must mention what has been deprecated. - Additionally, the [deprecated log](docs/root/intro/deprecated.rst) must be updated with relevant - RST links for fields and messages as part of the commit. + Additionally, the [version history](docs/root/version_history/current.rst) must be updated with + relevant RST links for fields and messages as part of the commit. * Please consider joining the [envoy-dev](https://groups.google.com/forum/#!forum/envoy-dev) mailing list. * If your PR involves any changes to @@ -167,7 +167,7 @@ There are four suggested options for testing new runtime features: 3. Set up integration tests with custom runtime defaults as documented in the [integration test README](https://github.com/envoyproxy/envoy/blob/master/test/integration/README.md) 4. Run a given unit test with the new runtime value explicitly set true as done - for [runtime_flag_override_test](https://github.com/envoyproxy/envoy/blob/master/test/common/runtime/BUILD) + for [runtime_flag_override_test](https://github.com/envoyproxy/envoy/blob/master/test/common/runtime/BUILD) Runtime code is held to the same standard as regular Envoy code, so both the old path and the new should have 100% coverage both with the feature defaulting true diff --git a/PULL_REQUESTS.md b/PULL_REQUESTS.md index 91ed6e3f574f..deb77bb326fd 100644 --- a/PULL_REQUESTS.md +++ b/PULL_REQUESTS.md @@ -54,7 +54,7 @@ N/A if there were no documentation changes. ### Release notes If this change is user impacting OR extension developer impacting (filter API, etc.) you **must** -add a release note to the [version history](docs/root/version_history/current.rst) for the +add a release note to the [version history](docs/root/version_history/current.rst) for the current version. Please include any relevant links. Each release note should be prefixed with the relevant subsystem in **alphabetical order** (see existing examples as a guide) and include links to relevant parts of the documentation. Thank you! Please write in N/A if there are no release notes. @@ -73,11 +73,10 @@ you may instead just tag the PR with the issue: ### Deprecated -If this PR deprecates existing Envoy APIs or code, it should include -an update to the [deprecated file](docs/root/intro/deprecated.rst) and a one line note in the PR -description. +If this PR deprecates existing Envoy APIs or code, it should include an update to the deprecated +section of the [version history](docs/root/version_history/current.rst) and a one line note in the +PR description. If you mark existing APIs or code as deprecated, when the next release is cut, the deprecation script will create and assign an issue to you for cleaning up the deprecated code path. - diff --git a/api/BUILD b/api/BUILD index 084351a7a7de..0dafe82267e9 100644 --- a/api/BUILD +++ b/api/BUILD @@ -216,6 +216,7 @@ proto_library( "//envoy/extensions/filters/network/rbac/v3:pkg", "//envoy/extensions/filters/network/redis_proxy/v3:pkg", "//envoy/extensions/filters/network/sni_cluster/v3:pkg", + "//envoy/extensions/filters/network/sni_dynamic_forward_proxy/v3alpha:pkg", "//envoy/extensions/filters/network/tcp_proxy/v3:pkg", "//envoy/extensions/filters/network/thrift_proxy/filters/ratelimit/v3:pkg", "//envoy/extensions/filters/network/thrift_proxy/v3:pkg", diff --git a/api/envoy/config/bootstrap/v2/bootstrap.proto b/api/envoy/config/bootstrap/v2/bootstrap.proto index 0942c78bd4ab..622304483eb2 100644 --- a/api/envoy/config/bootstrap/v2/bootstrap.proto +++ b/api/envoy/config/bootstrap/v2/bootstrap.proto @@ -12,7 +12,7 @@ import "envoy/api/v2/core/socket_option.proto"; import "envoy/api/v2/listener.proto"; import "envoy/config/metrics/v2/stats.proto"; import "envoy/config/overload/v2alpha/overload.proto"; -import "envoy/config/trace/v2/trace.proto"; +import "envoy/config/trace/v2/http_tracer.proto"; import "google/protobuf/duration.proto"; import "google/protobuf/struct.proto"; diff --git a/api/envoy/config/bootstrap/v3/bootstrap.proto b/api/envoy/config/bootstrap/v3/bootstrap.proto index 9df1bbd8372d..c8219d1b22e3 100644 --- a/api/envoy/config/bootstrap/v3/bootstrap.proto +++ b/api/envoy/config/bootstrap/v3/bootstrap.proto @@ -11,7 +11,7 @@ import "envoy/config/core/v3/socket_option.proto"; import "envoy/config/listener/v3/listener.proto"; import "envoy/config/metrics/v3/stats.proto"; import "envoy/config/overload/v3/overload.proto"; -import "envoy/config/trace/v3/trace.proto"; +import "envoy/config/trace/v3/http_tracer.proto"; import "envoy/extensions/transport_sockets/tls/v3/cert.proto"; import "google/protobuf/duration.proto"; diff --git a/api/envoy/config/core/v3/health_check.proto b/api/envoy/config/core/v3/health_check.proto index 52dda6f9b3c2..f4ef02e0f966 100644 --- a/api/envoy/config/core/v3/health_check.proto +++ b/api/envoy/config/core/v3/health_check.proto @@ -87,11 +87,13 @@ message HealthCheck { // left empty (default value), the name of the cluster this health check is associated // with will be used. The host header can be customized for a specific endpoint by setting the // :ref:`hostname ` field. - string host = 1; + string host = 1 [(validate.rules).string = {well_known_regex: HTTP_HEADER_VALUE strict: false}]; // Specifies the HTTP path that will be requested during health checking. For example // */healthcheck*. - string path = 2 [(validate.rules).string = {min_bytes: 1}]; + string path = 2 [ + (validate.rules).string = {min_bytes: 1 well_known_regex: HTTP_HEADER_VALUE strict: false} + ]; // [#not-implemented-hide:] HTTP specific payload. Payload send = 3; @@ -108,7 +110,9 @@ message HealthCheck { // Specifies a list of HTTP headers that should be removed from each request that is sent to the // health checked cluster. - repeated string request_headers_to_remove = 8; + repeated string request_headers_to_remove = 8 [(validate.rules).repeated = { + items {string {well_known_regex: HTTP_HEADER_NAME strict: false}} + }]; // Specifies a list of HTTP response statuses considered healthy. If provided, replaces default // 200-only policy - 200 must be included explicitly as needed. Ranges follow half-open @@ -169,7 +173,8 @@ message HealthCheck { // left empty (default value), the name of the cluster this health check is associated // with will be used. The authority header can be customized for a specific endpoint by setting // the :ref:`hostname ` field. - string authority = 2; + string authority = 2 + [(validate.rules).string = {well_known_regex: HTTP_HEADER_VALUE strict: false}]; } // Custom health check. diff --git a/api/envoy/config/core/v4alpha/health_check.proto b/api/envoy/config/core/v4alpha/health_check.proto index 0e6c4e73c2a2..1975c309a7de 100644 --- a/api/envoy/config/core/v4alpha/health_check.proto +++ b/api/envoy/config/core/v4alpha/health_check.proto @@ -87,11 +87,13 @@ message HealthCheck { // left empty (default value), the name of the cluster this health check is associated // with will be used. The host header can be customized for a specific endpoint by setting the // :ref:`hostname ` field. - string host = 1; + string host = 1 [(validate.rules).string = {well_known_regex: HTTP_HEADER_VALUE strict: false}]; // Specifies the HTTP path that will be requested during health checking. For example // */healthcheck*. - string path = 2 [(validate.rules).string = {min_bytes: 1}]; + string path = 2 [ + (validate.rules).string = {min_bytes: 1 well_known_regex: HTTP_HEADER_VALUE strict: false} + ]; // [#not-implemented-hide:] HTTP specific payload. Payload send = 3; @@ -108,7 +110,9 @@ message HealthCheck { // Specifies a list of HTTP headers that should be removed from each request that is sent to the // health checked cluster. - repeated string request_headers_to_remove = 8; + repeated string request_headers_to_remove = 8 [(validate.rules).repeated = { + items {string {well_known_regex: HTTP_HEADER_NAME strict: false}} + }]; // Specifies a list of HTTP response statuses considered healthy. If provided, replaces default // 200-only policy - 200 must be included explicitly as needed. Ranges follow half-open @@ -169,7 +173,8 @@ message HealthCheck { // left empty (default value), the name of the cluster this health check is associated // with will be used. The authority header can be customized for a specific endpoint by setting // the :ref:`hostname ` field. - string authority = 2; + string authority = 2 + [(validate.rules).string = {well_known_regex: HTTP_HEADER_VALUE strict: false}]; } // Custom health check. diff --git a/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto b/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto index f760c25dcde3..c78e69b2ae30 100644 --- a/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto +++ b/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto @@ -7,7 +7,7 @@ import "envoy/api/v2/core/protocol.proto"; import "envoy/api/v2/route.proto"; import "envoy/api/v2/scoped_route.proto"; import "envoy/config/filter/accesslog/v2/accesslog.proto"; -import "envoy/config/trace/v2/trace.proto"; +import "envoy/config/trace/v2/http_tracer.proto"; import "envoy/type/percent.proto"; import "envoy/type/tracing/v2/custom_tag.proto"; diff --git a/api/envoy/config/route/v3/route_components.proto b/api/envoy/config/route/v3/route_components.proto index f63f0961249f..70c52010efa0 100644 --- a/api/envoy/config/route/v3/route_components.proto +++ b/api/envoy/config/route/v3/route_components.proto @@ -108,7 +108,9 @@ message VirtualHost { // Specifies a list of HTTP headers that should be removed from each request // handled by this virtual host. - repeated string request_headers_to_remove = 13; + repeated string request_headers_to_remove = 13 [(validate.rules).repeated = { + items {string {min_bytes: 1 well_known_regex: HTTP_HEADER_NAME strict: false}} + }]; // Specifies a list of HTTP headers that should be added to each response // handled by this virtual host. Headers specified at this level are applied @@ -252,7 +254,9 @@ message Route { // Specifies a list of HTTP headers that should be removed from each request // matching this route. - repeated string request_headers_to_remove = 12; + repeated string request_headers_to_remove = 12 [(validate.rules).repeated = { + items {string {min_bytes: 1 well_known_regex: HTTP_HEADER_NAME strict: false}} + }]; // Specifies a set of headers that will be added to responses to requests // matching this route. Headers specified at this level are applied before diff --git a/api/envoy/config/route/v4alpha/route_components.proto b/api/envoy/config/route/v4alpha/route_components.proto index 33f8d64543df..e813b632edb0 100644 --- a/api/envoy/config/route/v4alpha/route_components.proto +++ b/api/envoy/config/route/v4alpha/route_components.proto @@ -108,7 +108,9 @@ message VirtualHost { // Specifies a list of HTTP headers that should be removed from each request // handled by this virtual host. - repeated string request_headers_to_remove = 13; + repeated string request_headers_to_remove = 13 [(validate.rules).repeated = { + items {string {min_bytes: 1 well_known_regex: HTTP_HEADER_NAME strict: false}} + }]; // Specifies a list of HTTP headers that should be added to each response // handled by this virtual host. Headers specified at this level are applied @@ -252,7 +254,9 @@ message Route { // Specifies a list of HTTP headers that should be removed from each request // matching this route. - repeated string request_headers_to_remove = 12; + repeated string request_headers_to_remove = 12 [(validate.rules).repeated = { + items {string {min_bytes: 1 well_known_regex: HTTP_HEADER_NAME strict: false}} + }]; // Specifies a set of headers that will be added to responses to requests // matching this route. Headers specified at this level are applied before diff --git a/api/envoy/config/trace/v2/datadog.proto b/api/envoy/config/trace/v2/datadog.proto new file mode 100644 index 000000000000..0992601a8acc --- /dev/null +++ b/api/envoy/config/trace/v2/datadog.proto @@ -0,0 +1,23 @@ +syntax = "proto3"; + +package envoy.config.trace.v2; + +import "udpa/annotations/status.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.trace.v2"; +option java_outer_classname = "DatadogProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = FROZEN; + +// [#protodoc-title: Datadog tracer] + +// Configuration for the Datadog tracer. +// [#extension: envoy.tracers.datadog] +message DatadogConfig { + // The cluster to use for submitting traces to the Datadog agent. + string collector_cluster = 1 [(validate.rules).string = {min_bytes: 1}]; + + // The name used for the service when traces are generated by envoy. + string service_name = 2 [(validate.rules).string = {min_bytes: 1}]; +} diff --git a/api/envoy/config/trace/v2/dynamic_ot.proto b/api/envoy/config/trace/v2/dynamic_ot.proto new file mode 100644 index 000000000000..55c6d401b335 --- /dev/null +++ b/api/envoy/config/trace/v2/dynamic_ot.proto @@ -0,0 +1,29 @@ +syntax = "proto3"; + +package envoy.config.trace.v2; + +import "google/protobuf/struct.proto"; + +import "udpa/annotations/status.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.trace.v2"; +option java_outer_classname = "DynamicOtProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = FROZEN; + +// [#protodoc-title: Dynamically loadable OpenTracing tracer] + +// DynamicOtConfig is used to dynamically load a tracer from a shared library +// that implements the `OpenTracing dynamic loading API +// `_. +// [#extension: envoy.tracers.dynamic_ot] +message DynamicOtConfig { + // Dynamic library implementing the `OpenTracing API + // `_. + string library = 1 [(validate.rules).string = {min_bytes: 1}]; + + // The configuration to use when creating a tracer from the given dynamic + // library. + google.protobuf.Struct config = 2; +} diff --git a/api/envoy/config/trace/v2/http_tracer.proto b/api/envoy/config/trace/v2/http_tracer.proto new file mode 100644 index 000000000000..fba830b987b6 --- /dev/null +++ b/api/envoy/config/trace/v2/http_tracer.proto @@ -0,0 +1,65 @@ +syntax = "proto3"; + +package envoy.config.trace.v2; + +import "google/protobuf/any.proto"; +import "google/protobuf/struct.proto"; + +import "udpa/annotations/status.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.trace.v2"; +option java_outer_classname = "HttpTracerProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = FROZEN; + +// [#protodoc-title: Tracing] +// Tracing :ref:`architecture overview `. + +// The tracing configuration specifies settings for an HTTP tracer provider used by Envoy. +// +// Envoy may support other tracers in the future, but right now the HTTP tracer is the only one +// supported. +// +// .. attention:: +// +// Use of this message type has been deprecated in favor of direct use of +// :ref:`Tracing.Http `. +message Tracing { + // Configuration for an HTTP tracer provider used by Envoy. + // + // The configuration is defined by the + // :ref:`HttpConnectionManager.Tracing ` + // :ref:`provider ` + // field. + message Http { + // The name of the HTTP trace driver to instantiate. The name must match a + // supported HTTP trace driver. Built-in trace drivers: + // + // - *envoy.tracers.lightstep* + // - *envoy.tracers.zipkin* + // - *envoy.tracers.dynamic_ot* + // - *envoy.tracers.datadog* + // - *envoy.tracers.opencensus* + // - *envoy.tracers.xray* + string name = 1 [(validate.rules).string = {min_bytes: 1}]; + + // Trace driver specific configuration which depends on the driver being instantiated. + // See the trace drivers for examples: + // + // - :ref:`LightstepConfig ` + // - :ref:`ZipkinConfig ` + // - :ref:`DynamicOtConfig ` + // - :ref:`DatadogConfig ` + // - :ref:`OpenCensusConfig ` + // - :ref:`AWS X-Ray ` + oneof config_type { + google.protobuf.Struct config = 2 [deprecated = true]; + + google.protobuf.Any typed_config = 3; + } + } + + // Provides configuration for the HTTP tracer. + Http http = 1; +} diff --git a/api/envoy/config/trace/v2/lightstep.proto b/api/envoy/config/trace/v2/lightstep.proto new file mode 100644 index 000000000000..849749baaa0d --- /dev/null +++ b/api/envoy/config/trace/v2/lightstep.proto @@ -0,0 +1,43 @@ +syntax = "proto3"; + +package envoy.config.trace.v2; + +import "udpa/annotations/status.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.trace.v2"; +option java_outer_classname = "LightstepProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = FROZEN; + +// [#protodoc-title: LightStep tracer] + +// Configuration for the LightStep tracer. +// [#extension: envoy.tracers.lightstep] +message LightstepConfig { + // Available propagation modes + enum PropagationMode { + // Propagate trace context in the single header x-ot-span-context. + ENVOY = 0; + + // Propagate trace context using LightStep's native format. + LIGHTSTEP = 1; + + // Propagate trace context using the b3 format. + B3 = 2; + + // Propagation trace context using the w3 trace-context standard. + TRACE_CONTEXT = 3; + } + + // The cluster manager cluster that hosts the LightStep collectors. + string collector_cluster = 1 [(validate.rules).string = {min_bytes: 1}]; + + // File containing the access token to the `LightStep + // `_ API. + string access_token_file = 2 [(validate.rules).string = {min_bytes: 1}]; + + // Propagation modes to use by LightStep's tracer. + repeated PropagationMode propagation_modes = 3 + [(validate.rules).repeated = {items {enum {defined_only: true}}}]; +} diff --git a/api/envoy/config/trace/v2/opencensus.proto b/api/envoy/config/trace/v2/opencensus.proto new file mode 100644 index 000000000000..8c8dab94a875 --- /dev/null +++ b/api/envoy/config/trace/v2/opencensus.proto @@ -0,0 +1,93 @@ +syntax = "proto3"; + +package envoy.config.trace.v2; + +import "envoy/api/v2/core/grpc_service.proto"; + +import "opencensus/proto/trace/v1/trace_config.proto"; + +import "udpa/annotations/status.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.trace.v2"; +option java_outer_classname = "OpencensusProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = FROZEN; + +// [#protodoc-title: OpenCensus tracer] + +// Configuration for the OpenCensus tracer. +// [#next-free-field: 15] +// [#extension: envoy.tracers.opencensus] +message OpenCensusConfig { + enum TraceContext { + // No-op default, no trace context is utilized. + NONE = 0; + + // W3C Trace-Context format "traceparent:" header. + TRACE_CONTEXT = 1; + + // Binary "grpc-trace-bin:" header. + GRPC_TRACE_BIN = 2; + + // "X-Cloud-Trace-Context:" header. + CLOUD_TRACE_CONTEXT = 3; + + // X-B3-* headers. + B3 = 4; + } + + reserved 7; + + // Configures tracing, e.g. the sampler, max number of annotations, etc. + opencensus.proto.trace.v1.TraceConfig trace_config = 1; + + // Enables the stdout exporter if set to true. This is intended for debugging + // purposes. + bool stdout_exporter_enabled = 2; + + // Enables the Stackdriver exporter if set to true. The project_id must also + // be set. + bool stackdriver_exporter_enabled = 3; + + // The Cloud project_id to use for Stackdriver tracing. + string stackdriver_project_id = 4; + + // (optional) By default, the Stackdriver exporter will connect to production + // Stackdriver. If stackdriver_address is non-empty, it will instead connect + // to this address, which is in the gRPC format: + // https://github.com/grpc/grpc/blob/master/doc/naming.md + string stackdriver_address = 10; + + // (optional) The gRPC server that hosts Stackdriver tracing service. Only + // Google gRPC is supported. If :ref:`target_uri ` + // is not provided, the default production Stackdriver address will be used. + api.v2.core.GrpcService stackdriver_grpc_service = 13; + + // Enables the Zipkin exporter if set to true. The url and service name must + // also be set. + bool zipkin_exporter_enabled = 5; + + // The URL to Zipkin, e.g. "http://127.0.0.1:9411/api/v2/spans" + string zipkin_url = 6; + + // Enables the OpenCensus Agent exporter if set to true. The ocagent_address or + // ocagent_grpc_service must also be set. + bool ocagent_exporter_enabled = 11; + + // The address of the OpenCensus Agent, if its exporter is enabled, in gRPC + // format: https://github.com/grpc/grpc/blob/master/doc/naming.md + // [#comment:TODO: deprecate this field] + string ocagent_address = 12; + + // (optional) The gRPC server hosted by the OpenCensus Agent. Only Google gRPC is supported. + // This is only used if the ocagent_address is left empty. + api.v2.core.GrpcService ocagent_grpc_service = 14; + + // List of incoming trace context headers we will accept. First one found + // wins. + repeated TraceContext incoming_trace_context = 8; + + // List of outgoing trace context headers we will produce. + repeated TraceContext outgoing_trace_context = 9; +} diff --git a/api/envoy/config/trace/v2/service.proto b/api/envoy/config/trace/v2/service.proto new file mode 100644 index 000000000000..d102499b6261 --- /dev/null +++ b/api/envoy/config/trace/v2/service.proto @@ -0,0 +1,21 @@ +syntax = "proto3"; + +package envoy.config.trace.v2; + +import "envoy/api/v2/core/grpc_service.proto"; + +import "udpa/annotations/status.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.trace.v2"; +option java_outer_classname = "ServiceProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = FROZEN; + +// [#protodoc-title: Trace Service] + +// Configuration structure. +message TraceServiceConfig { + // The upstream gRPC cluster that hosts the metrics service. + api.v2.core.GrpcService grpc_service = 1 [(validate.rules).message = {required: true}]; +} diff --git a/api/envoy/config/trace/v2/trace.proto b/api/envoy/config/trace/v2/trace.proto index 308a97e4eef4..9f0670b28f7a 100644 --- a/api/envoy/config/trace/v2/trace.proto +++ b/api/envoy/config/trace/v2/trace.proto @@ -2,254 +2,16 @@ syntax = "proto3"; package envoy.config.trace.v2; -import "envoy/api/v2/core/grpc_service.proto"; - -import "google/protobuf/any.proto"; -import "google/protobuf/struct.proto"; -import "google/protobuf/wrappers.proto"; - -import "opencensus/proto/trace/v1/trace_config.proto"; - -import "envoy/annotations/deprecation.proto"; import "udpa/annotations/status.proto"; -import "validate/validate.proto"; + +import public "envoy/config/trace/v2/datadog.proto"; +import public "envoy/config/trace/v2/dynamic_ot.proto"; +import public "envoy/config/trace/v2/http_tracer.proto"; +import public "envoy/config/trace/v2/lightstep.proto"; +import public "envoy/config/trace/v2/opencensus.proto"; +import public "envoy/config/trace/v2/service.proto"; +import public "envoy/config/trace/v2/zipkin.proto"; option java_package = "io.envoyproxy.envoy.config.trace.v2"; option java_outer_classname = "TraceProto"; option java_multiple_files = true; -option (udpa.annotations.file_status).package_version_status = FROZEN; - -// [#protodoc-title: Tracing] -// Tracing :ref:`architecture overview `. - -// The tracing configuration specifies settings for an HTTP tracer provider used by Envoy. -// -// Envoy may support other tracers in the future, but right now the HTTP tracer is the only one -// supported. -// -// .. attention:: -// -// Use of this message type has been deprecated in favor of direct use of -// :ref:`Tracing.Http `. -message Tracing { - // Configuration for an HTTP tracer provider used by Envoy. - // - // The configuration is defined by the - // :ref:`HttpConnectionManager.Tracing ` - // :ref:`provider ` - // field. - message Http { - // The name of the HTTP trace driver to instantiate. The name must match a - // supported HTTP trace driver. Built-in trace drivers: - // - // - *envoy.tracers.lightstep* - // - *envoy.tracers.zipkin* - // - *envoy.tracers.dynamic_ot* - // - *envoy.tracers.datadog* - // - *envoy.tracers.opencensus* - // - *envoy.tracers.xray* - string name = 1 [(validate.rules).string = {min_bytes: 1}]; - - // Trace driver specific configuration which depends on the driver being instantiated. - // See the trace drivers for examples: - // - // - :ref:`LightstepConfig ` - // - :ref:`ZipkinConfig ` - // - :ref:`DynamicOtConfig ` - // - :ref:`DatadogConfig ` - // - :ref:`OpenCensusConfig ` - // - :ref:`AWS X-Ray ` - oneof config_type { - google.protobuf.Struct config = 2 [deprecated = true]; - - google.protobuf.Any typed_config = 3; - } - } - - // Provides configuration for the HTTP tracer. - Http http = 1; -} - -// Configuration for the LightStep tracer. -// [#extension: envoy.tracers.lightstep] -message LightstepConfig { - // Available propagation modes - enum PropagationMode { - // Propagate trace context in the single header x-ot-span-context. - ENVOY = 0; - - // Propagate trace context using LightStep's native format. - LIGHTSTEP = 1; - - // Propagate trace context using the b3 format. - B3 = 2; - - // Propagation trace context using the w3 trace-context standard. - TRACE_CONTEXT = 3; - } - - // The cluster manager cluster that hosts the LightStep collectors. - string collector_cluster = 1 [(validate.rules).string = {min_bytes: 1}]; - - // File containing the access token to the `LightStep - // `_ API. - string access_token_file = 2 [(validate.rules).string = {min_bytes: 1}]; - - // Propagation modes to use by LightStep's tracer. - repeated PropagationMode propagation_modes = 3 - [(validate.rules).repeated = {items {enum {defined_only: true}}}]; -} - -// Configuration for the Zipkin tracer. -// [#extension: envoy.tracers.zipkin] -// [#next-free-field: 6] -message ZipkinConfig { - // Available Zipkin collector endpoint versions. - enum CollectorEndpointVersion { - // Zipkin API v1, JSON over HTTP. - // [#comment: The default implementation of Zipkin client before this field is added was only v1 - // and the way user configure this was by not explicitly specifying the version. Consequently, - // before this is added, the corresponding Zipkin collector expected to receive v1 payload. - // Hence the motivation of adding HTTP_JSON_V1 as the default is to avoid a breaking change when - // user upgrading Envoy with this change. Furthermore, we also immediately deprecate this field, - // since in Zipkin realm this v1 version is considered to be not preferable anymore.] - HTTP_JSON_V1 = 0 [deprecated = true, (envoy.annotations.disallowed_by_default_enum) = true]; - - // Zipkin API v2, JSON over HTTP. - HTTP_JSON = 1; - - // Zipkin API v2, protobuf over HTTP. - HTTP_PROTO = 2; - - // [#not-implemented-hide:] - GRPC = 3; - } - - // The cluster manager cluster that hosts the Zipkin collectors. Note that the - // Zipkin cluster must be defined in the :ref:`Bootstrap static cluster - // resources `. - string collector_cluster = 1 [(validate.rules).string = {min_bytes: 1}]; - - // The API endpoint of the Zipkin service where the spans will be sent. When - // using a standard Zipkin installation, the API endpoint is typically - // /api/v1/spans, which is the default value. - string collector_endpoint = 2 [(validate.rules).string = {min_bytes: 1}]; - - // Determines whether a 128bit trace id will be used when creating a new - // trace instance. The default value is false, which will result in a 64 bit trace id being used. - bool trace_id_128bit = 3; - - // Determines whether client and server spans will share the same span context. - // The default value is true. - google.protobuf.BoolValue shared_span_context = 4; - - // Determines the selected collector endpoint version. By default, the ``HTTP_JSON_V1`` will be - // used. - CollectorEndpointVersion collector_endpoint_version = 5; -} - -// DynamicOtConfig is used to dynamically load a tracer from a shared library -// that implements the `OpenTracing dynamic loading API -// `_. -// [#extension: envoy.tracers.dynamic_ot] -message DynamicOtConfig { - // Dynamic library implementing the `OpenTracing API - // `_. - string library = 1 [(validate.rules).string = {min_bytes: 1}]; - - // The configuration to use when creating a tracer from the given dynamic - // library. - google.protobuf.Struct config = 2; -} - -// Configuration for the Datadog tracer. -// [#extension: envoy.tracers.datadog] -message DatadogConfig { - // The cluster to use for submitting traces to the Datadog agent. - string collector_cluster = 1 [(validate.rules).string = {min_bytes: 1}]; - - // The name used for the service when traces are generated by envoy. - string service_name = 2 [(validate.rules).string = {min_bytes: 1}]; -} - -// Configuration for the OpenCensus tracer. -// [#next-free-field: 15] -// [#extension: envoy.tracers.opencensus] -message OpenCensusConfig { - enum TraceContext { - // No-op default, no trace context is utilized. - NONE = 0; - - // W3C Trace-Context format "traceparent:" header. - TRACE_CONTEXT = 1; - - // Binary "grpc-trace-bin:" header. - GRPC_TRACE_BIN = 2; - - // "X-Cloud-Trace-Context:" header. - CLOUD_TRACE_CONTEXT = 3; - - // X-B3-* headers. - B3 = 4; - } - - reserved 7; - - // Configures tracing, e.g. the sampler, max number of annotations, etc. - opencensus.proto.trace.v1.TraceConfig trace_config = 1; - - // Enables the stdout exporter if set to true. This is intended for debugging - // purposes. - bool stdout_exporter_enabled = 2; - - // Enables the Stackdriver exporter if set to true. The project_id must also - // be set. - bool stackdriver_exporter_enabled = 3; - - // The Cloud project_id to use for Stackdriver tracing. - string stackdriver_project_id = 4; - - // (optional) By default, the Stackdriver exporter will connect to production - // Stackdriver. If stackdriver_address is non-empty, it will instead connect - // to this address, which is in the gRPC format: - // https://github.com/grpc/grpc/blob/master/doc/naming.md - string stackdriver_address = 10; - - // (optional) The gRPC server that hosts Stackdriver tracing service. Only - // Google gRPC is supported. If :ref:`target_uri ` - // is not provided, the default production Stackdriver address will be used. - api.v2.core.GrpcService stackdriver_grpc_service = 13; - - // Enables the Zipkin exporter if set to true. The url and service name must - // also be set. - bool zipkin_exporter_enabled = 5; - - // The URL to Zipkin, e.g. "http://127.0.0.1:9411/api/v2/spans" - string zipkin_url = 6; - - // Enables the OpenCensus Agent exporter if set to true. The ocagent_address or - // ocagent_grpc_service must also be set. - bool ocagent_exporter_enabled = 11; - - // The address of the OpenCensus Agent, if its exporter is enabled, in gRPC - // format: https://github.com/grpc/grpc/blob/master/doc/naming.md - // [#comment:TODO: deprecate this field] - string ocagent_address = 12; - - // (optional) The gRPC server hosted by the OpenCensus Agent. Only Google gRPC is supported. - // This is only used if the ocagent_address is left empty. - api.v2.core.GrpcService ocagent_grpc_service = 14; - - // List of incoming trace context headers we will accept. First one found - // wins. - repeated TraceContext incoming_trace_context = 8; - - // List of outgoing trace context headers we will produce. - repeated TraceContext outgoing_trace_context = 9; -} - -// Configuration structure. -message TraceServiceConfig { - // The upstream gRPC cluster that hosts the metrics service. - api.v2.core.GrpcService grpc_service = 1 [(validate.rules).message = {required: true}]; -} diff --git a/api/envoy/config/trace/v2/zipkin.proto b/api/envoy/config/trace/v2/zipkin.proto new file mode 100644 index 000000000000..a825d85bb7f9 --- /dev/null +++ b/api/envoy/config/trace/v2/zipkin.proto @@ -0,0 +1,64 @@ +syntax = "proto3"; + +package envoy.config.trace.v2; + +import "google/protobuf/wrappers.proto"; + +import "envoy/annotations/deprecation.proto"; +import "udpa/annotations/status.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.trace.v2"; +option java_outer_classname = "ZipkinProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = FROZEN; + +// [#protodoc-title: Zipkin tracer] + +// Configuration for the Zipkin tracer. +// [#extension: envoy.tracers.zipkin] +// [#next-free-field: 6] +message ZipkinConfig { + // Available Zipkin collector endpoint versions. + enum CollectorEndpointVersion { + // Zipkin API v1, JSON over HTTP. + // [#comment: The default implementation of Zipkin client before this field is added was only v1 + // and the way user configure this was by not explicitly specifying the version. Consequently, + // before this is added, the corresponding Zipkin collector expected to receive v1 payload. + // Hence the motivation of adding HTTP_JSON_V1 as the default is to avoid a breaking change when + // user upgrading Envoy with this change. Furthermore, we also immediately deprecate this field, + // since in Zipkin realm this v1 version is considered to be not preferable anymore.] + HTTP_JSON_V1 = 0 [deprecated = true, (envoy.annotations.disallowed_by_default_enum) = true]; + + // Zipkin API v2, JSON over HTTP. + HTTP_JSON = 1; + + // Zipkin API v2, protobuf over HTTP. + HTTP_PROTO = 2; + + // [#not-implemented-hide:] + GRPC = 3; + } + + // The cluster manager cluster that hosts the Zipkin collectors. Note that the + // Zipkin cluster must be defined in the :ref:`Bootstrap static cluster + // resources `. + string collector_cluster = 1 [(validate.rules).string = {min_bytes: 1}]; + + // The API endpoint of the Zipkin service where the spans will be sent. When + // using a standard Zipkin installation, the API endpoint is typically + // /api/v1/spans, which is the default value. + string collector_endpoint = 2 [(validate.rules).string = {min_bytes: 1}]; + + // Determines whether a 128bit trace id will be used when creating a new + // trace instance. The default value is false, which will result in a 64 bit trace id being used. + bool trace_id_128bit = 3; + + // Determines whether client and server spans will share the same span context. + // The default value is true. + google.protobuf.BoolValue shared_span_context = 4; + + // Determines the selected collector endpoint version. By default, the ``HTTP_JSON_V1`` will be + // used. + CollectorEndpointVersion collector_endpoint_version = 5; +} diff --git a/api/envoy/config/trace/v3/datadog.proto b/api/envoy/config/trace/v3/datadog.proto new file mode 100644 index 000000000000..f1fe3e666125 --- /dev/null +++ b/api/envoy/config/trace/v3/datadog.proto @@ -0,0 +1,29 @@ +syntax = "proto3"; + +package envoy.config.trace.v3; + +import "udpa/annotations/migrate.proto"; +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.trace.v3"; +option java_outer_classname = "DatadogProto"; +option java_multiple_files = true; +option (udpa.annotations.file_migrate).move_to_package = "envoy.extensions.tracers.datadog.v4alpha"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Datadog tracer] + +// Configuration for the Datadog tracer. +// [#extension: envoy.tracers.datadog] +message DatadogConfig { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.trace.v2.DatadogConfig"; + + // The cluster to use for submitting traces to the Datadog agent. + string collector_cluster = 1 [(validate.rules).string = {min_bytes: 1}]; + + // The name used for the service when traces are generated by envoy. + string service_name = 2 [(validate.rules).string = {min_bytes: 1}]; +} diff --git a/api/envoy/config/trace/v3/dynamic_ot.proto b/api/envoy/config/trace/v3/dynamic_ot.proto new file mode 100644 index 000000000000..fb372da8c52a --- /dev/null +++ b/api/envoy/config/trace/v3/dynamic_ot.proto @@ -0,0 +1,36 @@ +syntax = "proto3"; + +package envoy.config.trace.v3; + +import "google/protobuf/struct.proto"; + +import "udpa/annotations/migrate.proto"; +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.trace.v3"; +option java_outer_classname = "DynamicOtProto"; +option java_multiple_files = true; +option (udpa.annotations.file_migrate).move_to_package = + "envoy.extensions.tracers.dynamic_ot.v4alpha"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Dynamically loadable OpenTracing tracer] + +// DynamicOtConfig is used to dynamically load a tracer from a shared library +// that implements the `OpenTracing dynamic loading API +// `_. +// [#extension: envoy.tracers.dynamic_ot] +message DynamicOtConfig { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.trace.v2.DynamicOtConfig"; + + // Dynamic library implementing the `OpenTracing API + // `_. + string library = 1 [(validate.rules).string = {min_bytes: 1}]; + + // The configuration to use when creating a tracer from the given dynamic + // library. + google.protobuf.Struct config = 2; +} diff --git a/api/envoy/config/trace/v3/http_tracer.proto b/api/envoy/config/trace/v3/http_tracer.proto new file mode 100644 index 000000000000..2a87a28db25e --- /dev/null +++ b/api/envoy/config/trace/v3/http_tracer.proto @@ -0,0 +1,73 @@ +syntax = "proto3"; + +package envoy.config.trace.v3; + +import "google/protobuf/any.proto"; +import "google/protobuf/struct.proto"; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.trace.v3"; +option java_outer_classname = "HttpTracerProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Tracing] +// Tracing :ref:`architecture overview `. + +// The tracing configuration specifies settings for an HTTP tracer provider used by Envoy. +// +// Envoy may support other tracers in the future, but right now the HTTP tracer is the only one +// supported. +// +// .. attention:: +// +// Use of this message type has been deprecated in favor of direct use of +// :ref:`Tracing.Http `. +message Tracing { + option (udpa.annotations.versioning).previous_message_type = "envoy.config.trace.v2.Tracing"; + + // Configuration for an HTTP tracer provider used by Envoy. + // + // The configuration is defined by the + // :ref:`HttpConnectionManager.Tracing ` + // :ref:`provider ` + // field. + message Http { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.trace.v2.Tracing.Http"; + + reserved 2; + + reserved "config"; + + // The name of the HTTP trace driver to instantiate. The name must match a + // supported HTTP trace driver. Built-in trace drivers: + // + // - *envoy.tracers.lightstep* + // - *envoy.tracers.zipkin* + // - *envoy.tracers.dynamic_ot* + // - *envoy.tracers.datadog* + // - *envoy.tracers.opencensus* + // - *envoy.tracers.xray* + string name = 1 [(validate.rules).string = {min_bytes: 1}]; + + // Trace driver specific configuration which depends on the driver being instantiated. + // See the trace drivers for examples: + // + // - :ref:`LightstepConfig ` + // - :ref:`ZipkinConfig ` + // - :ref:`DynamicOtConfig ` + // - :ref:`DatadogConfig ` + // - :ref:`OpenCensusConfig ` + // - :ref:`AWS X-Ray ` + oneof config_type { + google.protobuf.Any typed_config = 3; + } + } + + // Provides configuration for the HTTP tracer. + Http http = 1; +} diff --git a/api/envoy/config/trace/v3/lightstep.proto b/api/envoy/config/trace/v3/lightstep.proto new file mode 100644 index 000000000000..0e0b60b5bddb --- /dev/null +++ b/api/envoy/config/trace/v3/lightstep.proto @@ -0,0 +1,50 @@ +syntax = "proto3"; + +package envoy.config.trace.v3; + +import "udpa/annotations/migrate.proto"; +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.trace.v3"; +option java_outer_classname = "LightstepProto"; +option java_multiple_files = true; +option (udpa.annotations.file_migrate).move_to_package = + "envoy.extensions.tracers.lightstep.v4alpha"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: LightStep tracer] + +// Configuration for the LightStep tracer. +// [#extension: envoy.tracers.lightstep] +message LightstepConfig { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.trace.v2.LightstepConfig"; + + // Available propagation modes + enum PropagationMode { + // Propagate trace context in the single header x-ot-span-context. + ENVOY = 0; + + // Propagate trace context using LightStep's native format. + LIGHTSTEP = 1; + + // Propagate trace context using the b3 format. + B3 = 2; + + // Propagation trace context using the w3 trace-context standard. + TRACE_CONTEXT = 3; + } + + // The cluster manager cluster that hosts the LightStep collectors. + string collector_cluster = 1 [(validate.rules).string = {min_bytes: 1}]; + + // File containing the access token to the `LightStep + // `_ API. + string access_token_file = 2 [(validate.rules).string = {min_bytes: 1}]; + + // Propagation modes to use by LightStep's tracer. + repeated PropagationMode propagation_modes = 3 + [(validate.rules).repeated = {items {enum {defined_only: true}}}]; +} diff --git a/api/envoy/config/trace/v3/opencensus.proto b/api/envoy/config/trace/v3/opencensus.proto new file mode 100644 index 000000000000..39313139177f --- /dev/null +++ b/api/envoy/config/trace/v3/opencensus.proto @@ -0,0 +1,100 @@ +syntax = "proto3"; + +package envoy.config.trace.v3; + +import "envoy/config/core/v3/grpc_service.proto"; + +import "opencensus/proto/trace/v1/trace_config.proto"; + +import "udpa/annotations/migrate.proto"; +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.trace.v3"; +option java_outer_classname = "OpencensusProto"; +option java_multiple_files = true; +option (udpa.annotations.file_migrate).move_to_package = + "envoy.extensions.tracers.opencensus.v4alpha"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: OpenCensus tracer] + +// Configuration for the OpenCensus tracer. +// [#next-free-field: 15] +// [#extension: envoy.tracers.opencensus] +message OpenCensusConfig { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.trace.v2.OpenCensusConfig"; + + enum TraceContext { + // No-op default, no trace context is utilized. + NONE = 0; + + // W3C Trace-Context format "traceparent:" header. + TRACE_CONTEXT = 1; + + // Binary "grpc-trace-bin:" header. + GRPC_TRACE_BIN = 2; + + // "X-Cloud-Trace-Context:" header. + CLOUD_TRACE_CONTEXT = 3; + + // X-B3-* headers. + B3 = 4; + } + + reserved 7; + + // Configures tracing, e.g. the sampler, max number of annotations, etc. + opencensus.proto.trace.v1.TraceConfig trace_config = 1; + + // Enables the stdout exporter if set to true. This is intended for debugging + // purposes. + bool stdout_exporter_enabled = 2; + + // Enables the Stackdriver exporter if set to true. The project_id must also + // be set. + bool stackdriver_exporter_enabled = 3; + + // The Cloud project_id to use for Stackdriver tracing. + string stackdriver_project_id = 4; + + // (optional) By default, the Stackdriver exporter will connect to production + // Stackdriver. If stackdriver_address is non-empty, it will instead connect + // to this address, which is in the gRPC format: + // https://github.com/grpc/grpc/blob/master/doc/naming.md + string stackdriver_address = 10; + + // (optional) The gRPC server that hosts Stackdriver tracing service. Only + // Google gRPC is supported. If :ref:`target_uri ` + // is not provided, the default production Stackdriver address will be used. + core.v3.GrpcService stackdriver_grpc_service = 13; + + // Enables the Zipkin exporter if set to true. The url and service name must + // also be set. + bool zipkin_exporter_enabled = 5; + + // The URL to Zipkin, e.g. "http://127.0.0.1:9411/api/v2/spans" + string zipkin_url = 6; + + // Enables the OpenCensus Agent exporter if set to true. The ocagent_address or + // ocagent_grpc_service must also be set. + bool ocagent_exporter_enabled = 11; + + // The address of the OpenCensus Agent, if its exporter is enabled, in gRPC + // format: https://github.com/grpc/grpc/blob/master/doc/naming.md + // [#comment:TODO: deprecate this field] + string ocagent_address = 12; + + // (optional) The gRPC server hosted by the OpenCensus Agent. Only Google gRPC is supported. + // This is only used if the ocagent_address is left empty. + core.v3.GrpcService ocagent_grpc_service = 14; + + // List of incoming trace context headers we will accept. First one found + // wins. + repeated TraceContext incoming_trace_context = 8; + + // List of outgoing trace context headers we will produce. + repeated TraceContext outgoing_trace_context = 9; +} diff --git a/api/envoy/config/trace/v3/service.proto b/api/envoy/config/trace/v3/service.proto new file mode 100644 index 000000000000..1e01ff61847f --- /dev/null +++ b/api/envoy/config/trace/v3/service.proto @@ -0,0 +1,25 @@ +syntax = "proto3"; + +package envoy.config.trace.v3; + +import "envoy/config/core/v3/grpc_service.proto"; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.trace.v3"; +option java_outer_classname = "ServiceProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Trace Service] + +// Configuration structure. +message TraceServiceConfig { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.trace.v2.TraceServiceConfig"; + + // The upstream gRPC cluster that hosts the metrics service. + core.v3.GrpcService grpc_service = 1 [(validate.rules).message = {required: true}]; +} diff --git a/api/envoy/config/trace/v3/trace.proto b/api/envoy/config/trace/v3/trace.proto index 55534a895078..e1db72a2fd5a 100644 --- a/api/envoy/config/trace/v3/trace.proto +++ b/api/envoy/config/trace/v3/trace.proto @@ -2,280 +2,16 @@ syntax = "proto3"; package envoy.config.trace.v3; -import "envoy/config/core/v3/grpc_service.proto"; - -import "google/protobuf/any.proto"; -import "google/protobuf/struct.proto"; -import "google/protobuf/wrappers.proto"; - -import "opencensus/proto/trace/v1/trace_config.proto"; - -import "envoy/annotations/deprecation.proto"; import "udpa/annotations/status.proto"; -import "udpa/annotations/versioning.proto"; -import "validate/validate.proto"; + +import public "envoy/config/trace/v3/datadog.proto"; +import public "envoy/config/trace/v3/dynamic_ot.proto"; +import public "envoy/config/trace/v3/http_tracer.proto"; +import public "envoy/config/trace/v3/lightstep.proto"; +import public "envoy/config/trace/v3/opencensus.proto"; +import public "envoy/config/trace/v3/service.proto"; +import public "envoy/config/trace/v3/zipkin.proto"; option java_package = "io.envoyproxy.envoy.config.trace.v3"; option java_outer_classname = "TraceProto"; option java_multiple_files = true; -option (udpa.annotations.file_status).package_version_status = ACTIVE; - -// [#protodoc-title: Tracing] -// Tracing :ref:`architecture overview `. - -// The tracing configuration specifies settings for an HTTP tracer provider used by Envoy. -// -// Envoy may support other tracers in the future, but right now the HTTP tracer is the only one -// supported. -// -// .. attention:: -// -// Use of this message type has been deprecated in favor of direct use of -// :ref:`Tracing.Http `. -message Tracing { - option (udpa.annotations.versioning).previous_message_type = "envoy.config.trace.v2.Tracing"; - - // Configuration for an HTTP tracer provider used by Envoy. - // - // The configuration is defined by the - // :ref:`HttpConnectionManager.Tracing ` - // :ref:`provider ` - // field. - message Http { - option (udpa.annotations.versioning).previous_message_type = - "envoy.config.trace.v2.Tracing.Http"; - - reserved 2; - - reserved "config"; - - // The name of the HTTP trace driver to instantiate. The name must match a - // supported HTTP trace driver. Built-in trace drivers: - // - // - *envoy.tracers.lightstep* - // - *envoy.tracers.zipkin* - // - *envoy.tracers.dynamic_ot* - // - *envoy.tracers.datadog* - // - *envoy.tracers.opencensus* - // - *envoy.tracers.xray* - string name = 1 [(validate.rules).string = {min_bytes: 1}]; - - // Trace driver specific configuration which depends on the driver being instantiated. - // See the trace drivers for examples: - // - // - :ref:`LightstepConfig ` - // - :ref:`ZipkinConfig ` - // - :ref:`DynamicOtConfig ` - // - :ref:`DatadogConfig ` - // - :ref:`OpenCensusConfig ` - // - :ref:`AWS X-Ray ` - oneof config_type { - google.protobuf.Any typed_config = 3; - } - } - - // Provides configuration for the HTTP tracer. - Http http = 1; -} - -// Configuration for the LightStep tracer. -// [#extension: envoy.tracers.lightstep] -message LightstepConfig { - option (udpa.annotations.versioning).previous_message_type = - "envoy.config.trace.v2.LightstepConfig"; - - // Available propagation modes - enum PropagationMode { - // Propagate trace context in the single header x-ot-span-context. - ENVOY = 0; - - // Propagate trace context using LightStep's native format. - LIGHTSTEP = 1; - - // Propagate trace context using the b3 format. - B3 = 2; - - // Propagation trace context using the w3 trace-context standard. - TRACE_CONTEXT = 3; - } - - // The cluster manager cluster that hosts the LightStep collectors. - string collector_cluster = 1 [(validate.rules).string = {min_bytes: 1}]; - - // File containing the access token to the `LightStep - // `_ API. - string access_token_file = 2 [(validate.rules).string = {min_bytes: 1}]; - - // Propagation modes to use by LightStep's tracer. - repeated PropagationMode propagation_modes = 3 - [(validate.rules).repeated = {items {enum {defined_only: true}}}]; -} - -// Configuration for the Zipkin tracer. -// [#extension: envoy.tracers.zipkin] -// [#next-free-field: 6] -message ZipkinConfig { - option (udpa.annotations.versioning).previous_message_type = "envoy.config.trace.v2.ZipkinConfig"; - - // Available Zipkin collector endpoint versions. - enum CollectorEndpointVersion { - // Zipkin API v1, JSON over HTTP. - // [#comment: The default implementation of Zipkin client before this field is added was only v1 - // and the way user configure this was by not explicitly specifying the version. Consequently, - // before this is added, the corresponding Zipkin collector expected to receive v1 payload. - // Hence the motivation of adding HTTP_JSON_V1 as the default is to avoid a breaking change when - // user upgrading Envoy with this change. Furthermore, we also immediately deprecate this field, - // since in Zipkin realm this v1 version is considered to be not preferable anymore.] - DEPRECATED_AND_UNAVAILABLE_DO_NOT_USE = 0 - [deprecated = true, (envoy.annotations.disallowed_by_default_enum) = true]; - - // Zipkin API v2, JSON over HTTP. - HTTP_JSON = 1; - - // Zipkin API v2, protobuf over HTTP. - HTTP_PROTO = 2; - - // [#not-implemented-hide:] - GRPC = 3; - } - - // The cluster manager cluster that hosts the Zipkin collectors. Note that the - // Zipkin cluster must be defined in the :ref:`Bootstrap static cluster - // resources `. - string collector_cluster = 1 [(validate.rules).string = {min_bytes: 1}]; - - // The API endpoint of the Zipkin service where the spans will be sent. When - // using a standard Zipkin installation, the API endpoint is typically - // /api/v1/spans, which is the default value. - string collector_endpoint = 2 [(validate.rules).string = {min_bytes: 1}]; - - // Determines whether a 128bit trace id will be used when creating a new - // trace instance. The default value is false, which will result in a 64 bit trace id being used. - bool trace_id_128bit = 3; - - // Determines whether client and server spans will share the same span context. - // The default value is true. - google.protobuf.BoolValue shared_span_context = 4; - - // Determines the selected collector endpoint version. By default, the ``HTTP_JSON_V1`` will be - // used. - CollectorEndpointVersion collector_endpoint_version = 5; -} - -// DynamicOtConfig is used to dynamically load a tracer from a shared library -// that implements the `OpenTracing dynamic loading API -// `_. -// [#extension: envoy.tracers.dynamic_ot] -message DynamicOtConfig { - option (udpa.annotations.versioning).previous_message_type = - "envoy.config.trace.v2.DynamicOtConfig"; - - // Dynamic library implementing the `OpenTracing API - // `_. - string library = 1 [(validate.rules).string = {min_bytes: 1}]; - - // The configuration to use when creating a tracer from the given dynamic - // library. - google.protobuf.Struct config = 2; -} - -// Configuration for the Datadog tracer. -// [#extension: envoy.tracers.datadog] -message DatadogConfig { - option (udpa.annotations.versioning).previous_message_type = - "envoy.config.trace.v2.DatadogConfig"; - - // The cluster to use for submitting traces to the Datadog agent. - string collector_cluster = 1 [(validate.rules).string = {min_bytes: 1}]; - - // The name used for the service when traces are generated by envoy. - string service_name = 2 [(validate.rules).string = {min_bytes: 1}]; -} - -// Configuration for the OpenCensus tracer. -// [#next-free-field: 15] -// [#extension: envoy.tracers.opencensus] -message OpenCensusConfig { - option (udpa.annotations.versioning).previous_message_type = - "envoy.config.trace.v2.OpenCensusConfig"; - - enum TraceContext { - // No-op default, no trace context is utilized. - NONE = 0; - - // W3C Trace-Context format "traceparent:" header. - TRACE_CONTEXT = 1; - - // Binary "grpc-trace-bin:" header. - GRPC_TRACE_BIN = 2; - - // "X-Cloud-Trace-Context:" header. - CLOUD_TRACE_CONTEXT = 3; - - // X-B3-* headers. - B3 = 4; - } - - reserved 7; - - // Configures tracing, e.g. the sampler, max number of annotations, etc. - opencensus.proto.trace.v1.TraceConfig trace_config = 1; - - // Enables the stdout exporter if set to true. This is intended for debugging - // purposes. - bool stdout_exporter_enabled = 2; - - // Enables the Stackdriver exporter if set to true. The project_id must also - // be set. - bool stackdriver_exporter_enabled = 3; - - // The Cloud project_id to use for Stackdriver tracing. - string stackdriver_project_id = 4; - - // (optional) By default, the Stackdriver exporter will connect to production - // Stackdriver. If stackdriver_address is non-empty, it will instead connect - // to this address, which is in the gRPC format: - // https://github.com/grpc/grpc/blob/master/doc/naming.md - string stackdriver_address = 10; - - // (optional) The gRPC server that hosts Stackdriver tracing service. Only - // Google gRPC is supported. If :ref:`target_uri ` - // is not provided, the default production Stackdriver address will be used. - core.v3.GrpcService stackdriver_grpc_service = 13; - - // Enables the Zipkin exporter if set to true. The url and service name must - // also be set. - bool zipkin_exporter_enabled = 5; - - // The URL to Zipkin, e.g. "http://127.0.0.1:9411/api/v2/spans" - string zipkin_url = 6; - - // Enables the OpenCensus Agent exporter if set to true. The ocagent_address or - // ocagent_grpc_service must also be set. - bool ocagent_exporter_enabled = 11; - - // The address of the OpenCensus Agent, if its exporter is enabled, in gRPC - // format: https://github.com/grpc/grpc/blob/master/doc/naming.md - // [#comment:TODO: deprecate this field] - string ocagent_address = 12; - - // (optional) The gRPC server hosted by the OpenCensus Agent. Only Google gRPC is supported. - // This is only used if the ocagent_address is left empty. - core.v3.GrpcService ocagent_grpc_service = 14; - - // List of incoming trace context headers we will accept. First one found - // wins. - repeated TraceContext incoming_trace_context = 8; - - // List of outgoing trace context headers we will produce. - repeated TraceContext outgoing_trace_context = 9; -} - -// Configuration structure. -message TraceServiceConfig { - option (udpa.annotations.versioning).previous_message_type = - "envoy.config.trace.v2.TraceServiceConfig"; - - // The upstream gRPC cluster that hosts the metrics service. - core.v3.GrpcService grpc_service = 1 [(validate.rules).message = {required: true}]; -} diff --git a/api/envoy/config/trace/v3/xray.proto b/api/envoy/config/trace/v3/xray.proto index c4259177d657..ba3fa66ad3e2 100644 --- a/api/envoy/config/trace/v3/xray.proto +++ b/api/envoy/config/trace/v3/xray.proto @@ -5,6 +5,7 @@ package envoy.config.trace.v3; import "envoy/config/core/v3/address.proto"; import "envoy/config/core/v3/base.proto"; +import "udpa/annotations/migrate.proto"; import "udpa/annotations/status.proto"; import "udpa/annotations/versioning.proto"; import "validate/validate.proto"; @@ -12,6 +13,7 @@ import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.config.trace.v3"; option java_outer_classname = "XrayProto"; option java_multiple_files = true; +option (udpa.annotations.file_migrate).move_to_package = "envoy.extensions.tracers.xray.v4alpha"; option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: AWS X-Ray Tracer Configuration] diff --git a/api/envoy/config/trace/v3/zipkin.proto b/api/envoy/config/trace/v3/zipkin.proto new file mode 100644 index 000000000000..5c5349cdf155 --- /dev/null +++ b/api/envoy/config/trace/v3/zipkin.proto @@ -0,0 +1,70 @@ +syntax = "proto3"; + +package envoy.config.trace.v3; + +import "google/protobuf/wrappers.proto"; + +import "envoy/annotations/deprecation.proto"; +import "udpa/annotations/migrate.proto"; +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.trace.v3"; +option java_outer_classname = "ZipkinProto"; +option java_multiple_files = true; +option (udpa.annotations.file_migrate).move_to_package = "envoy.extensions.tracers.zipkin.v4alpha"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Zipkin tracer] + +// Configuration for the Zipkin tracer. +// [#extension: envoy.tracers.zipkin] +// [#next-free-field: 6] +message ZipkinConfig { + option (udpa.annotations.versioning).previous_message_type = "envoy.config.trace.v2.ZipkinConfig"; + + // Available Zipkin collector endpoint versions. + enum CollectorEndpointVersion { + // Zipkin API v1, JSON over HTTP. + // [#comment: The default implementation of Zipkin client before this field is added was only v1 + // and the way user configure this was by not explicitly specifying the version. Consequently, + // before this is added, the corresponding Zipkin collector expected to receive v1 payload. + // Hence the motivation of adding HTTP_JSON_V1 as the default is to avoid a breaking change when + // user upgrading Envoy with this change. Furthermore, we also immediately deprecate this field, + // since in Zipkin realm this v1 version is considered to be not preferable anymore.] + DEPRECATED_AND_UNAVAILABLE_DO_NOT_USE = 0 + [deprecated = true, (envoy.annotations.disallowed_by_default_enum) = true]; + + // Zipkin API v2, JSON over HTTP. + HTTP_JSON = 1; + + // Zipkin API v2, protobuf over HTTP. + HTTP_PROTO = 2; + + // [#not-implemented-hide:] + GRPC = 3; + } + + // The cluster manager cluster that hosts the Zipkin collectors. Note that the + // Zipkin cluster must be defined in the :ref:`Bootstrap static cluster + // resources `. + string collector_cluster = 1 [(validate.rules).string = {min_bytes: 1}]; + + // The API endpoint of the Zipkin service where the spans will be sent. When + // using a standard Zipkin installation, the API endpoint is typically + // /api/v1/spans, which is the default value. + string collector_endpoint = 2 [(validate.rules).string = {min_bytes: 1}]; + + // Determines whether a 128bit trace id will be used when creating a new + // trace instance. The default value is false, which will result in a 64 bit trace id being used. + bool trace_id_128bit = 3; + + // Determines whether client and server spans will share the same span context. + // The default value is true. + google.protobuf.BoolValue shared_span_context = 4; + + // Determines the selected collector endpoint version. By default, the ``HTTP_JSON_V1`` will be + // used. + CollectorEndpointVersion collector_endpoint_version = 5; +} diff --git a/api/envoy/config/trace/v4alpha/BUILD b/api/envoy/config/trace/v4alpha/BUILD index 53ae98aac140..d8ce683c41d6 100644 --- a/api/envoy/config/trace/v4alpha/BUILD +++ b/api/envoy/config/trace/v4alpha/BUILD @@ -6,10 +6,8 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ - "//envoy/annotations:pkg", "//envoy/config/core/v4alpha:pkg", "//envoy/config/trace/v3:pkg", "@com_github_cncf_udpa//udpa/annotations:pkg", - "@opencensus_proto//opencensus/proto/trace/v1:trace_config_proto", ], ) diff --git a/api/envoy/config/trace/v4alpha/http_tracer.proto b/api/envoy/config/trace/v4alpha/http_tracer.proto new file mode 100644 index 000000000000..663886a97bb4 --- /dev/null +++ b/api/envoy/config/trace/v4alpha/http_tracer.proto @@ -0,0 +1,73 @@ +syntax = "proto3"; + +package envoy.config.trace.v4alpha; + +import "google/protobuf/any.proto"; +import "google/protobuf/struct.proto"; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.trace.v4alpha"; +option java_outer_classname = "HttpTracerProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; + +// [#protodoc-title: Tracing] +// Tracing :ref:`architecture overview `. + +// The tracing configuration specifies settings for an HTTP tracer provider used by Envoy. +// +// Envoy may support other tracers in the future, but right now the HTTP tracer is the only one +// supported. +// +// .. attention:: +// +// Use of this message type has been deprecated in favor of direct use of +// :ref:`Tracing.Http `. +message Tracing { + option (udpa.annotations.versioning).previous_message_type = "envoy.config.trace.v3.Tracing"; + + // Configuration for an HTTP tracer provider used by Envoy. + // + // The configuration is defined by the + // :ref:`HttpConnectionManager.Tracing ` + // :ref:`provider ` + // field. + message Http { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.trace.v3.Tracing.Http"; + + reserved 2; + + reserved "config"; + + // The name of the HTTP trace driver to instantiate. The name must match a + // supported HTTP trace driver. Built-in trace drivers: + // + // - *envoy.tracers.lightstep* + // - *envoy.tracers.zipkin* + // - *envoy.tracers.dynamic_ot* + // - *envoy.tracers.datadog* + // - *envoy.tracers.opencensus* + // - *envoy.tracers.xray* + string name = 1 [(validate.rules).string = {min_bytes: 1}]; + + // Trace driver specific configuration which depends on the driver being instantiated. + // See the trace drivers for examples: + // + // - :ref:`LightstepConfig ` + // - :ref:`ZipkinConfig ` + // - :ref:`DynamicOtConfig ` + // - :ref:`DatadogConfig ` + // - :ref:`OpenCensusConfig ` + // - :ref:`AWS X-Ray ` + oneof config_type { + google.protobuf.Any typed_config = 3; + } + } + + // Provides configuration for the HTTP tracer. + Http http = 1; +} diff --git a/api/envoy/config/trace/v4alpha/service.proto b/api/envoy/config/trace/v4alpha/service.proto new file mode 100644 index 000000000000..d132b32dd79d --- /dev/null +++ b/api/envoy/config/trace/v4alpha/service.proto @@ -0,0 +1,25 @@ +syntax = "proto3"; + +package envoy.config.trace.v4alpha; + +import "envoy/config/core/v4alpha/grpc_service.proto"; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.trace.v4alpha"; +option java_outer_classname = "ServiceProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; + +// [#protodoc-title: Trace Service] + +// Configuration structure. +message TraceServiceConfig { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.trace.v3.TraceServiceConfig"; + + // The upstream gRPC cluster that hosts the metrics service. + core.v4alpha.GrpcService grpc_service = 1 [(validate.rules).message = {required: true}]; +} diff --git a/api/envoy/config/trace/v4alpha/trace.proto b/api/envoy/config/trace/v4alpha/trace.proto deleted file mode 100644 index f43c8df24b33..000000000000 --- a/api/envoy/config/trace/v4alpha/trace.proto +++ /dev/null @@ -1,281 +0,0 @@ -syntax = "proto3"; - -package envoy.config.trace.v4alpha; - -import "envoy/config/core/v4alpha/grpc_service.proto"; - -import "google/protobuf/any.proto"; -import "google/protobuf/struct.proto"; -import "google/protobuf/wrappers.proto"; - -import "opencensus/proto/trace/v1/trace_config.proto"; - -import "envoy/annotations/deprecation.proto"; -import "udpa/annotations/status.proto"; -import "udpa/annotations/versioning.proto"; -import "validate/validate.proto"; - -option java_package = "io.envoyproxy.envoy.config.trace.v4alpha"; -option java_outer_classname = "TraceProto"; -option java_multiple_files = true; -option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; - -// [#protodoc-title: Tracing] -// Tracing :ref:`architecture overview `. - -// The tracing configuration specifies settings for an HTTP tracer provider used by Envoy. -// -// Envoy may support other tracers in the future, but right now the HTTP tracer is the only one -// supported. -// -// .. attention:: -// -// Use of this message type has been deprecated in favor of direct use of -// :ref:`Tracing.Http `. -message Tracing { - option (udpa.annotations.versioning).previous_message_type = "envoy.config.trace.v3.Tracing"; - - // Configuration for an HTTP tracer provider used by Envoy. - // - // The configuration is defined by the - // :ref:`HttpConnectionManager.Tracing ` - // :ref:`provider ` - // field. - message Http { - option (udpa.annotations.versioning).previous_message_type = - "envoy.config.trace.v3.Tracing.Http"; - - reserved 2; - - reserved "config"; - - // The name of the HTTP trace driver to instantiate. The name must match a - // supported HTTP trace driver. Built-in trace drivers: - // - // - *envoy.tracers.lightstep* - // - *envoy.tracers.zipkin* - // - *envoy.tracers.dynamic_ot* - // - *envoy.tracers.datadog* - // - *envoy.tracers.opencensus* - // - *envoy.tracers.xray* - string name = 1 [(validate.rules).string = {min_bytes: 1}]; - - // Trace driver specific configuration which depends on the driver being instantiated. - // See the trace drivers for examples: - // - // - :ref:`LightstepConfig ` - // - :ref:`ZipkinConfig ` - // - :ref:`DynamicOtConfig ` - // - :ref:`DatadogConfig ` - // - :ref:`OpenCensusConfig ` - // - :ref:`AWS X-Ray ` - oneof config_type { - google.protobuf.Any typed_config = 3; - } - } - - // Provides configuration for the HTTP tracer. - Http http = 1; -} - -// Configuration for the LightStep tracer. -// [#extension: envoy.tracers.lightstep] -message LightstepConfig { - option (udpa.annotations.versioning).previous_message_type = - "envoy.config.trace.v3.LightstepConfig"; - - // Available propagation modes - enum PropagationMode { - // Propagate trace context in the single header x-ot-span-context. - ENVOY = 0; - - // Propagate trace context using LightStep's native format. - LIGHTSTEP = 1; - - // Propagate trace context using the b3 format. - B3 = 2; - - // Propagation trace context using the w3 trace-context standard. - TRACE_CONTEXT = 3; - } - - // The cluster manager cluster that hosts the LightStep collectors. - string collector_cluster = 1 [(validate.rules).string = {min_bytes: 1}]; - - // File containing the access token to the `LightStep - // `_ API. - string access_token_file = 2 [(validate.rules).string = {min_bytes: 1}]; - - // Propagation modes to use by LightStep's tracer. - repeated PropagationMode propagation_modes = 3 - [(validate.rules).repeated = {items {enum {defined_only: true}}}]; -} - -// Configuration for the Zipkin tracer. -// [#extension: envoy.tracers.zipkin] -// [#next-free-field: 6] -message ZipkinConfig { - option (udpa.annotations.versioning).previous_message_type = "envoy.config.trace.v3.ZipkinConfig"; - - // Available Zipkin collector endpoint versions. - enum CollectorEndpointVersion { - // Zipkin API v1, JSON over HTTP. - // [#comment: The default implementation of Zipkin client before this field is added was only v1 - // and the way user configure this was by not explicitly specifying the version. Consequently, - // before this is added, the corresponding Zipkin collector expected to receive v1 payload. - // Hence the motivation of adding HTTP_JSON_V1 as the default is to avoid a breaking change when - // user upgrading Envoy with this change. Furthermore, we also immediately deprecate this field, - // since in Zipkin realm this v1 version is considered to be not preferable anymore.] - DEPRECATED_AND_UNAVAILABLE_DO_NOT_USE = 0 - [deprecated = true, (envoy.annotations.disallowed_by_default_enum) = true]; - - // Zipkin API v2, JSON over HTTP. - HTTP_JSON = 1; - - // Zipkin API v2, protobuf over HTTP. - HTTP_PROTO = 2; - - // [#not-implemented-hide:] - GRPC = 3; - } - - // The cluster manager cluster that hosts the Zipkin collectors. Note that the - // Zipkin cluster must be defined in the :ref:`Bootstrap static cluster - // resources `. - string collector_cluster = 1 [(validate.rules).string = {min_bytes: 1}]; - - // The API endpoint of the Zipkin service where the spans will be sent. When - // using a standard Zipkin installation, the API endpoint is typically - // /api/v1/spans, which is the default value. - string collector_endpoint = 2 [(validate.rules).string = {min_bytes: 1}]; - - // Determines whether a 128bit trace id will be used when creating a new - // trace instance. The default value is false, which will result in a 64 bit trace id being used. - bool trace_id_128bit = 3; - - // Determines whether client and server spans will share the same span context. - // The default value is true. - google.protobuf.BoolValue shared_span_context = 4; - - // Determines the selected collector endpoint version. By default, the ``HTTP_JSON_V1`` will be - // used. - CollectorEndpointVersion collector_endpoint_version = 5; -} - -// DynamicOtConfig is used to dynamically load a tracer from a shared library -// that implements the `OpenTracing dynamic loading API -// `_. -// [#extension: envoy.tracers.dynamic_ot] -message DynamicOtConfig { - option (udpa.annotations.versioning).previous_message_type = - "envoy.config.trace.v3.DynamicOtConfig"; - - // Dynamic library implementing the `OpenTracing API - // `_. - string library = 1 [(validate.rules).string = {min_bytes: 1}]; - - // The configuration to use when creating a tracer from the given dynamic - // library. - google.protobuf.Struct config = 2; -} - -// Configuration for the Datadog tracer. -// [#extension: envoy.tracers.datadog] -message DatadogConfig { - option (udpa.annotations.versioning).previous_message_type = - "envoy.config.trace.v3.DatadogConfig"; - - // The cluster to use for submitting traces to the Datadog agent. - string collector_cluster = 1 [(validate.rules).string = {min_bytes: 1}]; - - // The name used for the service when traces are generated by envoy. - string service_name = 2 [(validate.rules).string = {min_bytes: 1}]; -} - -// Configuration for the OpenCensus tracer. -// [#next-free-field: 15] -// [#extension: envoy.tracers.opencensus] -message OpenCensusConfig { - option (udpa.annotations.versioning).previous_message_type = - "envoy.config.trace.v3.OpenCensusConfig"; - - enum TraceContext { - // No-op default, no trace context is utilized. - NONE = 0; - - // W3C Trace-Context format "traceparent:" header. - TRACE_CONTEXT = 1; - - // Binary "grpc-trace-bin:" header. - GRPC_TRACE_BIN = 2; - - // "X-Cloud-Trace-Context:" header. - CLOUD_TRACE_CONTEXT = 3; - - // X-B3-* headers. - B3 = 4; - } - - reserved 7; - - // Configures tracing, e.g. the sampler, max number of annotations, etc. - opencensus.proto.trace.v1.TraceConfig trace_config = 1; - - // Enables the stdout exporter if set to true. This is intended for debugging - // purposes. - bool stdout_exporter_enabled = 2; - - // Enables the Stackdriver exporter if set to true. The project_id must also - // be set. - bool stackdriver_exporter_enabled = 3; - - // The Cloud project_id to use for Stackdriver tracing. - string stackdriver_project_id = 4; - - // (optional) By default, the Stackdriver exporter will connect to production - // Stackdriver. If stackdriver_address is non-empty, it will instead connect - // to this address, which is in the gRPC format: - // https://github.com/grpc/grpc/blob/master/doc/naming.md - string stackdriver_address = 10; - - // (optional) The gRPC server that hosts Stackdriver tracing service. Only - // Google gRPC is supported. If :ref:`target_uri ` - // is not provided, the default production Stackdriver address will be used. - core.v4alpha.GrpcService stackdriver_grpc_service = 13; - - // Enables the Zipkin exporter if set to true. The url and service name must - // also be set. - bool zipkin_exporter_enabled = 5; - - // The URL to Zipkin, e.g. "http://127.0.0.1:9411/api/v2/spans" - string zipkin_url = 6; - - // Enables the OpenCensus Agent exporter if set to true. The ocagent_address or - // ocagent_grpc_service must also be set. - bool ocagent_exporter_enabled = 11; - - // The address of the OpenCensus Agent, if its exporter is enabled, in gRPC - // format: https://github.com/grpc/grpc/blob/master/doc/naming.md - // [#comment:TODO: deprecate this field] - string ocagent_address = 12; - - // (optional) The gRPC server hosted by the OpenCensus Agent. Only Google gRPC is supported. - // This is only used if the ocagent_address is left empty. - core.v4alpha.GrpcService ocagent_grpc_service = 14; - - // List of incoming trace context headers we will accept. First one found - // wins. - repeated TraceContext incoming_trace_context = 8; - - // List of outgoing trace context headers we will produce. - repeated TraceContext outgoing_trace_context = 9; -} - -// Configuration structure. -message TraceServiceConfig { - option (udpa.annotations.versioning).previous_message_type = - "envoy.config.trace.v3.TraceServiceConfig"; - - // The upstream gRPC cluster that hosts the metrics service. - core.v4alpha.GrpcService grpc_service = 1 [(validate.rules).message = {required: true}]; -} diff --git a/api/envoy/extensions/filters/http/grpc_stats/v3/config.proto b/api/envoy/extensions/filters/http/grpc_stats/v3/config.proto index 1fecdaea0a16..ff56066410cb 100644 --- a/api/envoy/extensions/filters/http/grpc_stats/v3/config.proto +++ b/api/envoy/extensions/filters/http/grpc_stats/v3/config.proto @@ -54,6 +54,14 @@ message FilterConfig { // `envoy.deprecated_features.grpc_stats_filter_enable_stats_for_all_methods_by_default`. google.protobuf.BoolValue stats_for_all_methods = 3; } + + // If true, the filter will gather a histogram for the request time of the upstream. + // It works with :ref:`stats_for_all_methods + // ` + // and :ref:`individual_method_stats_allowlist + // ` the same way + // request_message_count and response_message_count works. + bool enable_upstream_stats = 4; } // gRPC statistics filter state object in protobuf form. diff --git a/api/envoy/extensions/filters/http/jwt_authn/v3/config.proto b/api/envoy/extensions/filters/http/jwt_authn/v3/config.proto index 1aabe1bd4390..39fe6187f64f 100644 --- a/api/envoy/extensions/filters/http/jwt_authn/v3/config.proto +++ b/api/envoy/extensions/filters/http/jwt_authn/v3/config.proto @@ -171,7 +171,8 @@ message JwtProvider { // base64url_encoded(jwt_payload_in_JSON) // // If it is not specified, the payload will not be forwarded. - string forward_payload_header = 8; + string forward_payload_header = 8 + [(validate.rules).string = {well_known_regex: HTTP_HEADER_NAME strict: false}]; // If non empty, successfully verified JWT payloads will be written to StreamInfo DynamicMetadata // in the format as: *namespace* is the jwt_authn filter name as **envoy.filters.http.jwt_authn** @@ -218,12 +219,14 @@ message JwtHeader { "envoy.config.filter.http.jwt_authn.v2alpha.JwtHeader"; // The HTTP header name. - string name = 1 [(validate.rules).string = {min_bytes: 1}]; + string name = 1 + [(validate.rules).string = {min_bytes: 1 well_known_regex: HTTP_HEADER_NAME strict: false}]; // The value prefix. The value format is "value_prefix" // For example, for "Authorization: Bearer ", value_prefix="Bearer " with a space at the // end. - string value_prefix = 2; + string value_prefix = 2 + [(validate.rules).string = {well_known_regex: HTTP_HEADER_VALUE strict: false}]; } // Specify a required provider with audiences. diff --git a/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto b/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto index 1fd2e82eed5f..d802ec4ce774 100644 --- a/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto +++ b/api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto @@ -7,7 +7,7 @@ import "envoy/config/core/v3/config_source.proto"; import "envoy/config/core/v3/protocol.proto"; import "envoy/config/route/v3/route.proto"; import "envoy/config/route/v3/scoped_route.proto"; -import "envoy/config/trace/v3/trace.proto"; +import "envoy/config/trace/v3/http_tracer.proto"; import "envoy/type/tracing/v3/custom_tag.proto"; import "envoy/type/v3/percent.proto"; diff --git a/api/envoy/extensions/filters/network/http_connection_manager/v4alpha/http_connection_manager.proto b/api/envoy/extensions/filters/network/http_connection_manager/v4alpha/http_connection_manager.proto index 5a05a8531f35..975b71cc892f 100644 --- a/api/envoy/extensions/filters/network/http_connection_manager/v4alpha/http_connection_manager.proto +++ b/api/envoy/extensions/filters/network/http_connection_manager/v4alpha/http_connection_manager.proto @@ -7,7 +7,7 @@ import "envoy/config/core/v4alpha/config_source.proto"; import "envoy/config/core/v4alpha/protocol.proto"; import "envoy/config/route/v4alpha/route.proto"; import "envoy/config/route/v4alpha/scoped_route.proto"; -import "envoy/config/trace/v4alpha/trace.proto"; +import "envoy/config/trace/v4alpha/http_tracer.proto"; import "envoy/type/tracing/v3/custom_tag.proto"; import "envoy/type/v3/percent.proto"; diff --git a/api/envoy/extensions/filters/network/sni_dynamic_forward_proxy/v3alpha/BUILD b/api/envoy/extensions/filters/network/sni_dynamic_forward_proxy/v3alpha/BUILD new file mode 100644 index 000000000000..1dcb37e5c342 --- /dev/null +++ b/api/envoy/extensions/filters/network/sni_dynamic_forward_proxy/v3alpha/BUILD @@ -0,0 +1,12 @@ +# DO NOT EDIT. This file is generated by tools/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/extensions/common/dynamic_forward_proxy/v3:pkg", + "@com_github_cncf_udpa//udpa/annotations:pkg", + ], +) diff --git a/api/envoy/extensions/filters/network/sni_dynamic_forward_proxy/v3alpha/sni_dynamic_forward_proxy.proto b/api/envoy/extensions/filters/network/sni_dynamic_forward_proxy/v3alpha/sni_dynamic_forward_proxy.proto new file mode 100644 index 000000000000..502a66893174 --- /dev/null +++ b/api/envoy/extensions/filters/network/sni_dynamic_forward_proxy/v3alpha/sni_dynamic_forward_proxy.proto @@ -0,0 +1,37 @@ +syntax = "proto3"; + +package envoy.extensions.filters.network.sni_dynamic_forward_proxy.v3alpha; + +import "envoy/extensions/common/dynamic_forward_proxy/v3/dns_cache.proto"; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.filters.network.sni_dynamic_forward_proxy.v3alpha"; +option java_outer_classname = "SniDynamicForwardProxyProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).work_in_progress = true; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: SNI dynamic forward proxy] + +// Configuration for the SNI-based dynamic forward proxy filter. See the +// :ref:`architecture overview ` for +// more information. Note this filter must be configured along with +// :ref:`TLS inspector listener filter ` +// to work. +// [#extension: envoy.filters.network.sni_dynamic_forward_proxy] +message FilterConfig { + // The DNS cache configuration that the filter will attach to. Note this + // configuration must match that of associated :ref:`dynamic forward proxy + // cluster configuration + // `. + common.dynamic_forward_proxy.v3.DnsCacheConfig dns_cache_config = 1 + [(validate.rules).message = {required: true}]; + + oneof port_specifier { + // The port number to connect to the upstream. + uint32 port_value = 2 [(validate.rules).uint32 = {lte: 65535 gt: 0}]; + } +} diff --git a/api/envoy/extensions/tracers/datadog/v4alpha/BUILD b/api/envoy/extensions/tracers/datadog/v4alpha/BUILD new file mode 100644 index 000000000000..d58bd8a0fca4 --- /dev/null +++ b/api/envoy/extensions/tracers/datadog/v4alpha/BUILD @@ -0,0 +1,12 @@ +# DO NOT EDIT. This file is generated by tools/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/config/trace/v3:pkg", + "@com_github_cncf_udpa//udpa/annotations:pkg", + ], +) diff --git a/api/envoy/extensions/tracers/datadog/v4alpha/datadog.proto b/api/envoy/extensions/tracers/datadog/v4alpha/datadog.proto new file mode 100644 index 000000000000..94359ce837bf --- /dev/null +++ b/api/envoy/extensions/tracers/datadog/v4alpha/datadog.proto @@ -0,0 +1,27 @@ +syntax = "proto3"; + +package envoy.extensions.tracers.datadog.v4alpha; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.tracers.datadog.v4alpha"; +option java_outer_classname = "DatadogProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; + +// [#protodoc-title: Datadog tracer] + +// Configuration for the Datadog tracer. +// [#extension: envoy.tracers.datadog] +message DatadogConfig { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.trace.v3.DatadogConfig"; + + // The cluster to use for submitting traces to the Datadog agent. + string collector_cluster = 1 [(validate.rules).string = {min_bytes: 1}]; + + // The name used for the service when traces are generated by envoy. + string service_name = 2 [(validate.rules).string = {min_bytes: 1}]; +} diff --git a/api/envoy/extensions/tracers/dynamic_ot/v4alpha/BUILD b/api/envoy/extensions/tracers/dynamic_ot/v4alpha/BUILD new file mode 100644 index 000000000000..d58bd8a0fca4 --- /dev/null +++ b/api/envoy/extensions/tracers/dynamic_ot/v4alpha/BUILD @@ -0,0 +1,12 @@ +# DO NOT EDIT. This file is generated by tools/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/config/trace/v3:pkg", + "@com_github_cncf_udpa//udpa/annotations:pkg", + ], +) diff --git a/api/envoy/extensions/tracers/dynamic_ot/v4alpha/dynamic_ot.proto b/api/envoy/extensions/tracers/dynamic_ot/v4alpha/dynamic_ot.proto new file mode 100644 index 000000000000..d311304a3ddf --- /dev/null +++ b/api/envoy/extensions/tracers/dynamic_ot/v4alpha/dynamic_ot.proto @@ -0,0 +1,33 @@ +syntax = "proto3"; + +package envoy.extensions.tracers.dynamic_ot.v4alpha; + +import "google/protobuf/struct.proto"; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.tracers.dynamic_ot.v4alpha"; +option java_outer_classname = "DynamicOtProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; + +// [#protodoc-title: Dynamically loadable OpenTracing tracer] + +// DynamicOtConfig is used to dynamically load a tracer from a shared library +// that implements the `OpenTracing dynamic loading API +// `_. +// [#extension: envoy.tracers.dynamic_ot] +message DynamicOtConfig { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.trace.v3.DynamicOtConfig"; + + // Dynamic library implementing the `OpenTracing API + // `_. + string library = 1 [(validate.rules).string = {min_bytes: 1}]; + + // The configuration to use when creating a tracer from the given dynamic + // library. + google.protobuf.Struct config = 2; +} diff --git a/api/envoy/extensions/tracers/lightstep/v4alpha/BUILD b/api/envoy/extensions/tracers/lightstep/v4alpha/BUILD new file mode 100644 index 000000000000..d58bd8a0fca4 --- /dev/null +++ b/api/envoy/extensions/tracers/lightstep/v4alpha/BUILD @@ -0,0 +1,12 @@ +# DO NOT EDIT. This file is generated by tools/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/config/trace/v3:pkg", + "@com_github_cncf_udpa//udpa/annotations:pkg", + ], +) diff --git a/api/envoy/extensions/tracers/lightstep/v4alpha/lightstep.proto b/api/envoy/extensions/tracers/lightstep/v4alpha/lightstep.proto new file mode 100644 index 000000000000..93ea47ba6a10 --- /dev/null +++ b/api/envoy/extensions/tracers/lightstep/v4alpha/lightstep.proto @@ -0,0 +1,47 @@ +syntax = "proto3"; + +package envoy.extensions.tracers.lightstep.v4alpha; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.tracers.lightstep.v4alpha"; +option java_outer_classname = "LightstepProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; + +// [#protodoc-title: LightStep tracer] + +// Configuration for the LightStep tracer. +// [#extension: envoy.tracers.lightstep] +message LightstepConfig { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.trace.v3.LightstepConfig"; + + // Available propagation modes + enum PropagationMode { + // Propagate trace context in the single header x-ot-span-context. + ENVOY = 0; + + // Propagate trace context using LightStep's native format. + LIGHTSTEP = 1; + + // Propagate trace context using the b3 format. + B3 = 2; + + // Propagation trace context using the w3 trace-context standard. + TRACE_CONTEXT = 3; + } + + // The cluster manager cluster that hosts the LightStep collectors. + string collector_cluster = 1 [(validate.rules).string = {min_bytes: 1}]; + + // File containing the access token to the `LightStep + // `_ API. + string access_token_file = 2 [(validate.rules).string = {min_bytes: 1}]; + + // Propagation modes to use by LightStep's tracer. + repeated PropagationMode propagation_modes = 3 + [(validate.rules).repeated = {items {enum {defined_only: true}}}]; +} diff --git a/api/envoy/extensions/tracers/opencensus/v4alpha/BUILD b/api/envoy/extensions/tracers/opencensus/v4alpha/BUILD new file mode 100644 index 000000000000..05e29893b9cf --- /dev/null +++ b/api/envoy/extensions/tracers/opencensus/v4alpha/BUILD @@ -0,0 +1,14 @@ +# DO NOT EDIT. This file is generated by tools/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/config/core/v4alpha:pkg", + "//envoy/config/trace/v3:pkg", + "@com_github_cncf_udpa//udpa/annotations:pkg", + "@opencensus_proto//opencensus/proto/trace/v1:trace_config_proto", + ], +) diff --git a/api/envoy/extensions/tracers/opencensus/v4alpha/opencensus.proto b/api/envoy/extensions/tracers/opencensus/v4alpha/opencensus.proto new file mode 100644 index 000000000000..f64507b13827 --- /dev/null +++ b/api/envoy/extensions/tracers/opencensus/v4alpha/opencensus.proto @@ -0,0 +1,97 @@ +syntax = "proto3"; + +package envoy.extensions.tracers.opencensus.v4alpha; + +import "envoy/config/core/v4alpha/grpc_service.proto"; + +import "opencensus/proto/trace/v1/trace_config.proto"; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.tracers.opencensus.v4alpha"; +option java_outer_classname = "OpencensusProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; + +// [#protodoc-title: OpenCensus tracer] + +// Configuration for the OpenCensus tracer. +// [#next-free-field: 15] +// [#extension: envoy.tracers.opencensus] +message OpenCensusConfig { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.trace.v3.OpenCensusConfig"; + + enum TraceContext { + // No-op default, no trace context is utilized. + NONE = 0; + + // W3C Trace-Context format "traceparent:" header. + TRACE_CONTEXT = 1; + + // Binary "grpc-trace-bin:" header. + GRPC_TRACE_BIN = 2; + + // "X-Cloud-Trace-Context:" header. + CLOUD_TRACE_CONTEXT = 3; + + // X-B3-* headers. + B3 = 4; + } + + reserved 7; + + // Configures tracing, e.g. the sampler, max number of annotations, etc. + .opencensus.proto.trace.v1.TraceConfig trace_config = 1; + + // Enables the stdout exporter if set to true. This is intended for debugging + // purposes. + bool stdout_exporter_enabled = 2; + + // Enables the Stackdriver exporter if set to true. The project_id must also + // be set. + bool stackdriver_exporter_enabled = 3; + + // The Cloud project_id to use for Stackdriver tracing. + string stackdriver_project_id = 4; + + // (optional) By default, the Stackdriver exporter will connect to production + // Stackdriver. If stackdriver_address is non-empty, it will instead connect + // to this address, which is in the gRPC format: + // https://github.com/grpc/grpc/blob/master/doc/naming.md + string stackdriver_address = 10; + + // (optional) The gRPC server that hosts Stackdriver tracing service. Only + // Google gRPC is supported. If :ref:`target_uri ` + // is not provided, the default production Stackdriver address will be used. + config.core.v4alpha.GrpcService stackdriver_grpc_service = 13; + + // Enables the Zipkin exporter if set to true. The url and service name must + // also be set. + bool zipkin_exporter_enabled = 5; + + // The URL to Zipkin, e.g. "http://127.0.0.1:9411/api/v2/spans" + string zipkin_url = 6; + + // Enables the OpenCensus Agent exporter if set to true. The ocagent_address or + // ocagent_grpc_service must also be set. + bool ocagent_exporter_enabled = 11; + + // The address of the OpenCensus Agent, if its exporter is enabled, in gRPC + // format: https://github.com/grpc/grpc/blob/master/doc/naming.md + // [#comment:TODO: deprecate this field] + string ocagent_address = 12; + + // (optional) The gRPC server hosted by the OpenCensus Agent. Only Google gRPC is supported. + // This is only used if the ocagent_address is left empty. + config.core.v4alpha.GrpcService ocagent_grpc_service = 14; + + // List of incoming trace context headers we will accept. First one found + // wins. + repeated TraceContext incoming_trace_context = 8; + + // List of outgoing trace context headers we will produce. + repeated TraceContext outgoing_trace_context = 9; +} diff --git a/api/envoy/extensions/tracers/xray/v4alpha/BUILD b/api/envoy/extensions/tracers/xray/v4alpha/BUILD new file mode 100644 index 000000000000..d8ce683c41d6 --- /dev/null +++ b/api/envoy/extensions/tracers/xray/v4alpha/BUILD @@ -0,0 +1,13 @@ +# DO NOT EDIT. This file is generated by tools/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/config/core/v4alpha:pkg", + "//envoy/config/trace/v3:pkg", + "@com_github_cncf_udpa//udpa/annotations:pkg", + ], +) diff --git a/generated_api_shadow/envoy/config/trace/v4alpha/xray.proto b/api/envoy/extensions/tracers/xray/v4alpha/xray.proto similarity index 82% rename from generated_api_shadow/envoy/config/trace/v4alpha/xray.proto rename to api/envoy/extensions/tracers/xray/v4alpha/xray.proto index 39bcebd1bad7..27a9b5407dda 100644 --- a/generated_api_shadow/envoy/config/trace/v4alpha/xray.proto +++ b/api/envoy/extensions/tracers/xray/v4alpha/xray.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package envoy.config.trace.v4alpha; +package envoy.extensions.tracers.xray.v4alpha; import "envoy/config/core/v4alpha/address.proto"; import "envoy/config/core/v4alpha/base.proto"; @@ -9,7 +9,7 @@ import "udpa/annotations/status.proto"; import "udpa/annotations/versioning.proto"; import "validate/validate.proto"; -option java_package = "io.envoyproxy.envoy.config.trace.v4alpha"; +option java_package = "io.envoyproxy.envoy.extensions.tracers.xray.v4alpha"; option java_outer_classname = "XrayProto"; option java_multiple_files = true; option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; @@ -22,7 +22,7 @@ message XRayConfig { // The UDP endpoint of the X-Ray Daemon where the spans will be sent. // If this value is not set, the default value of 127.0.0.1:2000 will be used. - core.v4alpha.SocketAddress daemon_endpoint = 1; + config.core.v4alpha.SocketAddress daemon_endpoint = 1; // The name of the X-Ray segment. string segment_name = 2 [(validate.rules).string = {min_len: 1}]; @@ -31,5 +31,5 @@ message XRayConfig { // For an example of the sampling rules see: // `X-Ray SDK documentation // `_ - core.v4alpha.DataSource sampling_rule_manifest = 3; + config.core.v4alpha.DataSource sampling_rule_manifest = 3; } diff --git a/api/envoy/extensions/tracers/zipkin/v4alpha/BUILD b/api/envoy/extensions/tracers/zipkin/v4alpha/BUILD new file mode 100644 index 000000000000..8e8f00da8af0 --- /dev/null +++ b/api/envoy/extensions/tracers/zipkin/v4alpha/BUILD @@ -0,0 +1,13 @@ +# DO NOT EDIT. This file is generated by tools/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/annotations:pkg", + "//envoy/config/trace/v3:pkg", + "@com_github_cncf_udpa//udpa/annotations:pkg", + ], +) diff --git a/api/envoy/extensions/tracers/zipkin/v4alpha/zipkin.proto b/api/envoy/extensions/tracers/zipkin/v4alpha/zipkin.proto new file mode 100644 index 000000000000..3abbcad2de15 --- /dev/null +++ b/api/envoy/extensions/tracers/zipkin/v4alpha/zipkin.proto @@ -0,0 +1,68 @@ +syntax = "proto3"; + +package envoy.extensions.tracers.zipkin.v4alpha; + +import "google/protobuf/wrappers.proto"; + +import "envoy/annotations/deprecation.proto"; +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.tracers.zipkin.v4alpha"; +option java_outer_classname = "ZipkinProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; + +// [#protodoc-title: Zipkin tracer] + +// Configuration for the Zipkin tracer. +// [#extension: envoy.tracers.zipkin] +// [#next-free-field: 6] +message ZipkinConfig { + option (udpa.annotations.versioning).previous_message_type = "envoy.config.trace.v3.ZipkinConfig"; + + // Available Zipkin collector endpoint versions. + enum CollectorEndpointVersion { + // Zipkin API v1, JSON over HTTP. + // [#comment: The default implementation of Zipkin client before this field is added was only v1 + // and the way user configure this was by not explicitly specifying the version. Consequently, + // before this is added, the corresponding Zipkin collector expected to receive v1 payload. + // Hence the motivation of adding HTTP_JSON_V1 as the default is to avoid a breaking change when + // user upgrading Envoy with this change. Furthermore, we also immediately deprecate this field, + // since in Zipkin realm this v1 version is considered to be not preferable anymore.] + DEPRECATED_AND_UNAVAILABLE_DO_NOT_USE = 0 + [deprecated = true, (envoy.annotations.disallowed_by_default_enum) = true]; + + // Zipkin API v2, JSON over HTTP. + HTTP_JSON = 1; + + // Zipkin API v2, protobuf over HTTP. + HTTP_PROTO = 2; + + // [#not-implemented-hide:] + GRPC = 3; + } + + // The cluster manager cluster that hosts the Zipkin collectors. Note that the + // Zipkin cluster must be defined in the :ref:`Bootstrap static cluster + // resources `. + string collector_cluster = 1 [(validate.rules).string = {min_bytes: 1}]; + + // The API endpoint of the Zipkin service where the spans will be sent. When + // using a standard Zipkin installation, the API endpoint is typically + // /api/v1/spans, which is the default value. + string collector_endpoint = 2 [(validate.rules).string = {min_bytes: 1}]; + + // Determines whether a 128bit trace id will be used when creating a new + // trace instance. The default value is false, which will result in a 64 bit trace id being used. + bool trace_id_128bit = 3; + + // Determines whether client and server spans will share the same span context. + // The default value is true. + google.protobuf.BoolValue shared_span_context = 4; + + // Determines the selected collector endpoint version. By default, the ``HTTP_JSON_V1`` will be + // used. + CollectorEndpointVersion collector_endpoint_version = 5; +} diff --git a/api/versioning/BUILD b/api/versioning/BUILD index 086cf1201a8a..0ffaf85a1cdd 100644 --- a/api/versioning/BUILD +++ b/api/versioning/BUILD @@ -97,6 +97,7 @@ proto_library( "//envoy/extensions/filters/network/rbac/v3:pkg", "//envoy/extensions/filters/network/redis_proxy/v3:pkg", "//envoy/extensions/filters/network/sni_cluster/v3:pkg", + "//envoy/extensions/filters/network/sni_dynamic_forward_proxy/v3alpha:pkg", "//envoy/extensions/filters/network/tcp_proxy/v3:pkg", "//envoy/extensions/filters/network/thrift_proxy/filters/ratelimit/v3:pkg", "//envoy/extensions/filters/network/thrift_proxy/v3:pkg", diff --git a/bazel/README.md b/bazel/README.md index 4f1512e9e171..60aa5c9eb915 100644 --- a/bazel/README.md +++ b/bazel/README.md @@ -6,15 +6,18 @@ It is recommended to use [Bazelisk](https://github.com/bazelbuild/bazelisk) inst On Linux, run the following commands: ``` -sudo wget -O /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/v0.0.8/bazelisk-linux-amd64 +sudo wget -O /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/latest/download/bazelisk-linux-amd64 sudo chmod +x /usr/local/bin/bazel ``` -On macOS, run the follwing command: +On macOS, run the following command: ``` brew install bazelbuild/tap/bazelisk ``` +On Windows, download [bazelisk-windows-amd64.exe](https://github.com/bazelbuild/bazelisk/releases/latest/download/bazelisk-windows-amd64.exe) +and save this binary in a directory on the PATH as `bazel.exe`. + If you're building from an revision of Envoy prior to August 2019, which doesn't contains a `.bazelversion` file, run `ci/run_envoy_docker.sh "bazel version"` to find the right version of Bazel and set the version to `USE_BAZEL_VERSION` environment variable to build. @@ -94,6 +97,37 @@ for how to update or override dependencies. version of `ar` on the PATH, so if you run into issues building third party code like luajit consider uninstalling binutils. + On Windows, additional dependencies are required: + + Install the [MSYS2 shell](https://msys2.github.io/) and install the `diffutils`, `patch`, + `unzip`, and `zip` packages using `pacman`. Set the `BAZEL_SH` environment variable to the path + of the installed MSYS2 `bash.exe` executable. Setting the `MSYS2_ARG_CONV_EXCL` environment + variable to a value of `*` is often advisable to ensure argument parsing in the MSYS2 shell + behaves as expected. + + `Git` is required. The version installable via MSYS2 is sufficient. + + Install the Windows-native [python3](https://www.python.org/downloads/), the POSIX flavor + available via MSYS2 will not work. + + For building with MSVC (the `msvc-cl` config option), you must install at least the VC++ + workload from the + [Build Tools for Visual Studio 2019](https://visualstudio.microsoft.com/downloads/#build-tools-for-visual-studio-2019). + You may also download Visual Studio 2019 and use the Build Tools packaged with that + installation. Earlier versions of VC++ Build Tools/Visual Studio are not recommended at this + time. If installed in a non-standard filesystem location, be sure to set the `BAZEL_VC` + environment variable to the path of the VC++ package to allow Bazel to find your installation + of VC++. Use caution to ensure the `link.exe` that resolves on your PATH is from VC++ Build Tools and + not MSYS2. + + Ensure `CMake` and `ninja` binaries are on the PATH. The versions packaged with VC++ Build + Tools are sufficient. + + In addition, because of the behavior of the `rules_foreign_cc` component of Bazel, set the + `TMPDIR` environment variable to a path usable as a temporary directory (e.g. + `C:\Windows\TEMP`). This variable is used frequently by `mktemp` from MSYS2 in the Envoy Bazel + build and can cause problems if not set to a value outside the MSYS2 filesystem. + 1. Install Golang on your machine. This is required as part of building [BoringSSL](https://boringssl.googlesource.com/boringssl/+/HEAD/BUILDING.md) and also for [Buildifer](https://github.com/bazelbuild/buildtools) which is used for formatting bazel BUILD files. 1. `go get -u github.com/bazelbuild/buildtools/buildifier` to install buildifier. You may need to set `BUILDIFIER_BIN` to `$GOPATH/bin/buildifier` @@ -652,7 +686,7 @@ that the Go toolchain can find the necessary dependencies): go run github.com/buchgr/bazel-remote --dir ${HOME}/bazel_cache --host 127.0.0.1 --port 28080 --max_size 64 ``` -See [Bazel remote cache](github.com/buchgr/bazel-remote) for more information on the parameters. +See [Bazel remote cache](https://github.com/buchgr/bazel-remote) for more information on the parameters. The command above will setup a maximum 64 GiB cache at `~/bazel_cache` on port 28080. You might want to setup a larger cache if you run ASAN builds. diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index d89b139ee5f8..58a5c8878608 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -9,11 +9,11 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.19.1/bazel-gazelle-v0.19.1.tar.gz"], ), bazel_toolchains = dict( - sha256 = "1342f84d4324987f63307eb6a5aac2dff6d27967860a129f5cd40f8f9b6fd7dd", - strip_prefix = "bazel-toolchains-2.2.0", + sha256 = "239a1a673861eabf988e9804f45da3b94da28d1aff05c373b013193c315d9d9e", + strip_prefix = "bazel-toolchains-3.0.1", urls = [ - "https://github.com/bazelbuild/bazel-toolchains/releases/download/2.2.0/bazel-toolchains-2.2.0.tar.gz", - "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/2.2.0.tar.gz", + "https://github.com/bazelbuild/bazel-toolchains/releases/download/3.0.1/bazel-toolchains-3.0.1.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/3.0.1.tar.gz", ], ), build_bazel_rules_apple = dict( @@ -21,10 +21,10 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://github.com/bazelbuild/rules_apple/releases/download/0.19.0/rules_apple.0.19.0.tar.gz"], ), envoy_build_tools = dict( - sha256 = "c4193e6ab0c93db3e519dc8aeaf588e3dc414620063e00003150f64f03ad1f3f", - strip_prefix = "envoy-build-tools-84ca08de00eedd0ba08e7d5551108d6f03f5d362", - # 2020-03-24 - urls = ["https://github.com/envoyproxy/envoy-build-tools/archive/84ca08de00eedd0ba08e7d5551108d6f03f5d362.tar.gz"], + sha256 = "9d348f92ae8fb2495393109aac28aea314ad1fb013cdec1ab7b1224f804be1b7", + strip_prefix = "envoy-build-tools-823c2e9386eee5117f7ef9e3d7c90e784cd0d047", + # 2020-04-07 + urls = ["https://github.com/envoyproxy/envoy-build-tools/archive/823c2e9386eee5117f7ef9e3d7c90e784cd0d047.tar.gz"], ), boringssl = dict( sha256 = "a3d4de4f03cb321ef943678d72a045c9a19d26b23d6f4e313f97600c65201a27", diff --git a/ci/do_circle_ci.sh b/ci/do_circle_ci.sh index 9493ce1a51b6..036a75b1b8cb 100755 --- a/ci/do_circle_ci.sh +++ b/ci/do_circle_ci.sh @@ -16,7 +16,7 @@ export ENVOY_SRCDIR="$(pwd)" # xlarge resource_class. # See note: https://circleci.com/docs/2.0/configuration-reference/#resource_class for why we # hard code this (basically due to how docker works). -export NUM_CPUS=8 +export NUM_CPUS=6 # CircleCI doesn't support IPv6 by default, so we run all tests with IPv4 only. # IPv6 tests are run with Azure Pipelines. diff --git a/ci/envoy_build_sha.sh b/ci/envoy_build_sha.sh index 0a81b9fe8d35..6ea4600faeef 100644 --- a/ci/envoy_build_sha.sh +++ b/ci/envoy_build_sha.sh @@ -1,2 +1,2 @@ -ENVOY_BUILD_SHA=$(grep envoyproxy/envoy-build-ubuntu@sha256 $(dirname $0)/../.bazelrc | sed -e 's#.*envoyproxy/envoy-build-ubuntu@sha256:\(.*\)#\1#' | uniq) +ENVOY_BUILD_SHA=$(grep envoyproxy/envoy-build-ubuntu $(dirname $0)/../.bazelrc | sed -e 's#.*envoyproxy/envoy-build-ubuntu:\(.*\)#\1#' | uniq) [[ $(wc -l <<< "${ENVOY_BUILD_SHA}" | awk '{$1=$1};1') == 1 ]] || (echo ".bazelrc envoyproxy/envoy-build-ubuntu hashes are inconsistent!" && exit 1) diff --git a/ci/mac_ci_steps.sh b/ci/mac_ci_steps.sh index 552f9d7957ad..41e01d0fd134 100755 --- a/ci/mac_ci_steps.sh +++ b/ci/mac_ci_steps.sh @@ -26,10 +26,10 @@ BAZEL_BUILD_OPTIONS="--curses=no --show_task_finish --verbose_failures \ if [[ $# -gt 0 ]]; then TEST_TARGETS=$* else - TEST_TARGETS=//test/... + TEST_TARGETS=//test/integration/... fi -if [[ "$TEST_TARGETS" == "//test/..." ]]; then +if [[ "$TEST_TARGETS" == "//test/..." || "$TEST_TARGETS" == "//test/integration/..." ]]; then bazel build ${BAZEL_BUILD_OPTIONS} //source/exe:envoy-static fi bazel test ${BAZEL_BUILD_OPTIONS} ${TEST_TARGETS} diff --git a/ci/run_envoy_docker.sh b/ci/run_envoy_docker.sh index 615ddcd045ea..3ac8671d5b2d 100755 --- a/ci/run_envoy_docker.sh +++ b/ci/run_envoy_docker.sh @@ -10,7 +10,7 @@ set -e USER=root USER_GROUP=root -[[ -z "${IMAGE_NAME}" ]] && IMAGE_NAME="envoyproxy/envoy-build-ubuntu@sha256" +[[ -z "${IMAGE_NAME}" ]] && IMAGE_NAME="envoyproxy/envoy-build-ubuntu" # The IMAGE_ID defaults to the CI hash but can be set to an arbitrary image ID (found with 'docker # images'). [[ -z "${IMAGE_ID}" ]] && IMAGE_ID="${ENVOY_BUILD_SHA}" diff --git a/docs/requirements.txt b/docs/requirements.txt index 003a06ab5257..683f5c59b938 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,4 @@ -GitPython==3.0.0 +GitPython==3.1.1 Jinja2==2.10.3 MarkupSafe==1.1.1 Pygments==2.4.2 diff --git a/docs/root/api-v2/bootstrap/bootstrap.rst b/docs/root/api-v2/bootstrap/bootstrap.rst index 5208ac7f89a6..228d8a6019c8 100644 --- a/docs/root/api-v2/bootstrap/bootstrap.rst +++ b/docs/root/api-v2/bootstrap/bootstrap.rst @@ -10,4 +10,3 @@ Bootstrap ../config/metrics/v2/metrics_service.proto ../config/overload/v2alpha/overload.proto ../config/ratelimit/v2/rls.proto - tracing/tracing.rst diff --git a/docs/root/api-v2/bootstrap/tracing/tracing.rst b/docs/root/api-v2/bootstrap/tracing/tracing.rst deleted file mode 100644 index 04541081dc52..000000000000 --- a/docs/root/api-v2/bootstrap/tracing/tracing.rst +++ /dev/null @@ -1,9 +0,0 @@ -Tracing -======= - -.. toctree:: - :glob: - :maxdepth: 2 - - ../../config/trace/v2/trace.proto - ../../config/trace/v2alpha/xray.proto diff --git a/docs/root/api-v2/config/config.rst b/docs/root/api-v2/config/config.rst index 096a652fb7d1..feaa9c5b0c9a 100644 --- a/docs/root/api-v2/config/config.rst +++ b/docs/root/api-v2/config/config.rst @@ -16,4 +16,5 @@ Extensions listener/listener grpc_credential/grpc_credential retry/retry + trace/trace wasm/wasm diff --git a/docs/root/api-v2/config/trace/trace.rst b/docs/root/api-v2/config/trace/trace.rst new file mode 100644 index 000000000000..80f06cc5c349 --- /dev/null +++ b/docs/root/api-v2/config/trace/trace.rst @@ -0,0 +1,9 @@ +HTTP Tracers +============== + +.. toctree:: + :glob: + :maxdepth: 2 + + v2/* + v2alpha/* diff --git a/docs/root/api-v3/bootstrap/bootstrap.rst b/docs/root/api-v3/bootstrap/bootstrap.rst index bbaf343a20ab..d2397a9bf2ac 100644 --- a/docs/root/api-v3/bootstrap/bootstrap.rst +++ b/docs/root/api-v3/bootstrap/bootstrap.rst @@ -10,4 +10,3 @@ Bootstrap ../config/metrics/v3/metrics_service.proto ../config/overload/v3/overload.proto ../config/ratelimit/v3/rls.proto - tracing/tracing.rst diff --git a/docs/root/api-v3/bootstrap/tracing/tracing.rst b/docs/root/api-v3/bootstrap/tracing/tracing.rst deleted file mode 100644 index abc9e78169f7..000000000000 --- a/docs/root/api-v3/bootstrap/tracing/tracing.rst +++ /dev/null @@ -1,9 +0,0 @@ -Tracing -======= - -.. toctree:: - :glob: - :maxdepth: 2 - - ../../config/trace/v3/trace.proto - ../../config/trace/v3/xray.proto diff --git a/docs/root/api-v3/config/config.rst b/docs/root/api-v3/config/config.rst index 3725a558c7c8..d7e6e6edd43c 100644 --- a/docs/root/api-v3/config/config.rst +++ b/docs/root/api-v3/config/config.rst @@ -15,3 +15,4 @@ Extensions cluster/cluster grpc_credential/grpc_credential retry/retry + trace/trace diff --git a/docs/root/api-v3/config/trace/trace.rst b/docs/root/api-v3/config/trace/trace.rst new file mode 100644 index 000000000000..94f834998f3b --- /dev/null +++ b/docs/root/api-v3/config/trace/trace.rst @@ -0,0 +1,8 @@ +HTTP Tracers +============== + +.. toctree:: + :glob: + :maxdepth: 2 + + v3/* diff --git a/docs/root/configuration/http/http_filters/fault_filter.rst b/docs/root/configuration/http/http_filters/fault_filter.rst index 7b0bc9b83a12..80678714db69 100644 --- a/docs/root/configuration/http/http_filters/fault_filter.rst +++ b/docs/root/configuration/http/http_filters/fault_filter.rst @@ -36,19 +36,58 @@ The fault filter has the capability to allow fault configuration to be specified This is useful in certain scenarios in which it is desired to allow the client to specify its own fault configuration. The currently supported header controls are: -* Request abort configuration via the *x-envoy-fault-abort-request* header. The header value - should be an integer that specifies the HTTP status code to return in response to a request - and must be in the range [200, 600). In order for the header to work, :ref:`header_abort +x-envoy-fault-abort-request + HTTP status code to abort a request with. The header value should be an integer that specifies + the HTTP status code to return in response to a request and must be in the range [200, 600). + In order for the header to work, :ref:`header_abort ` needs to be set. -* Request delay configuration via the *x-envoy-fault-delay-request* header. The header value - should be an integer that specifies the number of milliseconds to throttle the latency for. - In order for the header to work, :ref:`header_delay + +x-envoy-fault-abort-request-percentage + The percentage of requests that should be failed with a status code that's defined + by the value of *x-envoy-fault-abort-request* HTTP header. The header value should be an integer + that specifies the numerator of the percentage of request to apply aborts to and must be greater + or equal to 0 and its maximum value is capped by the value of the numerator of + :ref:`percentage ` field. + Percentage's denominator is equal to default percentage's denominator + :ref:`percentage ` field. + In order for the header to work, :ref:`header_abort + ` needs to be set and + *x-envoy-fault-abort-request* HTTP header needs to be a part of a request. + +x-envoy-fault-delay-request + The duration to delay a request by. The header value should be an integer that specifies the number + of milliseconds to throttle the latency for. In order for the header to work, :ref:`header_delay ` needs to be set. -* Response rate limit configuration via the *x-envoy-fault-throughput-response* header. The - header value should be an integer that specifies the limit in KiB/s and must be > 0. In order - for the header to work, :ref:`header_limit + +x-envoy-fault-delay-request-percentage + The percentage of requests that should be delayed by a duration that's defined by the value of + *x-envoy-fault-delay-request* HTTP header. The header value should be an integer that + specifies the percentage of request to apply delays to and must be greater + or equal to 0 and its maximum value is capped by the value of the numerator of + :ref:`percentage ` field. + Percentage's denominator is equal to default percentage's denominator + :ref:`percentage ` field. + In order for the header to work, :ref:`header_delay + ` needs to be set and + *x-envoy-fault-delay-request* HTTP header needs to be a part of a request. + +x-envoy-fault-throughput-response + The rate limit to use when a response to a caller is sent. The header value should be an integer + that specifies the limit in KiB/s and must be > 0. In order for the header to work, :ref:`header_limit ` needs to be set. +x-envoy-fault-throughput-response-percentage + The percentage of requests whose response rate should be limited to the value of + *x-envoy-fault-throughput-response* HTTP header. The header value should be an integer that + specifies the percentage of request to apply delays to and must be greater + or equal to 0 and its maximum value is capped by the value of the numerator of + :ref:`percentage ` field. + Percentage's denominator is equal to default percentage's denominator + :ref:`percentage ` field. + In order for the header to work, :ref:`header_limit + ` needs to be set and + *x-envoy-fault-delay-request* HTTP header needs to be a part of a request. + .. attention:: Allowing header control is inherently dangerous if exposed to untrusted clients. In this case, @@ -98,7 +137,7 @@ fault.http.abort.abort_percent `. fault.http.abort.http_status - HTTP status code that will be used as the of requests that will be + HTTP status code that will be used as the response status code of requests that will be aborted if the headers match. Defaults to the HTTP status code specified in the config. If the config does not contain an *abort* block, then *http_status* defaults to 0. For historic reasons, this runtime key is diff --git a/docs/root/configuration/http/http_filters/grpc_json_transcoder_filter.rst b/docs/root/configuration/http/http_filters/grpc_json_transcoder_filter.rst index b4077984c62e..a1fdfdcccdf5 100644 --- a/docs/root/configuration/http/http_filters/grpc_json_transcoder_filter.rst +++ b/docs/root/configuration/http/http_filters/grpc_json_transcoder_filter.rst @@ -82,7 +82,10 @@ as its output message type. The implementation needs to set (which sets the value of the HTTP response `Content-Type` header) and `data `_ (which sets the HTTP response body) accordingly. - +Multiple `google.api.HttpBody `_ +can be send by the gRPC server in the server streaming case. +In this case, HTTP response header `Content-Type` will use the `content-type` from the first +`google.api.HttpBody `. Sample Envoy configuration -------------------------- diff --git a/docs/root/configuration/http/http_filters/grpc_stats_filter.rst b/docs/root/configuration/http/http_filters/grpc_stats_filter.rst index 78ab88624dff..984a2f9348d7 100644 --- a/docs/root/configuration/http/http_filters/grpc_stats_filter.rst +++ b/docs/root/configuration/http/http_filters/grpc_stats_filter.rst @@ -21,6 +21,8 @@ are shown in this form. See the documentation for :ref:`individual_method_stats_allowlist ` and :ref:`stats_for_all_methods `. +To enable *upstream_rq_time* (v3 API only) see :ref:`enable_upstream_stats `. + .. csv-table:: :header: Name, Type, Description @@ -31,3 +33,4 @@ and :ref:`stats_for_all_methods ..total, Counter, Total service/method calls ..request_message_count, Counter, Total request message count for service/method calls ..response_message_count, Counter, Total response message count for service/method calls + ..upstream_rq_time, Histogram, Request time milliseconds diff --git a/docs/root/configuration/http/http_filters/header_to_metadata_filter.rst b/docs/root/configuration/http/http_filters/header_to_metadata_filter.rst index 73ddc9a1f493..38a55736861e 100644 --- a/docs/root/configuration/http/http_filters/header_to_metadata_filter.rst +++ b/docs/root/configuration/http/http_filters/header_to_metadata_filter.rst @@ -60,6 +60,33 @@ This would then allow requests with the `x-version` header set to be matched aga endpoints with the corresponding version. Whereas requests with that header missing would be matched with the default endpoints. +Note that this filter also supports per route configuration: + +.. code-block:: yaml + + route_config: + name: local_route + virtual_hosts: + - name: local_service + domains: ["*"] + routes: + - match: { prefix: "/version-to-metadata" } + route: { cluster: service } + per_filter_config: + envoy.filters.http.header_to_metadata: + request_rules: + - header: x-version + on_header_present: + metadata_namespace: envoy.lb + key: version + type: STRING + remove: false + - match: { prefix: "/" } + route: { cluster: some_service } + +This can be used to either override the global configuration or if the global configuration +is empty (no rules), it can be used to only enable the filter at a per route level. + Statistics ---------- diff --git a/docs/root/configuration/http/http_filters/rate_limit_filter.rst b/docs/root/configuration/http/http_filters/rate_limit_filter.rst index 7a88facc199b..e76dda3d6f02 100644 --- a/docs/root/configuration/http/http_filters/rate_limit_filter.rst +++ b/docs/root/configuration/http/http_filters/rate_limit_filter.rst @@ -101,7 +101,7 @@ ratelimit.http_filter_enabled % of requests that will call the rate limit service. Defaults to 100. ratelimit.http_filter_enforcing - % of requests that will call the rate limit service and enforce the decision. Defaults to 100. + % of requests that that will have the rate limit service decision enforced. Defaults to 100. This can be used to test what would happen before fully enforcing the outcome. ratelimit..http_filter_enabled diff --git a/docs/root/configuration/http/http_filters/router_filter.rst b/docs/root/configuration/http/http_filters/router_filter.rst index d1a5d17f1a40..6575e0ed2336 100644 --- a/docs/root/configuration/http/http_filters/router_filter.rst +++ b/docs/root/configuration/http/http_filters/router_filter.rst @@ -388,7 +388,6 @@ owning HTTP connection manager. rq_direct_response, Counter, Total requests that resulted in a direct response rq_total, Counter, Total routed requests rq_reset_after_downstream_response_started, Counter, Total requests that were reset after downstream response had started - rq_retry_skipped_request_not_complete, Counter, Total retries that were skipped as the request is not yet complete .. _config_http_filters_router_vcluster_stats: diff --git a/docs/root/configuration/listeners/network_filters/network_filters.rst b/docs/root/configuration/listeners/network_filters/network_filters.rst index 9e75675316a5..65511250f84b 100644 --- a/docs/root/configuration/listeners/network_filters/network_filters.rst +++ b/docs/root/configuration/listeners/network_filters/network_filters.rst @@ -26,4 +26,5 @@ filters. tcp_proxy_filter thrift_proxy_filter sni_cluster_filter + sni_dynamic_forward_proxy_filter zookeeper_proxy_filter diff --git a/docs/root/configuration/listeners/network_filters/sni_dynamic_forward_proxy_filter.rst b/docs/root/configuration/listeners/network_filters/sni_dynamic_forward_proxy_filter.rst new file mode 100644 index 000000000000..4751b3a614e5 --- /dev/null +++ b/docs/root/configuration/listeners/network_filters/sni_dynamic_forward_proxy_filter.rst @@ -0,0 +1,72 @@ +.. _config_network_filters_sni_dynamic_forward_proxy: + +SNI dynamic forward proxy +========================= + +.. attention:: + + SNI dynamic forward proxy support should be considered alpha and not production ready. + +Through the combination of :ref:`TLS inspector ` listener filter, +this network filter and the +:ref:`dynamic forward proxy cluster `, +Envoy supports SNI based dynamic forward proxy. The implementation works just like the +:ref:`HTTP dynamic forward proxy `, but using the value in +SNI as target host instead. + +The following is a complete configuration that configures both this filter +as well as the :ref:`dynamic forward proxy cluster +`. Both filter and cluster +must be configured together and point to the same DNS cache parameters for Envoy to operate as an +SNI dynamic forward proxy. + +.. note:: + + The following config doesn't terminate TLS in listener, so there is no need to configure TLS context + in cluster. The TLS handshake is passed through by Envoy. + +.. code-block:: yaml + + admin: + access_log_path: /tmp/admin_access.log + address: + socket_address: + protocol: TCP + address: 127.0.0.1 + port_value: 9901 + static_resources: + listeners: + - name: listener_0 + address: + socket_address: + protocol: TCP + address: 0.0.0.0 + port_value: 10000 + listener_filters: + - name: envoy.filters.listener.tls_inspector + filter_chains: + - filters: + - name: envoy.filters.network.sni_dynamic_forward_proxy + typed_config: + "@type": type.googleapis.com/envoy.config.filter.network.sni_dynamic_forward_proxy.v2alpha.FilterConfig + port_value: 443 + dns_cache_config: + name: dynamic_forward_proxy_cache_config + dns_lookup_family: V4_ONLY + - name: envoy.tcp_proxy + typed_config: + "@type": type.googleapis.com/envoy.config.filter.network.tcp_proxy.v2.TcpProxy + stat_prefix: tcp + cluster: dynamic_forward_proxy_cluster + clusters: + - name: dynamic_forward_proxy_cluster + connect_timeout: 1s + lb_policy: CLUSTER_PROVIDED + cluster_type: + name: envoy.clusters.dynamic_forward_proxy + typed_config: + "@type": type.googleapis.com/envoy.config.cluster.dynamic_forward_proxy.v2alpha.ClusterConfig + dns_cache_config: + name: dynamic_forward_proxy_cache_config + dns_lookup_family: V4_ONLY + diff --git a/docs/root/configuration/observability/access_log.rst b/docs/root/configuration/observability/access_log.rst index 21eec6fd1bfe..46637c05ec4b 100644 --- a/docs/root/configuration/observability/access_log.rst +++ b/docs/root/configuration/observability/access_log.rst @@ -410,12 +410,17 @@ The following command operators are supported: JSON struct or list is rendered. Structs and lists may be nested. In any event, the maximum length is ignored -%FILTER_STATE(KEY):Z% +.. _config_access_log_format_filter_state: + +%FILTER_STATE(KEY:F):Z% HTTP :ref:`Filter State ` info, where the KEY is required to look up the filter state object. The serialized proto will be logged as JSON string if possible. If the serialized proto is unknown to Envoy it will be logged as protobuf debug string. Z is an optional parameter denoting string truncation up to Z characters long. + F is an optional parameter used to indicate which method FilterState uses for serialization. + If 'PLAIN' is set, the filter state object will be serialized as an unstructured string. + If 'TYPED' is set or no F provided, the filter state object will be serialized as an JSON string. TCP Same as HTTP, the filter state is from connection instead of a L7 request. diff --git a/docs/root/install/building.rst b/docs/root/install/building.rst index 9541bc6316bd..ada695fa53b5 100644 --- a/docs/root/install/building.rst +++ b/docs/root/install/building.rst @@ -41,13 +41,10 @@ be found in the following repositories: * `envoyproxy/envoy-alpine-debug `_: Release binary with debug symbols on top of a **glibc** alpine base. -In the above repositories, the *latest* tag points to the latest official release. - .. note:: - The above repositories used to contain the dev images described below. They remain to avoid - breaking existing users. New dev images are added to the repositories described in the following - section. + In the above repositories, we do **not** tag a *latest* image. As we now do security/stable + releases, *latest* has no good meaning and users should pin to a specific tag. On every master commit we additionally create a set of development Docker images. These images can be found in the following repositories: diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index dfc14eb93609..662f014a72d2 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -5,9 +5,18 @@ Changes ------- * access loggers: added GRPC_STATUS operator on logging format. +* access loggers: extened specifier for FilterStateFormatter to output :ref:`unstructured log string `. +* dynamic forward proxy: added :ref:`SNI based dynamic forward proxy ` support. +* fault: added support for controlling the percentage of requests that abort, delay and response rate limits faults + are applied to using :ref:`HTTP headers ` to the HTTP fault filter. +* filter: add `upstram_rq_time` stats to the GPRC stats filter. + Disabled by default and can be enabled via :ref:`enable_upstream_stats `. +* grpc-json: added support for streaming response using + `google.api.HttpBody `_. * http: fixed a bug where the upgrade header was not cleared on responses to non-upgrade requests. Can be reverted temporarily by setting runtime feature `envoy.reloadable_features.fix_upgrade_response` to false. * network filters: added a :ref:`postgres proxy filter `. +* router: allow retries of streaming or incomplete requests. This removes stat `rq_retry_skipped_request_not_complete`. * tracing: tracing configuration has been made fully dynamic and every HTTP connection manager can now have a separate :ref:`tracing provider `. diff --git a/generated_api_shadow/envoy/config/bootstrap/v2/bootstrap.proto b/generated_api_shadow/envoy/config/bootstrap/v2/bootstrap.proto index 0942c78bd4ab..622304483eb2 100644 --- a/generated_api_shadow/envoy/config/bootstrap/v2/bootstrap.proto +++ b/generated_api_shadow/envoy/config/bootstrap/v2/bootstrap.proto @@ -12,7 +12,7 @@ import "envoy/api/v2/core/socket_option.proto"; import "envoy/api/v2/listener.proto"; import "envoy/config/metrics/v2/stats.proto"; import "envoy/config/overload/v2alpha/overload.proto"; -import "envoy/config/trace/v2/trace.proto"; +import "envoy/config/trace/v2/http_tracer.proto"; import "google/protobuf/duration.proto"; import "google/protobuf/struct.proto"; diff --git a/generated_api_shadow/envoy/config/bootstrap/v3/bootstrap.proto b/generated_api_shadow/envoy/config/bootstrap/v3/bootstrap.proto index 04d714065f9f..3b0861d81850 100644 --- a/generated_api_shadow/envoy/config/bootstrap/v3/bootstrap.proto +++ b/generated_api_shadow/envoy/config/bootstrap/v3/bootstrap.proto @@ -11,7 +11,7 @@ import "envoy/config/core/v3/socket_option.proto"; import "envoy/config/listener/v3/listener.proto"; import "envoy/config/metrics/v3/stats.proto"; import "envoy/config/overload/v3/overload.proto"; -import "envoy/config/trace/v3/trace.proto"; +import "envoy/config/trace/v3/http_tracer.proto"; import "envoy/extensions/transport_sockets/tls/v3/cert.proto"; import "google/protobuf/duration.proto"; diff --git a/generated_api_shadow/envoy/config/bootstrap/v4alpha/bootstrap.proto b/generated_api_shadow/envoy/config/bootstrap/v4alpha/bootstrap.proto index f8ada24bf2e2..9177f186f6b5 100644 --- a/generated_api_shadow/envoy/config/bootstrap/v4alpha/bootstrap.proto +++ b/generated_api_shadow/envoy/config/bootstrap/v4alpha/bootstrap.proto @@ -11,7 +11,7 @@ import "envoy/config/core/v4alpha/socket_option.proto"; import "envoy/config/listener/v3/listener.proto"; import "envoy/config/metrics/v3/stats.proto"; import "envoy/config/overload/v3/overload.proto"; -import "envoy/config/trace/v4alpha/trace.proto"; +import "envoy/config/trace/v4alpha/http_tracer.proto"; import "envoy/extensions/transport_sockets/tls/v4alpha/cert.proto"; import "google/protobuf/duration.proto"; diff --git a/generated_api_shadow/envoy/config/core/v3/health_check.proto b/generated_api_shadow/envoy/config/core/v3/health_check.proto index 2ed3b69eaa4f..5b95ebe39de3 100644 --- a/generated_api_shadow/envoy/config/core/v3/health_check.proto +++ b/generated_api_shadow/envoy/config/core/v3/health_check.proto @@ -83,11 +83,13 @@ message HealthCheck { // left empty (default value), the name of the cluster this health check is associated // with will be used. The host header can be customized for a specific endpoint by setting the // :ref:`hostname ` field. - string host = 1; + string host = 1 [(validate.rules).string = {well_known_regex: HTTP_HEADER_VALUE strict: false}]; // Specifies the HTTP path that will be requested during health checking. For example // */healthcheck*. - string path = 2 [(validate.rules).string = {min_bytes: 1}]; + string path = 2 [ + (validate.rules).string = {min_bytes: 1 well_known_regex: HTTP_HEADER_VALUE strict: false} + ]; // [#not-implemented-hide:] HTTP specific payload. Payload send = 3; @@ -104,7 +106,9 @@ message HealthCheck { // Specifies a list of HTTP headers that should be removed from each request that is sent to the // health checked cluster. - repeated string request_headers_to_remove = 8; + repeated string request_headers_to_remove = 8 [(validate.rules).repeated = { + items {string {well_known_regex: HTTP_HEADER_NAME strict: false}} + }]; // Specifies a list of HTTP response statuses considered healthy. If provided, replaces default // 200-only policy - 200 must be included explicitly as needed. Ranges follow half-open @@ -170,7 +174,8 @@ message HealthCheck { // left empty (default value), the name of the cluster this health check is associated // with will be used. The authority header can be customized for a specific endpoint by setting // the :ref:`hostname ` field. - string authority = 2; + string authority = 2 + [(validate.rules).string = {well_known_regex: HTTP_HEADER_VALUE strict: false}]; } // Custom health check. diff --git a/generated_api_shadow/envoy/config/core/v4alpha/health_check.proto b/generated_api_shadow/envoy/config/core/v4alpha/health_check.proto index 0e6c4e73c2a2..1975c309a7de 100644 --- a/generated_api_shadow/envoy/config/core/v4alpha/health_check.proto +++ b/generated_api_shadow/envoy/config/core/v4alpha/health_check.proto @@ -87,11 +87,13 @@ message HealthCheck { // left empty (default value), the name of the cluster this health check is associated // with will be used. The host header can be customized for a specific endpoint by setting the // :ref:`hostname ` field. - string host = 1; + string host = 1 [(validate.rules).string = {well_known_regex: HTTP_HEADER_VALUE strict: false}]; // Specifies the HTTP path that will be requested during health checking. For example // */healthcheck*. - string path = 2 [(validate.rules).string = {min_bytes: 1}]; + string path = 2 [ + (validate.rules).string = {min_bytes: 1 well_known_regex: HTTP_HEADER_VALUE strict: false} + ]; // [#not-implemented-hide:] HTTP specific payload. Payload send = 3; @@ -108,7 +110,9 @@ message HealthCheck { // Specifies a list of HTTP headers that should be removed from each request that is sent to the // health checked cluster. - repeated string request_headers_to_remove = 8; + repeated string request_headers_to_remove = 8 [(validate.rules).repeated = { + items {string {well_known_regex: HTTP_HEADER_NAME strict: false}} + }]; // Specifies a list of HTTP response statuses considered healthy. If provided, replaces default // 200-only policy - 200 must be included explicitly as needed. Ranges follow half-open @@ -169,7 +173,8 @@ message HealthCheck { // left empty (default value), the name of the cluster this health check is associated // with will be used. The authority header can be customized for a specific endpoint by setting // the :ref:`hostname ` field. - string authority = 2; + string authority = 2 + [(validate.rules).string = {well_known_regex: HTTP_HEADER_VALUE strict: false}]; } // Custom health check. diff --git a/generated_api_shadow/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto b/generated_api_shadow/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto index f760c25dcde3..c78e69b2ae30 100644 --- a/generated_api_shadow/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto +++ b/generated_api_shadow/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto @@ -7,7 +7,7 @@ import "envoy/api/v2/core/protocol.proto"; import "envoy/api/v2/route.proto"; import "envoy/api/v2/scoped_route.proto"; import "envoy/config/filter/accesslog/v2/accesslog.proto"; -import "envoy/config/trace/v2/trace.proto"; +import "envoy/config/trace/v2/http_tracer.proto"; import "envoy/type/percent.proto"; import "envoy/type/tracing/v2/custom_tag.proto"; diff --git a/generated_api_shadow/envoy/config/route/v3/route_components.proto b/generated_api_shadow/envoy/config/route/v3/route_components.proto index 8ef58ba20798..616e76af302e 100644 --- a/generated_api_shadow/envoy/config/route/v3/route_components.proto +++ b/generated_api_shadow/envoy/config/route/v3/route_components.proto @@ -106,7 +106,9 @@ message VirtualHost { // Specifies a list of HTTP headers that should be removed from each request // handled by this virtual host. - repeated string request_headers_to_remove = 13; + repeated string request_headers_to_remove = 13 [(validate.rules).repeated = { + items {string {min_bytes: 1 well_known_regex: HTTP_HEADER_NAME strict: false}} + }]; // Specifies a list of HTTP headers that should be added to each response // handled by this virtual host. Headers specified at this level are applied @@ -225,7 +227,9 @@ message Route { // The metadata should go under the filter namespace that will need it. // For instance, if the metadata is intended for the Router filter, // the filter name should be specified as *envoy.filters.http.router*. - repeated string request_headers_to_remove = 12; + repeated string request_headers_to_remove = 12 [(validate.rules).repeated = { + items {string {min_bytes: 1 well_known_regex: HTTP_HEADER_NAME strict: false}} + }]; // Decorator for the matched route. repeated core.v3.HeaderValueOption response_headers_to_add = 10 diff --git a/generated_api_shadow/envoy/config/route/v4alpha/route_components.proto b/generated_api_shadow/envoy/config/route/v4alpha/route_components.proto index 33f8d64543df..e813b632edb0 100644 --- a/generated_api_shadow/envoy/config/route/v4alpha/route_components.proto +++ b/generated_api_shadow/envoy/config/route/v4alpha/route_components.proto @@ -108,7 +108,9 @@ message VirtualHost { // Specifies a list of HTTP headers that should be removed from each request // handled by this virtual host. - repeated string request_headers_to_remove = 13; + repeated string request_headers_to_remove = 13 [(validate.rules).repeated = { + items {string {min_bytes: 1 well_known_regex: HTTP_HEADER_NAME strict: false}} + }]; // Specifies a list of HTTP headers that should be added to each response // handled by this virtual host. Headers specified at this level are applied @@ -252,7 +254,9 @@ message Route { // Specifies a list of HTTP headers that should be removed from each request // matching this route. - repeated string request_headers_to_remove = 12; + repeated string request_headers_to_remove = 12 [(validate.rules).repeated = { + items {string {min_bytes: 1 well_known_regex: HTTP_HEADER_NAME strict: false}} + }]; // Specifies a set of headers that will be added to responses to requests // matching this route. Headers specified at this level are applied before diff --git a/generated_api_shadow/envoy/config/trace/v2/datadog.proto b/generated_api_shadow/envoy/config/trace/v2/datadog.proto new file mode 100644 index 000000000000..0992601a8acc --- /dev/null +++ b/generated_api_shadow/envoy/config/trace/v2/datadog.proto @@ -0,0 +1,23 @@ +syntax = "proto3"; + +package envoy.config.trace.v2; + +import "udpa/annotations/status.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.trace.v2"; +option java_outer_classname = "DatadogProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = FROZEN; + +// [#protodoc-title: Datadog tracer] + +// Configuration for the Datadog tracer. +// [#extension: envoy.tracers.datadog] +message DatadogConfig { + // The cluster to use for submitting traces to the Datadog agent. + string collector_cluster = 1 [(validate.rules).string = {min_bytes: 1}]; + + // The name used for the service when traces are generated by envoy. + string service_name = 2 [(validate.rules).string = {min_bytes: 1}]; +} diff --git a/generated_api_shadow/envoy/config/trace/v2/dynamic_ot.proto b/generated_api_shadow/envoy/config/trace/v2/dynamic_ot.proto new file mode 100644 index 000000000000..55c6d401b335 --- /dev/null +++ b/generated_api_shadow/envoy/config/trace/v2/dynamic_ot.proto @@ -0,0 +1,29 @@ +syntax = "proto3"; + +package envoy.config.trace.v2; + +import "google/protobuf/struct.proto"; + +import "udpa/annotations/status.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.trace.v2"; +option java_outer_classname = "DynamicOtProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = FROZEN; + +// [#protodoc-title: Dynamically loadable OpenTracing tracer] + +// DynamicOtConfig is used to dynamically load a tracer from a shared library +// that implements the `OpenTracing dynamic loading API +// `_. +// [#extension: envoy.tracers.dynamic_ot] +message DynamicOtConfig { + // Dynamic library implementing the `OpenTracing API + // `_. + string library = 1 [(validate.rules).string = {min_bytes: 1}]; + + // The configuration to use when creating a tracer from the given dynamic + // library. + google.protobuf.Struct config = 2; +} diff --git a/generated_api_shadow/envoy/config/trace/v2/http_tracer.proto b/generated_api_shadow/envoy/config/trace/v2/http_tracer.proto new file mode 100644 index 000000000000..fba830b987b6 --- /dev/null +++ b/generated_api_shadow/envoy/config/trace/v2/http_tracer.proto @@ -0,0 +1,65 @@ +syntax = "proto3"; + +package envoy.config.trace.v2; + +import "google/protobuf/any.proto"; +import "google/protobuf/struct.proto"; + +import "udpa/annotations/status.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.trace.v2"; +option java_outer_classname = "HttpTracerProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = FROZEN; + +// [#protodoc-title: Tracing] +// Tracing :ref:`architecture overview `. + +// The tracing configuration specifies settings for an HTTP tracer provider used by Envoy. +// +// Envoy may support other tracers in the future, but right now the HTTP tracer is the only one +// supported. +// +// .. attention:: +// +// Use of this message type has been deprecated in favor of direct use of +// :ref:`Tracing.Http `. +message Tracing { + // Configuration for an HTTP tracer provider used by Envoy. + // + // The configuration is defined by the + // :ref:`HttpConnectionManager.Tracing ` + // :ref:`provider ` + // field. + message Http { + // The name of the HTTP trace driver to instantiate. The name must match a + // supported HTTP trace driver. Built-in trace drivers: + // + // - *envoy.tracers.lightstep* + // - *envoy.tracers.zipkin* + // - *envoy.tracers.dynamic_ot* + // - *envoy.tracers.datadog* + // - *envoy.tracers.opencensus* + // - *envoy.tracers.xray* + string name = 1 [(validate.rules).string = {min_bytes: 1}]; + + // Trace driver specific configuration which depends on the driver being instantiated. + // See the trace drivers for examples: + // + // - :ref:`LightstepConfig ` + // - :ref:`ZipkinConfig ` + // - :ref:`DynamicOtConfig ` + // - :ref:`DatadogConfig ` + // - :ref:`OpenCensusConfig ` + // - :ref:`AWS X-Ray ` + oneof config_type { + google.protobuf.Struct config = 2 [deprecated = true]; + + google.protobuf.Any typed_config = 3; + } + } + + // Provides configuration for the HTTP tracer. + Http http = 1; +} diff --git a/generated_api_shadow/envoy/config/trace/v2/lightstep.proto b/generated_api_shadow/envoy/config/trace/v2/lightstep.proto new file mode 100644 index 000000000000..849749baaa0d --- /dev/null +++ b/generated_api_shadow/envoy/config/trace/v2/lightstep.proto @@ -0,0 +1,43 @@ +syntax = "proto3"; + +package envoy.config.trace.v2; + +import "udpa/annotations/status.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.trace.v2"; +option java_outer_classname = "LightstepProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = FROZEN; + +// [#protodoc-title: LightStep tracer] + +// Configuration for the LightStep tracer. +// [#extension: envoy.tracers.lightstep] +message LightstepConfig { + // Available propagation modes + enum PropagationMode { + // Propagate trace context in the single header x-ot-span-context. + ENVOY = 0; + + // Propagate trace context using LightStep's native format. + LIGHTSTEP = 1; + + // Propagate trace context using the b3 format. + B3 = 2; + + // Propagation trace context using the w3 trace-context standard. + TRACE_CONTEXT = 3; + } + + // The cluster manager cluster that hosts the LightStep collectors. + string collector_cluster = 1 [(validate.rules).string = {min_bytes: 1}]; + + // File containing the access token to the `LightStep + // `_ API. + string access_token_file = 2 [(validate.rules).string = {min_bytes: 1}]; + + // Propagation modes to use by LightStep's tracer. + repeated PropagationMode propagation_modes = 3 + [(validate.rules).repeated = {items {enum {defined_only: true}}}]; +} diff --git a/generated_api_shadow/envoy/config/trace/v2/opencensus.proto b/generated_api_shadow/envoy/config/trace/v2/opencensus.proto new file mode 100644 index 000000000000..8c8dab94a875 --- /dev/null +++ b/generated_api_shadow/envoy/config/trace/v2/opencensus.proto @@ -0,0 +1,93 @@ +syntax = "proto3"; + +package envoy.config.trace.v2; + +import "envoy/api/v2/core/grpc_service.proto"; + +import "opencensus/proto/trace/v1/trace_config.proto"; + +import "udpa/annotations/status.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.trace.v2"; +option java_outer_classname = "OpencensusProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = FROZEN; + +// [#protodoc-title: OpenCensus tracer] + +// Configuration for the OpenCensus tracer. +// [#next-free-field: 15] +// [#extension: envoy.tracers.opencensus] +message OpenCensusConfig { + enum TraceContext { + // No-op default, no trace context is utilized. + NONE = 0; + + // W3C Trace-Context format "traceparent:" header. + TRACE_CONTEXT = 1; + + // Binary "grpc-trace-bin:" header. + GRPC_TRACE_BIN = 2; + + // "X-Cloud-Trace-Context:" header. + CLOUD_TRACE_CONTEXT = 3; + + // X-B3-* headers. + B3 = 4; + } + + reserved 7; + + // Configures tracing, e.g. the sampler, max number of annotations, etc. + opencensus.proto.trace.v1.TraceConfig trace_config = 1; + + // Enables the stdout exporter if set to true. This is intended for debugging + // purposes. + bool stdout_exporter_enabled = 2; + + // Enables the Stackdriver exporter if set to true. The project_id must also + // be set. + bool stackdriver_exporter_enabled = 3; + + // The Cloud project_id to use for Stackdriver tracing. + string stackdriver_project_id = 4; + + // (optional) By default, the Stackdriver exporter will connect to production + // Stackdriver. If stackdriver_address is non-empty, it will instead connect + // to this address, which is in the gRPC format: + // https://github.com/grpc/grpc/blob/master/doc/naming.md + string stackdriver_address = 10; + + // (optional) The gRPC server that hosts Stackdriver tracing service. Only + // Google gRPC is supported. If :ref:`target_uri ` + // is not provided, the default production Stackdriver address will be used. + api.v2.core.GrpcService stackdriver_grpc_service = 13; + + // Enables the Zipkin exporter if set to true. The url and service name must + // also be set. + bool zipkin_exporter_enabled = 5; + + // The URL to Zipkin, e.g. "http://127.0.0.1:9411/api/v2/spans" + string zipkin_url = 6; + + // Enables the OpenCensus Agent exporter if set to true. The ocagent_address or + // ocagent_grpc_service must also be set. + bool ocagent_exporter_enabled = 11; + + // The address of the OpenCensus Agent, if its exporter is enabled, in gRPC + // format: https://github.com/grpc/grpc/blob/master/doc/naming.md + // [#comment:TODO: deprecate this field] + string ocagent_address = 12; + + // (optional) The gRPC server hosted by the OpenCensus Agent. Only Google gRPC is supported. + // This is only used if the ocagent_address is left empty. + api.v2.core.GrpcService ocagent_grpc_service = 14; + + // List of incoming trace context headers we will accept. First one found + // wins. + repeated TraceContext incoming_trace_context = 8; + + // List of outgoing trace context headers we will produce. + repeated TraceContext outgoing_trace_context = 9; +} diff --git a/generated_api_shadow/envoy/config/trace/v2/service.proto b/generated_api_shadow/envoy/config/trace/v2/service.proto new file mode 100644 index 000000000000..d102499b6261 --- /dev/null +++ b/generated_api_shadow/envoy/config/trace/v2/service.proto @@ -0,0 +1,21 @@ +syntax = "proto3"; + +package envoy.config.trace.v2; + +import "envoy/api/v2/core/grpc_service.proto"; + +import "udpa/annotations/status.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.trace.v2"; +option java_outer_classname = "ServiceProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = FROZEN; + +// [#protodoc-title: Trace Service] + +// Configuration structure. +message TraceServiceConfig { + // The upstream gRPC cluster that hosts the metrics service. + api.v2.core.GrpcService grpc_service = 1 [(validate.rules).message = {required: true}]; +} diff --git a/generated_api_shadow/envoy/config/trace/v2/trace.proto b/generated_api_shadow/envoy/config/trace/v2/trace.proto index 308a97e4eef4..9f0670b28f7a 100644 --- a/generated_api_shadow/envoy/config/trace/v2/trace.proto +++ b/generated_api_shadow/envoy/config/trace/v2/trace.proto @@ -2,254 +2,16 @@ syntax = "proto3"; package envoy.config.trace.v2; -import "envoy/api/v2/core/grpc_service.proto"; - -import "google/protobuf/any.proto"; -import "google/protobuf/struct.proto"; -import "google/protobuf/wrappers.proto"; - -import "opencensus/proto/trace/v1/trace_config.proto"; - -import "envoy/annotations/deprecation.proto"; import "udpa/annotations/status.proto"; -import "validate/validate.proto"; + +import public "envoy/config/trace/v2/datadog.proto"; +import public "envoy/config/trace/v2/dynamic_ot.proto"; +import public "envoy/config/trace/v2/http_tracer.proto"; +import public "envoy/config/trace/v2/lightstep.proto"; +import public "envoy/config/trace/v2/opencensus.proto"; +import public "envoy/config/trace/v2/service.proto"; +import public "envoy/config/trace/v2/zipkin.proto"; option java_package = "io.envoyproxy.envoy.config.trace.v2"; option java_outer_classname = "TraceProto"; option java_multiple_files = true; -option (udpa.annotations.file_status).package_version_status = FROZEN; - -// [#protodoc-title: Tracing] -// Tracing :ref:`architecture overview `. - -// The tracing configuration specifies settings for an HTTP tracer provider used by Envoy. -// -// Envoy may support other tracers in the future, but right now the HTTP tracer is the only one -// supported. -// -// .. attention:: -// -// Use of this message type has been deprecated in favor of direct use of -// :ref:`Tracing.Http `. -message Tracing { - // Configuration for an HTTP tracer provider used by Envoy. - // - // The configuration is defined by the - // :ref:`HttpConnectionManager.Tracing ` - // :ref:`provider ` - // field. - message Http { - // The name of the HTTP trace driver to instantiate. The name must match a - // supported HTTP trace driver. Built-in trace drivers: - // - // - *envoy.tracers.lightstep* - // - *envoy.tracers.zipkin* - // - *envoy.tracers.dynamic_ot* - // - *envoy.tracers.datadog* - // - *envoy.tracers.opencensus* - // - *envoy.tracers.xray* - string name = 1 [(validate.rules).string = {min_bytes: 1}]; - - // Trace driver specific configuration which depends on the driver being instantiated. - // See the trace drivers for examples: - // - // - :ref:`LightstepConfig ` - // - :ref:`ZipkinConfig ` - // - :ref:`DynamicOtConfig ` - // - :ref:`DatadogConfig ` - // - :ref:`OpenCensusConfig ` - // - :ref:`AWS X-Ray ` - oneof config_type { - google.protobuf.Struct config = 2 [deprecated = true]; - - google.protobuf.Any typed_config = 3; - } - } - - // Provides configuration for the HTTP tracer. - Http http = 1; -} - -// Configuration for the LightStep tracer. -// [#extension: envoy.tracers.lightstep] -message LightstepConfig { - // Available propagation modes - enum PropagationMode { - // Propagate trace context in the single header x-ot-span-context. - ENVOY = 0; - - // Propagate trace context using LightStep's native format. - LIGHTSTEP = 1; - - // Propagate trace context using the b3 format. - B3 = 2; - - // Propagation trace context using the w3 trace-context standard. - TRACE_CONTEXT = 3; - } - - // The cluster manager cluster that hosts the LightStep collectors. - string collector_cluster = 1 [(validate.rules).string = {min_bytes: 1}]; - - // File containing the access token to the `LightStep - // `_ API. - string access_token_file = 2 [(validate.rules).string = {min_bytes: 1}]; - - // Propagation modes to use by LightStep's tracer. - repeated PropagationMode propagation_modes = 3 - [(validate.rules).repeated = {items {enum {defined_only: true}}}]; -} - -// Configuration for the Zipkin tracer. -// [#extension: envoy.tracers.zipkin] -// [#next-free-field: 6] -message ZipkinConfig { - // Available Zipkin collector endpoint versions. - enum CollectorEndpointVersion { - // Zipkin API v1, JSON over HTTP. - // [#comment: The default implementation of Zipkin client before this field is added was only v1 - // and the way user configure this was by not explicitly specifying the version. Consequently, - // before this is added, the corresponding Zipkin collector expected to receive v1 payload. - // Hence the motivation of adding HTTP_JSON_V1 as the default is to avoid a breaking change when - // user upgrading Envoy with this change. Furthermore, we also immediately deprecate this field, - // since in Zipkin realm this v1 version is considered to be not preferable anymore.] - HTTP_JSON_V1 = 0 [deprecated = true, (envoy.annotations.disallowed_by_default_enum) = true]; - - // Zipkin API v2, JSON over HTTP. - HTTP_JSON = 1; - - // Zipkin API v2, protobuf over HTTP. - HTTP_PROTO = 2; - - // [#not-implemented-hide:] - GRPC = 3; - } - - // The cluster manager cluster that hosts the Zipkin collectors. Note that the - // Zipkin cluster must be defined in the :ref:`Bootstrap static cluster - // resources `. - string collector_cluster = 1 [(validate.rules).string = {min_bytes: 1}]; - - // The API endpoint of the Zipkin service where the spans will be sent. When - // using a standard Zipkin installation, the API endpoint is typically - // /api/v1/spans, which is the default value. - string collector_endpoint = 2 [(validate.rules).string = {min_bytes: 1}]; - - // Determines whether a 128bit trace id will be used when creating a new - // trace instance. The default value is false, which will result in a 64 bit trace id being used. - bool trace_id_128bit = 3; - - // Determines whether client and server spans will share the same span context. - // The default value is true. - google.protobuf.BoolValue shared_span_context = 4; - - // Determines the selected collector endpoint version. By default, the ``HTTP_JSON_V1`` will be - // used. - CollectorEndpointVersion collector_endpoint_version = 5; -} - -// DynamicOtConfig is used to dynamically load a tracer from a shared library -// that implements the `OpenTracing dynamic loading API -// `_. -// [#extension: envoy.tracers.dynamic_ot] -message DynamicOtConfig { - // Dynamic library implementing the `OpenTracing API - // `_. - string library = 1 [(validate.rules).string = {min_bytes: 1}]; - - // The configuration to use when creating a tracer from the given dynamic - // library. - google.protobuf.Struct config = 2; -} - -// Configuration for the Datadog tracer. -// [#extension: envoy.tracers.datadog] -message DatadogConfig { - // The cluster to use for submitting traces to the Datadog agent. - string collector_cluster = 1 [(validate.rules).string = {min_bytes: 1}]; - - // The name used for the service when traces are generated by envoy. - string service_name = 2 [(validate.rules).string = {min_bytes: 1}]; -} - -// Configuration for the OpenCensus tracer. -// [#next-free-field: 15] -// [#extension: envoy.tracers.opencensus] -message OpenCensusConfig { - enum TraceContext { - // No-op default, no trace context is utilized. - NONE = 0; - - // W3C Trace-Context format "traceparent:" header. - TRACE_CONTEXT = 1; - - // Binary "grpc-trace-bin:" header. - GRPC_TRACE_BIN = 2; - - // "X-Cloud-Trace-Context:" header. - CLOUD_TRACE_CONTEXT = 3; - - // X-B3-* headers. - B3 = 4; - } - - reserved 7; - - // Configures tracing, e.g. the sampler, max number of annotations, etc. - opencensus.proto.trace.v1.TraceConfig trace_config = 1; - - // Enables the stdout exporter if set to true. This is intended for debugging - // purposes. - bool stdout_exporter_enabled = 2; - - // Enables the Stackdriver exporter if set to true. The project_id must also - // be set. - bool stackdriver_exporter_enabled = 3; - - // The Cloud project_id to use for Stackdriver tracing. - string stackdriver_project_id = 4; - - // (optional) By default, the Stackdriver exporter will connect to production - // Stackdriver. If stackdriver_address is non-empty, it will instead connect - // to this address, which is in the gRPC format: - // https://github.com/grpc/grpc/blob/master/doc/naming.md - string stackdriver_address = 10; - - // (optional) The gRPC server that hosts Stackdriver tracing service. Only - // Google gRPC is supported. If :ref:`target_uri ` - // is not provided, the default production Stackdriver address will be used. - api.v2.core.GrpcService stackdriver_grpc_service = 13; - - // Enables the Zipkin exporter if set to true. The url and service name must - // also be set. - bool zipkin_exporter_enabled = 5; - - // The URL to Zipkin, e.g. "http://127.0.0.1:9411/api/v2/spans" - string zipkin_url = 6; - - // Enables the OpenCensus Agent exporter if set to true. The ocagent_address or - // ocagent_grpc_service must also be set. - bool ocagent_exporter_enabled = 11; - - // The address of the OpenCensus Agent, if its exporter is enabled, in gRPC - // format: https://github.com/grpc/grpc/blob/master/doc/naming.md - // [#comment:TODO: deprecate this field] - string ocagent_address = 12; - - // (optional) The gRPC server hosted by the OpenCensus Agent. Only Google gRPC is supported. - // This is only used if the ocagent_address is left empty. - api.v2.core.GrpcService ocagent_grpc_service = 14; - - // List of incoming trace context headers we will accept. First one found - // wins. - repeated TraceContext incoming_trace_context = 8; - - // List of outgoing trace context headers we will produce. - repeated TraceContext outgoing_trace_context = 9; -} - -// Configuration structure. -message TraceServiceConfig { - // The upstream gRPC cluster that hosts the metrics service. - api.v2.core.GrpcService grpc_service = 1 [(validate.rules).message = {required: true}]; -} diff --git a/generated_api_shadow/envoy/config/trace/v2/zipkin.proto b/generated_api_shadow/envoy/config/trace/v2/zipkin.proto new file mode 100644 index 000000000000..a825d85bb7f9 --- /dev/null +++ b/generated_api_shadow/envoy/config/trace/v2/zipkin.proto @@ -0,0 +1,64 @@ +syntax = "proto3"; + +package envoy.config.trace.v2; + +import "google/protobuf/wrappers.proto"; + +import "envoy/annotations/deprecation.proto"; +import "udpa/annotations/status.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.trace.v2"; +option java_outer_classname = "ZipkinProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = FROZEN; + +// [#protodoc-title: Zipkin tracer] + +// Configuration for the Zipkin tracer. +// [#extension: envoy.tracers.zipkin] +// [#next-free-field: 6] +message ZipkinConfig { + // Available Zipkin collector endpoint versions. + enum CollectorEndpointVersion { + // Zipkin API v1, JSON over HTTP. + // [#comment: The default implementation of Zipkin client before this field is added was only v1 + // and the way user configure this was by not explicitly specifying the version. Consequently, + // before this is added, the corresponding Zipkin collector expected to receive v1 payload. + // Hence the motivation of adding HTTP_JSON_V1 as the default is to avoid a breaking change when + // user upgrading Envoy with this change. Furthermore, we also immediately deprecate this field, + // since in Zipkin realm this v1 version is considered to be not preferable anymore.] + HTTP_JSON_V1 = 0 [deprecated = true, (envoy.annotations.disallowed_by_default_enum) = true]; + + // Zipkin API v2, JSON over HTTP. + HTTP_JSON = 1; + + // Zipkin API v2, protobuf over HTTP. + HTTP_PROTO = 2; + + // [#not-implemented-hide:] + GRPC = 3; + } + + // The cluster manager cluster that hosts the Zipkin collectors. Note that the + // Zipkin cluster must be defined in the :ref:`Bootstrap static cluster + // resources `. + string collector_cluster = 1 [(validate.rules).string = {min_bytes: 1}]; + + // The API endpoint of the Zipkin service where the spans will be sent. When + // using a standard Zipkin installation, the API endpoint is typically + // /api/v1/spans, which is the default value. + string collector_endpoint = 2 [(validate.rules).string = {min_bytes: 1}]; + + // Determines whether a 128bit trace id will be used when creating a new + // trace instance. The default value is false, which will result in a 64 bit trace id being used. + bool trace_id_128bit = 3; + + // Determines whether client and server spans will share the same span context. + // The default value is true. + google.protobuf.BoolValue shared_span_context = 4; + + // Determines the selected collector endpoint version. By default, the ``HTTP_JSON_V1`` will be + // used. + CollectorEndpointVersion collector_endpoint_version = 5; +} diff --git a/generated_api_shadow/envoy/config/trace/v3/datadog.proto b/generated_api_shadow/envoy/config/trace/v3/datadog.proto new file mode 100644 index 000000000000..f1fe3e666125 --- /dev/null +++ b/generated_api_shadow/envoy/config/trace/v3/datadog.proto @@ -0,0 +1,29 @@ +syntax = "proto3"; + +package envoy.config.trace.v3; + +import "udpa/annotations/migrate.proto"; +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.trace.v3"; +option java_outer_classname = "DatadogProto"; +option java_multiple_files = true; +option (udpa.annotations.file_migrate).move_to_package = "envoy.extensions.tracers.datadog.v4alpha"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Datadog tracer] + +// Configuration for the Datadog tracer. +// [#extension: envoy.tracers.datadog] +message DatadogConfig { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.trace.v2.DatadogConfig"; + + // The cluster to use for submitting traces to the Datadog agent. + string collector_cluster = 1 [(validate.rules).string = {min_bytes: 1}]; + + // The name used for the service when traces are generated by envoy. + string service_name = 2 [(validate.rules).string = {min_bytes: 1}]; +} diff --git a/generated_api_shadow/envoy/config/trace/v3/dynamic_ot.proto b/generated_api_shadow/envoy/config/trace/v3/dynamic_ot.proto new file mode 100644 index 000000000000..fb372da8c52a --- /dev/null +++ b/generated_api_shadow/envoy/config/trace/v3/dynamic_ot.proto @@ -0,0 +1,36 @@ +syntax = "proto3"; + +package envoy.config.trace.v3; + +import "google/protobuf/struct.proto"; + +import "udpa/annotations/migrate.proto"; +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.trace.v3"; +option java_outer_classname = "DynamicOtProto"; +option java_multiple_files = true; +option (udpa.annotations.file_migrate).move_to_package = + "envoy.extensions.tracers.dynamic_ot.v4alpha"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Dynamically loadable OpenTracing tracer] + +// DynamicOtConfig is used to dynamically load a tracer from a shared library +// that implements the `OpenTracing dynamic loading API +// `_. +// [#extension: envoy.tracers.dynamic_ot] +message DynamicOtConfig { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.trace.v2.DynamicOtConfig"; + + // Dynamic library implementing the `OpenTracing API + // `_. + string library = 1 [(validate.rules).string = {min_bytes: 1}]; + + // The configuration to use when creating a tracer from the given dynamic + // library. + google.protobuf.Struct config = 2; +} diff --git a/generated_api_shadow/envoy/config/trace/v3/http_tracer.proto b/generated_api_shadow/envoy/config/trace/v3/http_tracer.proto new file mode 100644 index 000000000000..6470a70de43d --- /dev/null +++ b/generated_api_shadow/envoy/config/trace/v3/http_tracer.proto @@ -0,0 +1,71 @@ +syntax = "proto3"; + +package envoy.config.trace.v3; + +import "google/protobuf/any.proto"; +import "google/protobuf/struct.proto"; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.trace.v3"; +option java_outer_classname = "HttpTracerProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Tracing] +// Tracing :ref:`architecture overview `. + +// The tracing configuration specifies settings for an HTTP tracer provider used by Envoy. +// +// Envoy may support other tracers in the future, but right now the HTTP tracer is the only one +// supported. +// +// .. attention:: +// +// Use of this message type has been deprecated in favor of direct use of +// :ref:`Tracing.Http `. +message Tracing { + option (udpa.annotations.versioning).previous_message_type = "envoy.config.trace.v2.Tracing"; + + // Configuration for an HTTP tracer provider used by Envoy. + // + // The configuration is defined by the + // :ref:`HttpConnectionManager.Tracing ` + // :ref:`provider ` + // field. + message Http { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.trace.v2.Tracing.Http"; + + // The name of the HTTP trace driver to instantiate. The name must match a + // supported HTTP trace driver. Built-in trace drivers: + // + // - *envoy.tracers.lightstep* + // - *envoy.tracers.zipkin* + // - *envoy.tracers.dynamic_ot* + // - *envoy.tracers.datadog* + // - *envoy.tracers.opencensus* + // - *envoy.tracers.xray* + string name = 1 [(validate.rules).string = {min_bytes: 1}]; + + // Trace driver specific configuration which depends on the driver being instantiated. + // See the trace drivers for examples: + // + // - :ref:`LightstepConfig ` + // - :ref:`ZipkinConfig ` + // - :ref:`DynamicOtConfig ` + // - :ref:`DatadogConfig ` + // - :ref:`OpenCensusConfig ` + // - :ref:`AWS X-Ray ` + oneof config_type { + google.protobuf.Any typed_config = 3; + + google.protobuf.Struct hidden_envoy_deprecated_config = 2 [deprecated = true]; + } + } + + // Provides configuration for the HTTP tracer. + Http http = 1; +} diff --git a/generated_api_shadow/envoy/config/trace/v3/lightstep.proto b/generated_api_shadow/envoy/config/trace/v3/lightstep.proto new file mode 100644 index 000000000000..0e0b60b5bddb --- /dev/null +++ b/generated_api_shadow/envoy/config/trace/v3/lightstep.proto @@ -0,0 +1,50 @@ +syntax = "proto3"; + +package envoy.config.trace.v3; + +import "udpa/annotations/migrate.proto"; +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.trace.v3"; +option java_outer_classname = "LightstepProto"; +option java_multiple_files = true; +option (udpa.annotations.file_migrate).move_to_package = + "envoy.extensions.tracers.lightstep.v4alpha"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: LightStep tracer] + +// Configuration for the LightStep tracer. +// [#extension: envoy.tracers.lightstep] +message LightstepConfig { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.trace.v2.LightstepConfig"; + + // Available propagation modes + enum PropagationMode { + // Propagate trace context in the single header x-ot-span-context. + ENVOY = 0; + + // Propagate trace context using LightStep's native format. + LIGHTSTEP = 1; + + // Propagate trace context using the b3 format. + B3 = 2; + + // Propagation trace context using the w3 trace-context standard. + TRACE_CONTEXT = 3; + } + + // The cluster manager cluster that hosts the LightStep collectors. + string collector_cluster = 1 [(validate.rules).string = {min_bytes: 1}]; + + // File containing the access token to the `LightStep + // `_ API. + string access_token_file = 2 [(validate.rules).string = {min_bytes: 1}]; + + // Propagation modes to use by LightStep's tracer. + repeated PropagationMode propagation_modes = 3 + [(validate.rules).repeated = {items {enum {defined_only: true}}}]; +} diff --git a/generated_api_shadow/envoy/config/trace/v3/opencensus.proto b/generated_api_shadow/envoy/config/trace/v3/opencensus.proto new file mode 100644 index 000000000000..39313139177f --- /dev/null +++ b/generated_api_shadow/envoy/config/trace/v3/opencensus.proto @@ -0,0 +1,100 @@ +syntax = "proto3"; + +package envoy.config.trace.v3; + +import "envoy/config/core/v3/grpc_service.proto"; + +import "opencensus/proto/trace/v1/trace_config.proto"; + +import "udpa/annotations/migrate.proto"; +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.trace.v3"; +option java_outer_classname = "OpencensusProto"; +option java_multiple_files = true; +option (udpa.annotations.file_migrate).move_to_package = + "envoy.extensions.tracers.opencensus.v4alpha"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: OpenCensus tracer] + +// Configuration for the OpenCensus tracer. +// [#next-free-field: 15] +// [#extension: envoy.tracers.opencensus] +message OpenCensusConfig { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.trace.v2.OpenCensusConfig"; + + enum TraceContext { + // No-op default, no trace context is utilized. + NONE = 0; + + // W3C Trace-Context format "traceparent:" header. + TRACE_CONTEXT = 1; + + // Binary "grpc-trace-bin:" header. + GRPC_TRACE_BIN = 2; + + // "X-Cloud-Trace-Context:" header. + CLOUD_TRACE_CONTEXT = 3; + + // X-B3-* headers. + B3 = 4; + } + + reserved 7; + + // Configures tracing, e.g. the sampler, max number of annotations, etc. + opencensus.proto.trace.v1.TraceConfig trace_config = 1; + + // Enables the stdout exporter if set to true. This is intended for debugging + // purposes. + bool stdout_exporter_enabled = 2; + + // Enables the Stackdriver exporter if set to true. The project_id must also + // be set. + bool stackdriver_exporter_enabled = 3; + + // The Cloud project_id to use for Stackdriver tracing. + string stackdriver_project_id = 4; + + // (optional) By default, the Stackdriver exporter will connect to production + // Stackdriver. If stackdriver_address is non-empty, it will instead connect + // to this address, which is in the gRPC format: + // https://github.com/grpc/grpc/blob/master/doc/naming.md + string stackdriver_address = 10; + + // (optional) The gRPC server that hosts Stackdriver tracing service. Only + // Google gRPC is supported. If :ref:`target_uri ` + // is not provided, the default production Stackdriver address will be used. + core.v3.GrpcService stackdriver_grpc_service = 13; + + // Enables the Zipkin exporter if set to true. The url and service name must + // also be set. + bool zipkin_exporter_enabled = 5; + + // The URL to Zipkin, e.g. "http://127.0.0.1:9411/api/v2/spans" + string zipkin_url = 6; + + // Enables the OpenCensus Agent exporter if set to true. The ocagent_address or + // ocagent_grpc_service must also be set. + bool ocagent_exporter_enabled = 11; + + // The address of the OpenCensus Agent, if its exporter is enabled, in gRPC + // format: https://github.com/grpc/grpc/blob/master/doc/naming.md + // [#comment:TODO: deprecate this field] + string ocagent_address = 12; + + // (optional) The gRPC server hosted by the OpenCensus Agent. Only Google gRPC is supported. + // This is only used if the ocagent_address is left empty. + core.v3.GrpcService ocagent_grpc_service = 14; + + // List of incoming trace context headers we will accept. First one found + // wins. + repeated TraceContext incoming_trace_context = 8; + + // List of outgoing trace context headers we will produce. + repeated TraceContext outgoing_trace_context = 9; +} diff --git a/generated_api_shadow/envoy/config/trace/v3/service.proto b/generated_api_shadow/envoy/config/trace/v3/service.proto new file mode 100644 index 000000000000..1e01ff61847f --- /dev/null +++ b/generated_api_shadow/envoy/config/trace/v3/service.proto @@ -0,0 +1,25 @@ +syntax = "proto3"; + +package envoy.config.trace.v3; + +import "envoy/config/core/v3/grpc_service.proto"; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.trace.v3"; +option java_outer_classname = "ServiceProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Trace Service] + +// Configuration structure. +message TraceServiceConfig { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.trace.v2.TraceServiceConfig"; + + // The upstream gRPC cluster that hosts the metrics service. + core.v3.GrpcService grpc_service = 1 [(validate.rules).message = {required: true}]; +} diff --git a/generated_api_shadow/envoy/config/trace/v3/trace.proto b/generated_api_shadow/envoy/config/trace/v3/trace.proto index 955f1949adff..e1db72a2fd5a 100644 --- a/generated_api_shadow/envoy/config/trace/v3/trace.proto +++ b/generated_api_shadow/envoy/config/trace/v3/trace.proto @@ -2,278 +2,16 @@ syntax = "proto3"; package envoy.config.trace.v3; -import "envoy/config/core/v3/grpc_service.proto"; - -import "google/protobuf/any.proto"; -import "google/protobuf/struct.proto"; -import "google/protobuf/wrappers.proto"; - -import "opencensus/proto/trace/v1/trace_config.proto"; - -import "envoy/annotations/deprecation.proto"; import "udpa/annotations/status.proto"; -import "udpa/annotations/versioning.proto"; -import "validate/validate.proto"; + +import public "envoy/config/trace/v3/datadog.proto"; +import public "envoy/config/trace/v3/dynamic_ot.proto"; +import public "envoy/config/trace/v3/http_tracer.proto"; +import public "envoy/config/trace/v3/lightstep.proto"; +import public "envoy/config/trace/v3/opencensus.proto"; +import public "envoy/config/trace/v3/service.proto"; +import public "envoy/config/trace/v3/zipkin.proto"; option java_package = "io.envoyproxy.envoy.config.trace.v3"; option java_outer_classname = "TraceProto"; option java_multiple_files = true; -option (udpa.annotations.file_status).package_version_status = ACTIVE; - -// [#protodoc-title: Tracing] -// Tracing :ref:`architecture overview `. - -// The tracing configuration specifies settings for an HTTP tracer provider used by Envoy. -// -// Envoy may support other tracers in the future, but right now the HTTP tracer is the only one -// supported. -// -// .. attention:: -// -// Use of this message type has been deprecated in favor of direct use of -// :ref:`Tracing.Http `. -message Tracing { - option (udpa.annotations.versioning).previous_message_type = "envoy.config.trace.v2.Tracing"; - - // Configuration for an HTTP tracer provider used by Envoy. - // - // The configuration is defined by the - // :ref:`HttpConnectionManager.Tracing ` - // :ref:`provider ` - // field. - message Http { - option (udpa.annotations.versioning).previous_message_type = - "envoy.config.trace.v2.Tracing.Http"; - - // The name of the HTTP trace driver to instantiate. The name must match a - // supported HTTP trace driver. Built-in trace drivers: - // - // - *envoy.tracers.lightstep* - // - *envoy.tracers.zipkin* - // - *envoy.tracers.dynamic_ot* - // - *envoy.tracers.datadog* - // - *envoy.tracers.opencensus* - // - *envoy.tracers.xray* - string name = 1 [(validate.rules).string = {min_bytes: 1}]; - - // Trace driver specific configuration which depends on the driver being instantiated. - // See the trace drivers for examples: - // - // - :ref:`LightstepConfig ` - // - :ref:`ZipkinConfig ` - // - :ref:`DynamicOtConfig ` - // - :ref:`DatadogConfig ` - // - :ref:`OpenCensusConfig ` - // - :ref:`AWS X-Ray ` - oneof config_type { - google.protobuf.Any typed_config = 3; - - google.protobuf.Struct hidden_envoy_deprecated_config = 2 [deprecated = true]; - } - } - - // Provides configuration for the HTTP tracer. - Http http = 1; -} - -// Configuration for the LightStep tracer. -// [#extension: envoy.tracers.lightstep] -message LightstepConfig { - option (udpa.annotations.versioning).previous_message_type = - "envoy.config.trace.v2.LightstepConfig"; - - // Available propagation modes - enum PropagationMode { - // Propagate trace context in the single header x-ot-span-context. - ENVOY = 0; - - // Propagate trace context using LightStep's native format. - LIGHTSTEP = 1; - - // Propagate trace context using the b3 format. - B3 = 2; - - // Propagation trace context using the w3 trace-context standard. - TRACE_CONTEXT = 3; - } - - // The cluster manager cluster that hosts the LightStep collectors. - string collector_cluster = 1 [(validate.rules).string = {min_bytes: 1}]; - - // File containing the access token to the `LightStep - // `_ API. - string access_token_file = 2 [(validate.rules).string = {min_bytes: 1}]; - - // Propagation modes to use by LightStep's tracer. - repeated PropagationMode propagation_modes = 3 - [(validate.rules).repeated = {items {enum {defined_only: true}}}]; -} - -// Configuration for the Zipkin tracer. -// [#extension: envoy.tracers.zipkin] -// [#next-free-field: 6] -message ZipkinConfig { - option (udpa.annotations.versioning).previous_message_type = "envoy.config.trace.v2.ZipkinConfig"; - - // Available Zipkin collector endpoint versions. - enum CollectorEndpointVersion { - // Zipkin API v1, JSON over HTTP. - // [#comment: The default implementation of Zipkin client before this field is added was only v1 - // and the way user configure this was by not explicitly specifying the version. Consequently, - // before this is added, the corresponding Zipkin collector expected to receive v1 payload. - // Hence the motivation of adding HTTP_JSON_V1 as the default is to avoid a breaking change when - // user upgrading Envoy with this change. Furthermore, we also immediately deprecate this field, - // since in Zipkin realm this v1 version is considered to be not preferable anymore.] - hidden_envoy_deprecated_HTTP_JSON_V1 = 0 - [deprecated = true, (envoy.annotations.disallowed_by_default_enum) = true]; - - // Zipkin API v2, JSON over HTTP. - HTTP_JSON = 1; - - // Zipkin API v2, protobuf over HTTP. - HTTP_PROTO = 2; - - // [#not-implemented-hide:] - GRPC = 3; - } - - // The cluster manager cluster that hosts the Zipkin collectors. Note that the - // Zipkin cluster must be defined in the :ref:`Bootstrap static cluster - // resources `. - string collector_cluster = 1 [(validate.rules).string = {min_bytes: 1}]; - - // The API endpoint of the Zipkin service where the spans will be sent. When - // using a standard Zipkin installation, the API endpoint is typically - // /api/v1/spans, which is the default value. - string collector_endpoint = 2 [(validate.rules).string = {min_bytes: 1}]; - - // Determines whether a 128bit trace id will be used when creating a new - // trace instance. The default value is false, which will result in a 64 bit trace id being used. - bool trace_id_128bit = 3; - - // Determines whether client and server spans will share the same span context. - // The default value is true. - google.protobuf.BoolValue shared_span_context = 4; - - // Determines the selected collector endpoint version. By default, the ``HTTP_JSON_V1`` will be - // used. - CollectorEndpointVersion collector_endpoint_version = 5; -} - -// DynamicOtConfig is used to dynamically load a tracer from a shared library -// that implements the `OpenTracing dynamic loading API -// `_. -// [#extension: envoy.tracers.dynamic_ot] -message DynamicOtConfig { - option (udpa.annotations.versioning).previous_message_type = - "envoy.config.trace.v2.DynamicOtConfig"; - - // Dynamic library implementing the `OpenTracing API - // `_. - string library = 1 [(validate.rules).string = {min_bytes: 1}]; - - // The configuration to use when creating a tracer from the given dynamic - // library. - google.protobuf.Struct config = 2; -} - -// Configuration for the Datadog tracer. -// [#extension: envoy.tracers.datadog] -message DatadogConfig { - option (udpa.annotations.versioning).previous_message_type = - "envoy.config.trace.v2.DatadogConfig"; - - // The cluster to use for submitting traces to the Datadog agent. - string collector_cluster = 1 [(validate.rules).string = {min_bytes: 1}]; - - // The name used for the service when traces are generated by envoy. - string service_name = 2 [(validate.rules).string = {min_bytes: 1}]; -} - -// Configuration for the OpenCensus tracer. -// [#next-free-field: 15] -// [#extension: envoy.tracers.opencensus] -message OpenCensusConfig { - option (udpa.annotations.versioning).previous_message_type = - "envoy.config.trace.v2.OpenCensusConfig"; - - enum TraceContext { - // No-op default, no trace context is utilized. - NONE = 0; - - // W3C Trace-Context format "traceparent:" header. - TRACE_CONTEXT = 1; - - // Binary "grpc-trace-bin:" header. - GRPC_TRACE_BIN = 2; - - // "X-Cloud-Trace-Context:" header. - CLOUD_TRACE_CONTEXT = 3; - - // X-B3-* headers. - B3 = 4; - } - - reserved 7; - - // Configures tracing, e.g. the sampler, max number of annotations, etc. - opencensus.proto.trace.v1.TraceConfig trace_config = 1; - - // Enables the stdout exporter if set to true. This is intended for debugging - // purposes. - bool stdout_exporter_enabled = 2; - - // Enables the Stackdriver exporter if set to true. The project_id must also - // be set. - bool stackdriver_exporter_enabled = 3; - - // The Cloud project_id to use for Stackdriver tracing. - string stackdriver_project_id = 4; - - // (optional) By default, the Stackdriver exporter will connect to production - // Stackdriver. If stackdriver_address is non-empty, it will instead connect - // to this address, which is in the gRPC format: - // https://github.com/grpc/grpc/blob/master/doc/naming.md - string stackdriver_address = 10; - - // (optional) The gRPC server that hosts Stackdriver tracing service. Only - // Google gRPC is supported. If :ref:`target_uri ` - // is not provided, the default production Stackdriver address will be used. - core.v3.GrpcService stackdriver_grpc_service = 13; - - // Enables the Zipkin exporter if set to true. The url and service name must - // also be set. - bool zipkin_exporter_enabled = 5; - - // The URL to Zipkin, e.g. "http://127.0.0.1:9411/api/v2/spans" - string zipkin_url = 6; - - // Enables the OpenCensus Agent exporter if set to true. The ocagent_address or - // ocagent_grpc_service must also be set. - bool ocagent_exporter_enabled = 11; - - // The address of the OpenCensus Agent, if its exporter is enabled, in gRPC - // format: https://github.com/grpc/grpc/blob/master/doc/naming.md - // [#comment:TODO: deprecate this field] - string ocagent_address = 12; - - // (optional) The gRPC server hosted by the OpenCensus Agent. Only Google gRPC is supported. - // This is only used if the ocagent_address is left empty. - core.v3.GrpcService ocagent_grpc_service = 14; - - // List of incoming trace context headers we will accept. First one found - // wins. - repeated TraceContext incoming_trace_context = 8; - - // List of outgoing trace context headers we will produce. - repeated TraceContext outgoing_trace_context = 9; -} - -// Configuration structure. -message TraceServiceConfig { - option (udpa.annotations.versioning).previous_message_type = - "envoy.config.trace.v2.TraceServiceConfig"; - - // The upstream gRPC cluster that hosts the metrics service. - core.v3.GrpcService grpc_service = 1 [(validate.rules).message = {required: true}]; -} diff --git a/generated_api_shadow/envoy/config/trace/v3/xray.proto b/generated_api_shadow/envoy/config/trace/v3/xray.proto index c4259177d657..ba3fa66ad3e2 100644 --- a/generated_api_shadow/envoy/config/trace/v3/xray.proto +++ b/generated_api_shadow/envoy/config/trace/v3/xray.proto @@ -5,6 +5,7 @@ package envoy.config.trace.v3; import "envoy/config/core/v3/address.proto"; import "envoy/config/core/v3/base.proto"; +import "udpa/annotations/migrate.proto"; import "udpa/annotations/status.proto"; import "udpa/annotations/versioning.proto"; import "validate/validate.proto"; @@ -12,6 +13,7 @@ import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.config.trace.v3"; option java_outer_classname = "XrayProto"; option java_multiple_files = true; +option (udpa.annotations.file_migrate).move_to_package = "envoy.extensions.tracers.xray.v4alpha"; option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: AWS X-Ray Tracer Configuration] diff --git a/generated_api_shadow/envoy/config/trace/v3/zipkin.proto b/generated_api_shadow/envoy/config/trace/v3/zipkin.proto new file mode 100644 index 000000000000..10cf6d4ec3c4 --- /dev/null +++ b/generated_api_shadow/envoy/config/trace/v3/zipkin.proto @@ -0,0 +1,70 @@ +syntax = "proto3"; + +package envoy.config.trace.v3; + +import "google/protobuf/wrappers.proto"; + +import "envoy/annotations/deprecation.proto"; +import "udpa/annotations/migrate.proto"; +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.trace.v3"; +option java_outer_classname = "ZipkinProto"; +option java_multiple_files = true; +option (udpa.annotations.file_migrate).move_to_package = "envoy.extensions.tracers.zipkin.v4alpha"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Zipkin tracer] + +// Configuration for the Zipkin tracer. +// [#extension: envoy.tracers.zipkin] +// [#next-free-field: 6] +message ZipkinConfig { + option (udpa.annotations.versioning).previous_message_type = "envoy.config.trace.v2.ZipkinConfig"; + + // Available Zipkin collector endpoint versions. + enum CollectorEndpointVersion { + // Zipkin API v1, JSON over HTTP. + // [#comment: The default implementation of Zipkin client before this field is added was only v1 + // and the way user configure this was by not explicitly specifying the version. Consequently, + // before this is added, the corresponding Zipkin collector expected to receive v1 payload. + // Hence the motivation of adding HTTP_JSON_V1 as the default is to avoid a breaking change when + // user upgrading Envoy with this change. Furthermore, we also immediately deprecate this field, + // since in Zipkin realm this v1 version is considered to be not preferable anymore.] + hidden_envoy_deprecated_HTTP_JSON_V1 = 0 + [deprecated = true, (envoy.annotations.disallowed_by_default_enum) = true]; + + // Zipkin API v2, JSON over HTTP. + HTTP_JSON = 1; + + // Zipkin API v2, protobuf over HTTP. + HTTP_PROTO = 2; + + // [#not-implemented-hide:] + GRPC = 3; + } + + // The cluster manager cluster that hosts the Zipkin collectors. Note that the + // Zipkin cluster must be defined in the :ref:`Bootstrap static cluster + // resources `. + string collector_cluster = 1 [(validate.rules).string = {min_bytes: 1}]; + + // The API endpoint of the Zipkin service where the spans will be sent. When + // using a standard Zipkin installation, the API endpoint is typically + // /api/v1/spans, which is the default value. + string collector_endpoint = 2 [(validate.rules).string = {min_bytes: 1}]; + + // Determines whether a 128bit trace id will be used when creating a new + // trace instance. The default value is false, which will result in a 64 bit trace id being used. + bool trace_id_128bit = 3; + + // Determines whether client and server spans will share the same span context. + // The default value is true. + google.protobuf.BoolValue shared_span_context = 4; + + // Determines the selected collector endpoint version. By default, the ``HTTP_JSON_V1`` will be + // used. + CollectorEndpointVersion collector_endpoint_version = 5; +} diff --git a/generated_api_shadow/envoy/config/trace/v4alpha/BUILD b/generated_api_shadow/envoy/config/trace/v4alpha/BUILD index 53ae98aac140..d8ce683c41d6 100644 --- a/generated_api_shadow/envoy/config/trace/v4alpha/BUILD +++ b/generated_api_shadow/envoy/config/trace/v4alpha/BUILD @@ -6,10 +6,8 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ - "//envoy/annotations:pkg", "//envoy/config/core/v4alpha:pkg", "//envoy/config/trace/v3:pkg", "@com_github_cncf_udpa//udpa/annotations:pkg", - "@opencensus_proto//opencensus/proto/trace/v1:trace_config_proto", ], ) diff --git a/generated_api_shadow/envoy/config/trace/v4alpha/http_tracer.proto b/generated_api_shadow/envoy/config/trace/v4alpha/http_tracer.proto new file mode 100644 index 000000000000..663886a97bb4 --- /dev/null +++ b/generated_api_shadow/envoy/config/trace/v4alpha/http_tracer.proto @@ -0,0 +1,73 @@ +syntax = "proto3"; + +package envoy.config.trace.v4alpha; + +import "google/protobuf/any.proto"; +import "google/protobuf/struct.proto"; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.trace.v4alpha"; +option java_outer_classname = "HttpTracerProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; + +// [#protodoc-title: Tracing] +// Tracing :ref:`architecture overview `. + +// The tracing configuration specifies settings for an HTTP tracer provider used by Envoy. +// +// Envoy may support other tracers in the future, but right now the HTTP tracer is the only one +// supported. +// +// .. attention:: +// +// Use of this message type has been deprecated in favor of direct use of +// :ref:`Tracing.Http `. +message Tracing { + option (udpa.annotations.versioning).previous_message_type = "envoy.config.trace.v3.Tracing"; + + // Configuration for an HTTP tracer provider used by Envoy. + // + // The configuration is defined by the + // :ref:`HttpConnectionManager.Tracing ` + // :ref:`provider ` + // field. + message Http { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.trace.v3.Tracing.Http"; + + reserved 2; + + reserved "config"; + + // The name of the HTTP trace driver to instantiate. The name must match a + // supported HTTP trace driver. Built-in trace drivers: + // + // - *envoy.tracers.lightstep* + // - *envoy.tracers.zipkin* + // - *envoy.tracers.dynamic_ot* + // - *envoy.tracers.datadog* + // - *envoy.tracers.opencensus* + // - *envoy.tracers.xray* + string name = 1 [(validate.rules).string = {min_bytes: 1}]; + + // Trace driver specific configuration which depends on the driver being instantiated. + // See the trace drivers for examples: + // + // - :ref:`LightstepConfig ` + // - :ref:`ZipkinConfig ` + // - :ref:`DynamicOtConfig ` + // - :ref:`DatadogConfig ` + // - :ref:`OpenCensusConfig ` + // - :ref:`AWS X-Ray ` + oneof config_type { + google.protobuf.Any typed_config = 3; + } + } + + // Provides configuration for the HTTP tracer. + Http http = 1; +} diff --git a/generated_api_shadow/envoy/config/trace/v4alpha/service.proto b/generated_api_shadow/envoy/config/trace/v4alpha/service.proto new file mode 100644 index 000000000000..d132b32dd79d --- /dev/null +++ b/generated_api_shadow/envoy/config/trace/v4alpha/service.proto @@ -0,0 +1,25 @@ +syntax = "proto3"; + +package envoy.config.trace.v4alpha; + +import "envoy/config/core/v4alpha/grpc_service.proto"; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.trace.v4alpha"; +option java_outer_classname = "ServiceProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; + +// [#protodoc-title: Trace Service] + +// Configuration structure. +message TraceServiceConfig { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.trace.v3.TraceServiceConfig"; + + // The upstream gRPC cluster that hosts the metrics service. + core.v4alpha.GrpcService grpc_service = 1 [(validate.rules).message = {required: true}]; +} diff --git a/generated_api_shadow/envoy/config/trace/v4alpha/trace.proto b/generated_api_shadow/envoy/config/trace/v4alpha/trace.proto deleted file mode 100644 index 310376381ad9..000000000000 --- a/generated_api_shadow/envoy/config/trace/v4alpha/trace.proto +++ /dev/null @@ -1,281 +0,0 @@ -syntax = "proto3"; - -package envoy.config.trace.v4alpha; - -import "envoy/config/core/v4alpha/grpc_service.proto"; - -import "google/protobuf/any.proto"; -import "google/protobuf/struct.proto"; -import "google/protobuf/wrappers.proto"; - -import "opencensus/proto/trace/v1/trace_config.proto"; - -import "envoy/annotations/deprecation.proto"; -import "udpa/annotations/status.proto"; -import "udpa/annotations/versioning.proto"; -import "validate/validate.proto"; - -option java_package = "io.envoyproxy.envoy.config.trace.v4alpha"; -option java_outer_classname = "TraceProto"; -option java_multiple_files = true; -option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; - -// [#protodoc-title: Tracing] -// Tracing :ref:`architecture overview `. - -// The tracing configuration specifies settings for an HTTP tracer provider used by Envoy. -// -// Envoy may support other tracers in the future, but right now the HTTP tracer is the only one -// supported. -// -// .. attention:: -// -// Use of this message type has been deprecated in favor of direct use of -// :ref:`Tracing.Http `. -message Tracing { - option (udpa.annotations.versioning).previous_message_type = "envoy.config.trace.v3.Tracing"; - - // Configuration for an HTTP tracer provider used by Envoy. - // - // The configuration is defined by the - // :ref:`HttpConnectionManager.Tracing ` - // :ref:`provider ` - // field. - message Http { - option (udpa.annotations.versioning).previous_message_type = - "envoy.config.trace.v3.Tracing.Http"; - - reserved 2; - - reserved "config"; - - // The name of the HTTP trace driver to instantiate. The name must match a - // supported HTTP trace driver. Built-in trace drivers: - // - // - *envoy.tracers.lightstep* - // - *envoy.tracers.zipkin* - // - *envoy.tracers.dynamic_ot* - // - *envoy.tracers.datadog* - // - *envoy.tracers.opencensus* - // - *envoy.tracers.xray* - string name = 1 [(validate.rules).string = {min_bytes: 1}]; - - // Trace driver specific configuration which depends on the driver being instantiated. - // See the trace drivers for examples: - // - // - :ref:`LightstepConfig ` - // - :ref:`ZipkinConfig ` - // - :ref:`DynamicOtConfig ` - // - :ref:`DatadogConfig ` - // - :ref:`OpenCensusConfig ` - // - :ref:`AWS X-Ray ` - oneof config_type { - google.protobuf.Any typed_config = 3; - } - } - - // Provides configuration for the HTTP tracer. - Http http = 1; -} - -// Configuration for the LightStep tracer. -// [#extension: envoy.tracers.lightstep] -message LightstepConfig { - option (udpa.annotations.versioning).previous_message_type = - "envoy.config.trace.v3.LightstepConfig"; - - // Available propagation modes - enum PropagationMode { - // Propagate trace context in the single header x-ot-span-context. - ENVOY = 0; - - // Propagate trace context using LightStep's native format. - LIGHTSTEP = 1; - - // Propagate trace context using the b3 format. - B3 = 2; - - // Propagation trace context using the w3 trace-context standard. - TRACE_CONTEXT = 3; - } - - // The cluster manager cluster that hosts the LightStep collectors. - string collector_cluster = 1 [(validate.rules).string = {min_bytes: 1}]; - - // File containing the access token to the `LightStep - // `_ API. - string access_token_file = 2 [(validate.rules).string = {min_bytes: 1}]; - - // Propagation modes to use by LightStep's tracer. - repeated PropagationMode propagation_modes = 3 - [(validate.rules).repeated = {items {enum {defined_only: true}}}]; -} - -// Configuration for the Zipkin tracer. -// [#extension: envoy.tracers.zipkin] -// [#next-free-field: 6] -message ZipkinConfig { - option (udpa.annotations.versioning).previous_message_type = "envoy.config.trace.v3.ZipkinConfig"; - - // Available Zipkin collector endpoint versions. - enum CollectorEndpointVersion { - // Zipkin API v1, JSON over HTTP. - // [#comment: The default implementation of Zipkin client before this field is added was only v1 - // and the way user configure this was by not explicitly specifying the version. Consequently, - // before this is added, the corresponding Zipkin collector expected to receive v1 payload. - // Hence the motivation of adding HTTP_JSON_V1 as the default is to avoid a breaking change when - // user upgrading Envoy with this change. Furthermore, we also immediately deprecate this field, - // since in Zipkin realm this v1 version is considered to be not preferable anymore.] - hidden_envoy_deprecated_DEPRECATED_AND_UNAVAILABLE_DO_NOT_USE = 0 - [deprecated = true, (envoy.annotations.disallowed_by_default_enum) = true]; - - // Zipkin API v2, JSON over HTTP. - HTTP_JSON = 1; - - // Zipkin API v2, protobuf over HTTP. - HTTP_PROTO = 2; - - // [#not-implemented-hide:] - GRPC = 3; - } - - // The cluster manager cluster that hosts the Zipkin collectors. Note that the - // Zipkin cluster must be defined in the :ref:`Bootstrap static cluster - // resources `. - string collector_cluster = 1 [(validate.rules).string = {min_bytes: 1}]; - - // The API endpoint of the Zipkin service where the spans will be sent. When - // using a standard Zipkin installation, the API endpoint is typically - // /api/v1/spans, which is the default value. - string collector_endpoint = 2 [(validate.rules).string = {min_bytes: 1}]; - - // Determines whether a 128bit trace id will be used when creating a new - // trace instance. The default value is false, which will result in a 64 bit trace id being used. - bool trace_id_128bit = 3; - - // Determines whether client and server spans will share the same span context. - // The default value is true. - google.protobuf.BoolValue shared_span_context = 4; - - // Determines the selected collector endpoint version. By default, the ``HTTP_JSON_V1`` will be - // used. - CollectorEndpointVersion collector_endpoint_version = 5; -} - -// DynamicOtConfig is used to dynamically load a tracer from a shared library -// that implements the `OpenTracing dynamic loading API -// `_. -// [#extension: envoy.tracers.dynamic_ot] -message DynamicOtConfig { - option (udpa.annotations.versioning).previous_message_type = - "envoy.config.trace.v3.DynamicOtConfig"; - - // Dynamic library implementing the `OpenTracing API - // `_. - string library = 1 [(validate.rules).string = {min_bytes: 1}]; - - // The configuration to use when creating a tracer from the given dynamic - // library. - google.protobuf.Struct config = 2; -} - -// Configuration for the Datadog tracer. -// [#extension: envoy.tracers.datadog] -message DatadogConfig { - option (udpa.annotations.versioning).previous_message_type = - "envoy.config.trace.v3.DatadogConfig"; - - // The cluster to use for submitting traces to the Datadog agent. - string collector_cluster = 1 [(validate.rules).string = {min_bytes: 1}]; - - // The name used for the service when traces are generated by envoy. - string service_name = 2 [(validate.rules).string = {min_bytes: 1}]; -} - -// Configuration for the OpenCensus tracer. -// [#next-free-field: 15] -// [#extension: envoy.tracers.opencensus] -message OpenCensusConfig { - option (udpa.annotations.versioning).previous_message_type = - "envoy.config.trace.v3.OpenCensusConfig"; - - enum TraceContext { - // No-op default, no trace context is utilized. - NONE = 0; - - // W3C Trace-Context format "traceparent:" header. - TRACE_CONTEXT = 1; - - // Binary "grpc-trace-bin:" header. - GRPC_TRACE_BIN = 2; - - // "X-Cloud-Trace-Context:" header. - CLOUD_TRACE_CONTEXT = 3; - - // X-B3-* headers. - B3 = 4; - } - - reserved 7; - - // Configures tracing, e.g. the sampler, max number of annotations, etc. - opencensus.proto.trace.v1.TraceConfig trace_config = 1; - - // Enables the stdout exporter if set to true. This is intended for debugging - // purposes. - bool stdout_exporter_enabled = 2; - - // Enables the Stackdriver exporter if set to true. The project_id must also - // be set. - bool stackdriver_exporter_enabled = 3; - - // The Cloud project_id to use for Stackdriver tracing. - string stackdriver_project_id = 4; - - // (optional) By default, the Stackdriver exporter will connect to production - // Stackdriver. If stackdriver_address is non-empty, it will instead connect - // to this address, which is in the gRPC format: - // https://github.com/grpc/grpc/blob/master/doc/naming.md - string stackdriver_address = 10; - - // (optional) The gRPC server that hosts Stackdriver tracing service. Only - // Google gRPC is supported. If :ref:`target_uri ` - // is not provided, the default production Stackdriver address will be used. - core.v4alpha.GrpcService stackdriver_grpc_service = 13; - - // Enables the Zipkin exporter if set to true. The url and service name must - // also be set. - bool zipkin_exporter_enabled = 5; - - // The URL to Zipkin, e.g. "http://127.0.0.1:9411/api/v2/spans" - string zipkin_url = 6; - - // Enables the OpenCensus Agent exporter if set to true. The ocagent_address or - // ocagent_grpc_service must also be set. - bool ocagent_exporter_enabled = 11; - - // The address of the OpenCensus Agent, if its exporter is enabled, in gRPC - // format: https://github.com/grpc/grpc/blob/master/doc/naming.md - // [#comment:TODO: deprecate this field] - string ocagent_address = 12; - - // (optional) The gRPC server hosted by the OpenCensus Agent. Only Google gRPC is supported. - // This is only used if the ocagent_address is left empty. - core.v4alpha.GrpcService ocagent_grpc_service = 14; - - // List of incoming trace context headers we will accept. First one found - // wins. - repeated TraceContext incoming_trace_context = 8; - - // List of outgoing trace context headers we will produce. - repeated TraceContext outgoing_trace_context = 9; -} - -// Configuration structure. -message TraceServiceConfig { - option (udpa.annotations.versioning).previous_message_type = - "envoy.config.trace.v3.TraceServiceConfig"; - - // The upstream gRPC cluster that hosts the metrics service. - core.v4alpha.GrpcService grpc_service = 1 [(validate.rules).message = {required: true}]; -} diff --git a/generated_api_shadow/envoy/extensions/filters/http/grpc_stats/v3/config.proto b/generated_api_shadow/envoy/extensions/filters/http/grpc_stats/v3/config.proto index 1fecdaea0a16..d5aca14ea530 100644 --- a/generated_api_shadow/envoy/extensions/filters/http/grpc_stats/v3/config.proto +++ b/generated_api_shadow/envoy/extensions/filters/http/grpc_stats/v3/config.proto @@ -28,12 +28,12 @@ message FilterConfig { // counts. bool emit_filter_state = 1; - oneof per_method_stat_specifier { - // If set, specifies an allowlist of service/methods that will have individual stats - // emitted for them. Any call that does not match the allowlist will be counted - // in a stat with no method specifier: `cluster..grpc.*`. - config.core.v3.GrpcMethodList individual_method_stats_allowlist = 2; + // If set, specifies an allowlist of service/methods that will have individual stats + // emitted for them. Any call that does not match the allowlist will be counted + // in a stat with no method specifier: `cluster..grpc.*`. + bool enable_upstream_stats = 4; + oneof per_method_stat_specifier { // If set to true, emit stats for all service/method names. // // If set to false, emit stats for all service/message types to the same stats without including @@ -52,6 +52,14 @@ message FilterConfig { // `stats_for_all_methods=false` in order to be safe by default. This behavior can be // controlled with runtime override // `envoy.deprecated_features.grpc_stats_filter_enable_stats_for_all_methods_by_default`. + config.core.v3.GrpcMethodList individual_method_stats_allowlist = 2; + + // If true, the filter will gather a histogram for the request time of the upstream. + // It works with :ref:`stats_for_all_methods + // ` + // and :ref:`individual_method_stats_allowlist + // ` the same way + // request_message_count and response_message_count works. google.protobuf.BoolValue stats_for_all_methods = 3; } } diff --git a/generated_api_shadow/envoy/extensions/filters/http/jwt_authn/v3/config.proto b/generated_api_shadow/envoy/extensions/filters/http/jwt_authn/v3/config.proto index 802a582a572a..592610819bdc 100644 --- a/generated_api_shadow/envoy/extensions/filters/http/jwt_authn/v3/config.proto +++ b/generated_api_shadow/envoy/extensions/filters/http/jwt_authn/v3/config.proto @@ -142,7 +142,8 @@ message JwtProvider { // // ``x-goog-iap-jwt-assertion: ``. // - string forward_payload_header = 8; + string forward_payload_header = 8 + [(validate.rules).string = {well_known_regex: HTTP_HEADER_NAME strict: false}]; // JWT is sent in a query parameter. `jwt_params` represents the query parameter names. // @@ -218,12 +219,14 @@ message JwtHeader { "envoy.config.filter.http.jwt_authn.v2alpha.JwtHeader"; // The HTTP header name. - string name = 1 [(validate.rules).string = {min_bytes: 1}]; + string name = 1 + [(validate.rules).string = {min_bytes: 1 well_known_regex: HTTP_HEADER_NAME strict: false}]; // The value prefix. The value format is "value_prefix" // For example, for "Authorization: Bearer ", value_prefix="Bearer " with a space at the // end. - string value_prefix = 2; + string value_prefix = 2 + [(validate.rules).string = {well_known_regex: HTTP_HEADER_VALUE strict: false}]; } // Specify a required provider with audiences. diff --git a/generated_api_shadow/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto b/generated_api_shadow/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto index e22a3a1ee188..b04d0861c953 100644 --- a/generated_api_shadow/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto +++ b/generated_api_shadow/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto @@ -7,7 +7,7 @@ import "envoy/config/core/v3/config_source.proto"; import "envoy/config/core/v3/protocol.proto"; import "envoy/config/route/v3/route.proto"; import "envoy/config/route/v3/scoped_route.proto"; -import "envoy/config/trace/v3/trace.proto"; +import "envoy/config/trace/v3/http_tracer.proto"; import "envoy/type/tracing/v3/custom_tag.proto"; import "envoy/type/v3/percent.proto"; diff --git a/generated_api_shadow/envoy/extensions/filters/network/http_connection_manager/v4alpha/http_connection_manager.proto b/generated_api_shadow/envoy/extensions/filters/network/http_connection_manager/v4alpha/http_connection_manager.proto index 5a05a8531f35..975b71cc892f 100644 --- a/generated_api_shadow/envoy/extensions/filters/network/http_connection_manager/v4alpha/http_connection_manager.proto +++ b/generated_api_shadow/envoy/extensions/filters/network/http_connection_manager/v4alpha/http_connection_manager.proto @@ -7,7 +7,7 @@ import "envoy/config/core/v4alpha/config_source.proto"; import "envoy/config/core/v4alpha/protocol.proto"; import "envoy/config/route/v4alpha/route.proto"; import "envoy/config/route/v4alpha/scoped_route.proto"; -import "envoy/config/trace/v4alpha/trace.proto"; +import "envoy/config/trace/v4alpha/http_tracer.proto"; import "envoy/type/tracing/v3/custom_tag.proto"; import "envoy/type/v3/percent.proto"; diff --git a/generated_api_shadow/envoy/extensions/filters/network/sni_dynamic_forward_proxy/v3alpha/BUILD b/generated_api_shadow/envoy/extensions/filters/network/sni_dynamic_forward_proxy/v3alpha/BUILD new file mode 100644 index 000000000000..1dcb37e5c342 --- /dev/null +++ b/generated_api_shadow/envoy/extensions/filters/network/sni_dynamic_forward_proxy/v3alpha/BUILD @@ -0,0 +1,12 @@ +# DO NOT EDIT. This file is generated by tools/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/extensions/common/dynamic_forward_proxy/v3:pkg", + "@com_github_cncf_udpa//udpa/annotations:pkg", + ], +) diff --git a/generated_api_shadow/envoy/extensions/filters/network/sni_dynamic_forward_proxy/v3alpha/sni_dynamic_forward_proxy.proto b/generated_api_shadow/envoy/extensions/filters/network/sni_dynamic_forward_proxy/v3alpha/sni_dynamic_forward_proxy.proto new file mode 100644 index 000000000000..502a66893174 --- /dev/null +++ b/generated_api_shadow/envoy/extensions/filters/network/sni_dynamic_forward_proxy/v3alpha/sni_dynamic_forward_proxy.proto @@ -0,0 +1,37 @@ +syntax = "proto3"; + +package envoy.extensions.filters.network.sni_dynamic_forward_proxy.v3alpha; + +import "envoy/extensions/common/dynamic_forward_proxy/v3/dns_cache.proto"; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.filters.network.sni_dynamic_forward_proxy.v3alpha"; +option java_outer_classname = "SniDynamicForwardProxyProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).work_in_progress = true; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: SNI dynamic forward proxy] + +// Configuration for the SNI-based dynamic forward proxy filter. See the +// :ref:`architecture overview ` for +// more information. Note this filter must be configured along with +// :ref:`TLS inspector listener filter ` +// to work. +// [#extension: envoy.filters.network.sni_dynamic_forward_proxy] +message FilterConfig { + // The DNS cache configuration that the filter will attach to. Note this + // configuration must match that of associated :ref:`dynamic forward proxy + // cluster configuration + // `. + common.dynamic_forward_proxy.v3.DnsCacheConfig dns_cache_config = 1 + [(validate.rules).message = {required: true}]; + + oneof port_specifier { + // The port number to connect to the upstream. + uint32 port_value = 2 [(validate.rules).uint32 = {lte: 65535 gt: 0}]; + } +} diff --git a/generated_api_shadow/envoy/extensions/tracers/datadog/v4alpha/BUILD b/generated_api_shadow/envoy/extensions/tracers/datadog/v4alpha/BUILD new file mode 100644 index 000000000000..d58bd8a0fca4 --- /dev/null +++ b/generated_api_shadow/envoy/extensions/tracers/datadog/v4alpha/BUILD @@ -0,0 +1,12 @@ +# DO NOT EDIT. This file is generated by tools/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/config/trace/v3:pkg", + "@com_github_cncf_udpa//udpa/annotations:pkg", + ], +) diff --git a/generated_api_shadow/envoy/extensions/tracers/datadog/v4alpha/datadog.proto b/generated_api_shadow/envoy/extensions/tracers/datadog/v4alpha/datadog.proto new file mode 100644 index 000000000000..94359ce837bf --- /dev/null +++ b/generated_api_shadow/envoy/extensions/tracers/datadog/v4alpha/datadog.proto @@ -0,0 +1,27 @@ +syntax = "proto3"; + +package envoy.extensions.tracers.datadog.v4alpha; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.tracers.datadog.v4alpha"; +option java_outer_classname = "DatadogProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; + +// [#protodoc-title: Datadog tracer] + +// Configuration for the Datadog tracer. +// [#extension: envoy.tracers.datadog] +message DatadogConfig { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.trace.v3.DatadogConfig"; + + // The cluster to use for submitting traces to the Datadog agent. + string collector_cluster = 1 [(validate.rules).string = {min_bytes: 1}]; + + // The name used for the service when traces are generated by envoy. + string service_name = 2 [(validate.rules).string = {min_bytes: 1}]; +} diff --git a/generated_api_shadow/envoy/extensions/tracers/dynamic_ot/v4alpha/BUILD b/generated_api_shadow/envoy/extensions/tracers/dynamic_ot/v4alpha/BUILD new file mode 100644 index 000000000000..d58bd8a0fca4 --- /dev/null +++ b/generated_api_shadow/envoy/extensions/tracers/dynamic_ot/v4alpha/BUILD @@ -0,0 +1,12 @@ +# DO NOT EDIT. This file is generated by tools/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/config/trace/v3:pkg", + "@com_github_cncf_udpa//udpa/annotations:pkg", + ], +) diff --git a/generated_api_shadow/envoy/extensions/tracers/dynamic_ot/v4alpha/dynamic_ot.proto b/generated_api_shadow/envoy/extensions/tracers/dynamic_ot/v4alpha/dynamic_ot.proto new file mode 100644 index 000000000000..d311304a3ddf --- /dev/null +++ b/generated_api_shadow/envoy/extensions/tracers/dynamic_ot/v4alpha/dynamic_ot.proto @@ -0,0 +1,33 @@ +syntax = "proto3"; + +package envoy.extensions.tracers.dynamic_ot.v4alpha; + +import "google/protobuf/struct.proto"; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.tracers.dynamic_ot.v4alpha"; +option java_outer_classname = "DynamicOtProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; + +// [#protodoc-title: Dynamically loadable OpenTracing tracer] + +// DynamicOtConfig is used to dynamically load a tracer from a shared library +// that implements the `OpenTracing dynamic loading API +// `_. +// [#extension: envoy.tracers.dynamic_ot] +message DynamicOtConfig { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.trace.v3.DynamicOtConfig"; + + // Dynamic library implementing the `OpenTracing API + // `_. + string library = 1 [(validate.rules).string = {min_bytes: 1}]; + + // The configuration to use when creating a tracer from the given dynamic + // library. + google.protobuf.Struct config = 2; +} diff --git a/generated_api_shadow/envoy/extensions/tracers/lightstep/v4alpha/BUILD b/generated_api_shadow/envoy/extensions/tracers/lightstep/v4alpha/BUILD new file mode 100644 index 000000000000..d58bd8a0fca4 --- /dev/null +++ b/generated_api_shadow/envoy/extensions/tracers/lightstep/v4alpha/BUILD @@ -0,0 +1,12 @@ +# DO NOT EDIT. This file is generated by tools/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/config/trace/v3:pkg", + "@com_github_cncf_udpa//udpa/annotations:pkg", + ], +) diff --git a/generated_api_shadow/envoy/extensions/tracers/lightstep/v4alpha/lightstep.proto b/generated_api_shadow/envoy/extensions/tracers/lightstep/v4alpha/lightstep.proto new file mode 100644 index 000000000000..93ea47ba6a10 --- /dev/null +++ b/generated_api_shadow/envoy/extensions/tracers/lightstep/v4alpha/lightstep.proto @@ -0,0 +1,47 @@ +syntax = "proto3"; + +package envoy.extensions.tracers.lightstep.v4alpha; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.tracers.lightstep.v4alpha"; +option java_outer_classname = "LightstepProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; + +// [#protodoc-title: LightStep tracer] + +// Configuration for the LightStep tracer. +// [#extension: envoy.tracers.lightstep] +message LightstepConfig { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.trace.v3.LightstepConfig"; + + // Available propagation modes + enum PropagationMode { + // Propagate trace context in the single header x-ot-span-context. + ENVOY = 0; + + // Propagate trace context using LightStep's native format. + LIGHTSTEP = 1; + + // Propagate trace context using the b3 format. + B3 = 2; + + // Propagation trace context using the w3 trace-context standard. + TRACE_CONTEXT = 3; + } + + // The cluster manager cluster that hosts the LightStep collectors. + string collector_cluster = 1 [(validate.rules).string = {min_bytes: 1}]; + + // File containing the access token to the `LightStep + // `_ API. + string access_token_file = 2 [(validate.rules).string = {min_bytes: 1}]; + + // Propagation modes to use by LightStep's tracer. + repeated PropagationMode propagation_modes = 3 + [(validate.rules).repeated = {items {enum {defined_only: true}}}]; +} diff --git a/generated_api_shadow/envoy/extensions/tracers/opencensus/v4alpha/BUILD b/generated_api_shadow/envoy/extensions/tracers/opencensus/v4alpha/BUILD new file mode 100644 index 000000000000..05e29893b9cf --- /dev/null +++ b/generated_api_shadow/envoy/extensions/tracers/opencensus/v4alpha/BUILD @@ -0,0 +1,14 @@ +# DO NOT EDIT. This file is generated by tools/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/config/core/v4alpha:pkg", + "//envoy/config/trace/v3:pkg", + "@com_github_cncf_udpa//udpa/annotations:pkg", + "@opencensus_proto//opencensus/proto/trace/v1:trace_config_proto", + ], +) diff --git a/generated_api_shadow/envoy/extensions/tracers/opencensus/v4alpha/opencensus.proto b/generated_api_shadow/envoy/extensions/tracers/opencensus/v4alpha/opencensus.proto new file mode 100644 index 000000000000..f64507b13827 --- /dev/null +++ b/generated_api_shadow/envoy/extensions/tracers/opencensus/v4alpha/opencensus.proto @@ -0,0 +1,97 @@ +syntax = "proto3"; + +package envoy.extensions.tracers.opencensus.v4alpha; + +import "envoy/config/core/v4alpha/grpc_service.proto"; + +import "opencensus/proto/trace/v1/trace_config.proto"; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.tracers.opencensus.v4alpha"; +option java_outer_classname = "OpencensusProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; + +// [#protodoc-title: OpenCensus tracer] + +// Configuration for the OpenCensus tracer. +// [#next-free-field: 15] +// [#extension: envoy.tracers.opencensus] +message OpenCensusConfig { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.trace.v3.OpenCensusConfig"; + + enum TraceContext { + // No-op default, no trace context is utilized. + NONE = 0; + + // W3C Trace-Context format "traceparent:" header. + TRACE_CONTEXT = 1; + + // Binary "grpc-trace-bin:" header. + GRPC_TRACE_BIN = 2; + + // "X-Cloud-Trace-Context:" header. + CLOUD_TRACE_CONTEXT = 3; + + // X-B3-* headers. + B3 = 4; + } + + reserved 7; + + // Configures tracing, e.g. the sampler, max number of annotations, etc. + .opencensus.proto.trace.v1.TraceConfig trace_config = 1; + + // Enables the stdout exporter if set to true. This is intended for debugging + // purposes. + bool stdout_exporter_enabled = 2; + + // Enables the Stackdriver exporter if set to true. The project_id must also + // be set. + bool stackdriver_exporter_enabled = 3; + + // The Cloud project_id to use for Stackdriver tracing. + string stackdriver_project_id = 4; + + // (optional) By default, the Stackdriver exporter will connect to production + // Stackdriver. If stackdriver_address is non-empty, it will instead connect + // to this address, which is in the gRPC format: + // https://github.com/grpc/grpc/blob/master/doc/naming.md + string stackdriver_address = 10; + + // (optional) The gRPC server that hosts Stackdriver tracing service. Only + // Google gRPC is supported. If :ref:`target_uri ` + // is not provided, the default production Stackdriver address will be used. + config.core.v4alpha.GrpcService stackdriver_grpc_service = 13; + + // Enables the Zipkin exporter if set to true. The url and service name must + // also be set. + bool zipkin_exporter_enabled = 5; + + // The URL to Zipkin, e.g. "http://127.0.0.1:9411/api/v2/spans" + string zipkin_url = 6; + + // Enables the OpenCensus Agent exporter if set to true. The ocagent_address or + // ocagent_grpc_service must also be set. + bool ocagent_exporter_enabled = 11; + + // The address of the OpenCensus Agent, if its exporter is enabled, in gRPC + // format: https://github.com/grpc/grpc/blob/master/doc/naming.md + // [#comment:TODO: deprecate this field] + string ocagent_address = 12; + + // (optional) The gRPC server hosted by the OpenCensus Agent. Only Google gRPC is supported. + // This is only used if the ocagent_address is left empty. + config.core.v4alpha.GrpcService ocagent_grpc_service = 14; + + // List of incoming trace context headers we will accept. First one found + // wins. + repeated TraceContext incoming_trace_context = 8; + + // List of outgoing trace context headers we will produce. + repeated TraceContext outgoing_trace_context = 9; +} diff --git a/generated_api_shadow/envoy/extensions/tracers/xray/v4alpha/BUILD b/generated_api_shadow/envoy/extensions/tracers/xray/v4alpha/BUILD new file mode 100644 index 000000000000..d8ce683c41d6 --- /dev/null +++ b/generated_api_shadow/envoy/extensions/tracers/xray/v4alpha/BUILD @@ -0,0 +1,13 @@ +# DO NOT EDIT. This file is generated by tools/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/config/core/v4alpha:pkg", + "//envoy/config/trace/v3:pkg", + "@com_github_cncf_udpa//udpa/annotations:pkg", + ], +) diff --git a/api/envoy/config/trace/v4alpha/xray.proto b/generated_api_shadow/envoy/extensions/tracers/xray/v4alpha/xray.proto similarity index 82% rename from api/envoy/config/trace/v4alpha/xray.proto rename to generated_api_shadow/envoy/extensions/tracers/xray/v4alpha/xray.proto index 39bcebd1bad7..27a9b5407dda 100644 --- a/api/envoy/config/trace/v4alpha/xray.proto +++ b/generated_api_shadow/envoy/extensions/tracers/xray/v4alpha/xray.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package envoy.config.trace.v4alpha; +package envoy.extensions.tracers.xray.v4alpha; import "envoy/config/core/v4alpha/address.proto"; import "envoy/config/core/v4alpha/base.proto"; @@ -9,7 +9,7 @@ import "udpa/annotations/status.proto"; import "udpa/annotations/versioning.proto"; import "validate/validate.proto"; -option java_package = "io.envoyproxy.envoy.config.trace.v4alpha"; +option java_package = "io.envoyproxy.envoy.extensions.tracers.xray.v4alpha"; option java_outer_classname = "XrayProto"; option java_multiple_files = true; option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; @@ -22,7 +22,7 @@ message XRayConfig { // The UDP endpoint of the X-Ray Daemon where the spans will be sent. // If this value is not set, the default value of 127.0.0.1:2000 will be used. - core.v4alpha.SocketAddress daemon_endpoint = 1; + config.core.v4alpha.SocketAddress daemon_endpoint = 1; // The name of the X-Ray segment. string segment_name = 2 [(validate.rules).string = {min_len: 1}]; @@ -31,5 +31,5 @@ message XRayConfig { // For an example of the sampling rules see: // `X-Ray SDK documentation // `_ - core.v4alpha.DataSource sampling_rule_manifest = 3; + config.core.v4alpha.DataSource sampling_rule_manifest = 3; } diff --git a/generated_api_shadow/envoy/extensions/tracers/zipkin/v4alpha/BUILD b/generated_api_shadow/envoy/extensions/tracers/zipkin/v4alpha/BUILD new file mode 100644 index 000000000000..8e8f00da8af0 --- /dev/null +++ b/generated_api_shadow/envoy/extensions/tracers/zipkin/v4alpha/BUILD @@ -0,0 +1,13 @@ +# DO NOT EDIT. This file is generated by tools/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/annotations:pkg", + "//envoy/config/trace/v3:pkg", + "@com_github_cncf_udpa//udpa/annotations:pkg", + ], +) diff --git a/generated_api_shadow/envoy/extensions/tracers/zipkin/v4alpha/zipkin.proto b/generated_api_shadow/envoy/extensions/tracers/zipkin/v4alpha/zipkin.proto new file mode 100644 index 000000000000..4207546ff701 --- /dev/null +++ b/generated_api_shadow/envoy/extensions/tracers/zipkin/v4alpha/zipkin.proto @@ -0,0 +1,68 @@ +syntax = "proto3"; + +package envoy.extensions.tracers.zipkin.v4alpha; + +import "google/protobuf/wrappers.proto"; + +import "envoy/annotations/deprecation.proto"; +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.tracers.zipkin.v4alpha"; +option java_outer_classname = "ZipkinProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; + +// [#protodoc-title: Zipkin tracer] + +// Configuration for the Zipkin tracer. +// [#extension: envoy.tracers.zipkin] +// [#next-free-field: 6] +message ZipkinConfig { + option (udpa.annotations.versioning).previous_message_type = "envoy.config.trace.v3.ZipkinConfig"; + + // Available Zipkin collector endpoint versions. + enum CollectorEndpointVersion { + // Zipkin API v1, JSON over HTTP. + // [#comment: The default implementation of Zipkin client before this field is added was only v1 + // and the way user configure this was by not explicitly specifying the version. Consequently, + // before this is added, the corresponding Zipkin collector expected to receive v1 payload. + // Hence the motivation of adding HTTP_JSON_V1 as the default is to avoid a breaking change when + // user upgrading Envoy with this change. Furthermore, we also immediately deprecate this field, + // since in Zipkin realm this v1 version is considered to be not preferable anymore.] + hidden_envoy_deprecated_DEPRECATED_AND_UNAVAILABLE_DO_NOT_USE = 0 + [deprecated = true, (envoy.annotations.disallowed_by_default_enum) = true]; + + // Zipkin API v2, JSON over HTTP. + HTTP_JSON = 1; + + // Zipkin API v2, protobuf over HTTP. + HTTP_PROTO = 2; + + // [#not-implemented-hide:] + GRPC = 3; + } + + // The cluster manager cluster that hosts the Zipkin collectors. Note that the + // Zipkin cluster must be defined in the :ref:`Bootstrap static cluster + // resources `. + string collector_cluster = 1 [(validate.rules).string = {min_bytes: 1}]; + + // The API endpoint of the Zipkin service where the spans will be sent. When + // using a standard Zipkin installation, the API endpoint is typically + // /api/v1/spans, which is the default value. + string collector_endpoint = 2 [(validate.rules).string = {min_bytes: 1}]; + + // Determines whether a 128bit trace id will be used when creating a new + // trace instance. The default value is false, which will result in a 64 bit trace id being used. + bool trace_id_128bit = 3; + + // Determines whether client and server spans will share the same span context. + // The default value is true. + google.protobuf.BoolValue shared_span_context = 4; + + // Determines the selected collector endpoint version. By default, the ``HTTP_JSON_V1`` will be + // used. + CollectorEndpointVersion collector_endpoint_version = 5; +} diff --git a/include/envoy/grpc/context.h b/include/envoy/grpc/context.h index 191a2583cb9e..1f2d2469e5d8 100644 --- a/include/envoy/grpc/context.h +++ b/include/envoy/grpc/context.h @@ -84,6 +84,16 @@ class Context { const absl::optional& request_names, uint64_t amount) PURE; + /** + * Charge upstream stat to a cluster/service/method. + * @param cluster supplies the target cluster. + * @param request_names supplies the request names. + * @param duration supplies the duration of the upstream request. + */ + virtual void chargeUpstreamStat(const Upstream::ClusterInfo& cluster, + const absl::optional& request_names, + std::chrono::milliseconds duration) PURE; + /** * @return a struct containing StatNames for gRPC stat tokens. */ diff --git a/include/envoy/http/context.h b/include/envoy/http/context.h index 73d05abd463e..a9d5ac5c9315 100644 --- a/include/envoy/http/context.h +++ b/include/envoy/http/context.h @@ -2,7 +2,7 @@ #include -#include "envoy/config/trace/v3/trace.pb.h" +#include "envoy/config/trace/v3/http_tracer.pb.h" #include "envoy/http/codes.h" namespace Envoy { diff --git a/include/envoy/network/connection_handler.h b/include/envoy/network/connection_handler.h index eefb69e7dd96..b8787df14ef8 100644 --- a/include/envoy/network/connection_handler.h +++ b/include/envoy/network/connection_handler.h @@ -38,10 +38,12 @@ class ConnectionHandler { virtual void decNumConnections() PURE; /** - * Adds a listener to the handler. + * Adds a listener to the handler, optionally replacing the existing listener. + * @param overridden_listener tag of the existing listener. nullopt if no previous listener. * @param config listener configuration options. */ - virtual void addListener(ListenerConfig& config) PURE; + virtual void addListener(absl::optional overridden_listener, + ListenerConfig& config) PURE; /** * Remove listeners using the listener tag as a key. All connections owned by the removed @@ -50,6 +52,17 @@ class ConnectionHandler { */ virtual void removeListeners(uint64_t listener_tag) PURE; + /** + * Remove the filter chains and the connections in the listener. All connections owned + * by the filter chains will be closed. Once all the connections are destroyed(connections + * could be deferred deleted!), invoke the completion. + * @param listener_tag supplies the tag passed to addListener(). + * @param filter_chains supplies the filter chains to be removed. + */ + virtual void removeFilterChains(uint64_t listener_tag, + const std::list& filter_chains, + std::function completion) PURE; + /** * Stop listeners using the listener tag as a key. This will not close any connections and is used * for draining. diff --git a/include/envoy/server/instance.h b/include/envoy/server/instance.h index 83996f09db05..1e6335962e71 100644 --- a/include/envoy/server/instance.h +++ b/include/envoy/server/instance.h @@ -7,7 +7,7 @@ #include "envoy/access_log/access_log.h" #include "envoy/api/api.h" #include "envoy/common/mutex_tracer.h" -#include "envoy/config/trace/v3/trace.pb.h" +#include "envoy/config/trace/v3/http_tracer.pb.h" #include "envoy/event/timer.h" #include "envoy/grpc/context.h" #include "envoy/http/context.h" diff --git a/include/envoy/server/worker.h b/include/envoy/server/worker.h index 474e844a0821..c2f273044150 100644 --- a/include/envoy/server/worker.h +++ b/include/envoy/server/worker.h @@ -24,12 +24,16 @@ class Worker { using AddListenerCompletion = std::function; /** - * Add a listener to the worker. + * Add a listener to the worker and replace the previous listener if any. If the previous listener + * doesn't exist, the behavior should be equivalent to add a new listener. + * @param overridden_listener The previous listener tag to be replaced. nullopt if it's a new + * listener. * @param listener supplies the listener to add. * @param completion supplies the completion to call when the listener has been added (or not) on * the worker. */ - virtual void addListener(Network::ListenerConfig& listener, + virtual void addListener(absl::optional overridden_listener, + Network::ListenerConfig& listener, AddListenerCompletion completion) PURE; /** @@ -63,6 +67,17 @@ class Worker { */ virtual void removeListener(Network::ListenerConfig& listener, std::function completion) PURE; + /** + * Remove the stale filter chains of the given listener but leave the listener running. + * @param listener_tag supplies the tag passed to addListener(). + * @param filter_chains supplies the filter chains to be removed. + * @param completion supplies the completion to be called when the listener removed all the + * untracked connections. This completion is called on the worker thread. No locking is performed + * by the worker. + */ + virtual void removeFilterChains(uint64_t listener_tag, + const std::list& filter_chains, + std::function completion) PURE; /** * Stop a listener from accepting new connections. This is used for server draining. diff --git a/include/envoy/stats/allocator.h b/include/envoy/stats/allocator.h index 681befa5f3b3..f6181553ad85 100644 --- a/include/envoy/stats/allocator.h +++ b/include/envoy/stats/allocator.h @@ -32,8 +32,7 @@ class Allocator { * @param name the full name of the stat. * @param tag_extracted_name the name of the stat with tag-values stripped out. * @param tags the tag values. - * @return CounterSharedPtr a counter, or nullptr if allocation failed, in which case - * tag_extracted_name and tags are not moved. + * @return CounterSharedPtr a counter. */ virtual CounterSharedPtr makeCounter(StatName name, StatName tag_extracted_name, const StatNameTagVector& stat_name_tags) PURE; @@ -42,13 +41,20 @@ class Allocator { * @param name the full name of the stat. * @param tag_extracted_name the name of the stat with tag-values stripped out. * @param stat_name_tags the tag values. - * @return GaugeSharedPtr a gauge, or nullptr if allocation failed, in which case - * tag_extracted_name and tags are not moved. + * @return GaugeSharedPtr a gauge. */ virtual GaugeSharedPtr makeGauge(StatName name, StatName tag_extracted_name, const StatNameTagVector& stat_name_tags, Gauge::ImportMode import_mode) PURE; + /** + * @param name the full name of the stat. + * @param tag_extracted_name the name of the stat with tag-values stripped out. + * @param tags the tag values. + * @return TextReadoutSharedPtr a text readout. + */ + virtual TextReadoutSharedPtr makeTextReadout(StatName name, StatName tag_extracted_name, + const StatNameTagVector& stat_name_tags) PURE; virtual const SymbolTable& constSymbolTable() const PURE; virtual SymbolTable& symbolTable() PURE; diff --git a/include/envoy/stats/scope.h b/include/envoy/stats/scope.h index 03a232063c13..408655bbb8a5 100644 --- a/include/envoy/stats/scope.h +++ b/include/envoy/stats/scope.h @@ -17,12 +17,14 @@ namespace Stats { class Counter; class Gauge; class Histogram; -class Scope; class NullGaugeImpl; +class Scope; +class TextReadout; using CounterOptConstRef = absl::optional>; using GaugeOptConstRef = absl::optional>; using HistogramOptConstRef = absl::optional>; +using TextReadoutOptConstRef = absl::optional>; using ScopePtr = std::unique_ptr; using ScopeSharedPtr = std::shared_ptr; @@ -136,6 +138,32 @@ class Scope { */ virtual Histogram& histogramFromString(const std::string& name, Histogram::Unit unit) PURE; + /** + * Creates a TextReadout from the stat name. Tag extraction will be performed on the name. + * @param name The name of the stat, obtained from the SymbolTable. + * @return a text readout within the scope's namespace. + */ + TextReadout& textReadoutFromStatName(const StatName& name) { + return textReadoutFromStatNameWithTags(name, absl::nullopt); + } + + /** + * Creates a TextReadout from the stat name and tags. If tags are not provided, tag extraction + * will be performed on the name. + * @param name The name of the stat, obtained from the SymbolTable. + * @param tags optionally specified tags. + * @return a text readout within the scope's namespace. + */ + virtual TextReadout& textReadoutFromStatNameWithTags(const StatName& name, + StatNameTagVectorOptConstRef tags) PURE; + + /** + * TODO(#6667): this variant is deprecated: use textReadoutFromStatName. + * @param name The name, expressed as a string. + * @return a text readout within the scope's namespace. + */ + virtual TextReadout& textReadoutFromString(const std::string& name) PURE; + /** * @param The name of the stat, obtained from the SymbolTable. * @return a reference to a counter within the scope's namespace, if it exists. @@ -155,6 +183,12 @@ class Scope { */ virtual HistogramOptConstRef findHistogram(StatName name) const PURE; + /** + * @param The name of the stat, obtained from the SymbolTable. + * @return a reference to a text readout within the scope's namespace, if it exists. + */ + virtual TextReadoutOptConstRef findTextReadout(StatName name) const PURE; + /** * @return a reference to the symbol table. */ diff --git a/include/envoy/stats/sink.h b/include/envoy/stats/sink.h index d58ea33220fa..f0ce08c1dd05 100644 --- a/include/envoy/stats/sink.h +++ b/include/envoy/stats/sink.h @@ -35,6 +35,7 @@ class MetricSnapshot { * @return a snapshot of all histograms. */ virtual const std::vector>& histograms() PURE; + // TODO(efimki): Add support of text readouts stats. }; /** diff --git a/include/envoy/stats/stats.h b/include/envoy/stats/stats.h index b1772fb63212..c723152ab549 100644 --- a/include/envoy/stats/stats.h +++ b/include/envoy/stats/stats.h @@ -156,5 +156,32 @@ class Gauge : public Metric { using GaugeSharedPtr = RefcountPtr; +/** + * A string, possibly non-ASCII. + */ +class TextReadout : public virtual Metric { +public: + // Text readout type is used internally to disambiguate isolated store + // constructors. In the future we can extend it to specify text encoding or + // some such. + enum class Type { + Default, // No particular meaning. + }; + + ~TextReadout() override = default; + + /** + * Sets the value of this TextReadout by moving the input |value| to minimize + * buffer copies under the lock. + */ + virtual void set(std::string&& value) PURE; + /** + * @return the copy of this TextReadout value. + */ + virtual std::string value() const PURE; +}; + +using TextReadoutSharedPtr = RefcountPtr; + } // namespace Stats } // namespace Envoy diff --git a/include/envoy/stats/stats_macros.h b/include/envoy/stats/stats_macros.h index 77d2243903f4..a37c5566e89c 100644 --- a/include/envoy/stats/stats_macros.h +++ b/include/envoy/stats/stats_macros.h @@ -39,6 +39,7 @@ namespace Envoy { #define GENERATE_COUNTER_STRUCT(NAME) Envoy::Stats::Counter& NAME##_; #define GENERATE_GAUGE_STRUCT(NAME, MODE) Envoy::Stats::Gauge& NAME##_; #define GENERATE_HISTOGRAM_STRUCT(NAME, UNIT) Envoy::Stats::Histogram& NAME##_; +#define GENERATE_TEXT_READOUT_STRUCT(NAME) Envoy::Stats::TextReadout& NAME##_; #define FINISH_STAT_DECL_(X) #X)), #define FINISH_STAT_DECL_MODE_(X, MODE) #X), Envoy::Stats::Gauge::ImportMode::MODE), @@ -57,10 +58,12 @@ static inline std::string statPrefixJoin(absl::string_view prefix, absl::string_ #define POOL_COUNTER_PREFIX(POOL, PREFIX) (POOL).counterFromString(Envoy::statPrefixJoin(PREFIX, FINISH_STAT_DECL_ #define POOL_GAUGE_PREFIX(POOL, PREFIX) (POOL).gaugeFromString(Envoy::statPrefixJoin(PREFIX, FINISH_STAT_DECL_MODE_ #define POOL_HISTOGRAM_PREFIX(POOL, PREFIX) (POOL).histogramFromString(Envoy::statPrefixJoin(PREFIX, FINISH_STAT_DECL_UNIT_ +#define POOL_TEXT_READOUT_PREFIX(POOL, PREFIX) (POOL).textReadoutFromString(Envoy::statPrefixJoin(PREFIX, FINISH_STAT_DECL_ #define POOL_COUNTER(POOL) POOL_COUNTER_PREFIX(POOL, "") #define POOL_GAUGE(POOL) POOL_GAUGE_PREFIX(POOL, "") #define POOL_HISTOGRAM(POOL) POOL_HISTOGRAM_PREFIX(POOL, "") +#define POOL_TEXT_READOUT(POOL) POOL_TEXT_READOUT_PREFIX(POOL, "") #define NULL_STAT_DECL_(X) std::string(#X)), #define NULL_STAT_DECL_IGNORE_MODE_(X, MODE) std::string(#X)), diff --git a/include/envoy/stats/store.h b/include/envoy/stats/store.h index d959a58b01d9..158f00518a51 100644 --- a/include/envoy/stats/store.h +++ b/include/envoy/stats/store.h @@ -38,6 +38,11 @@ class Store : public Scope { */ virtual std::vector gauges() const PURE; + /** + * @return a list of all known text readouts. + */ + virtual std::vector textReadouts() const PURE; + /** * @return a list of all known histograms. */ diff --git a/include/envoy/stream_info/filter_state.h b/include/envoy/stream_info/filter_state.h index 0e41d967cf19..f68fca790ab2 100644 --- a/include/envoy/stream_info/filter_state.h +++ b/include/envoy/stream_info/filter_state.h @@ -10,6 +10,7 @@ #include "common/protobuf/protobuf.h" #include "absl/strings/string_view.h" +#include "absl/types/optional.h" namespace Envoy { namespace StreamInfo { @@ -30,21 +31,16 @@ class FilterState { // // - FilterChain has the shortest life span, which is as long as the filter chain lives. // - // - DownstreamRequest is longer than FilterChain. When internal redirect is enabled, one - // downstream request may create multiple filter chains. DownstreamRequest allows an object to - // survive across filter chains for bookkeeping needs. + // - Request is longer than FilterChain. When internal redirect is enabled, one + // downstream request may create multiple filter chains. Request allows an object to + // survive across filter chains for bookkeeping needs. This is not used for the upstream case. // - // - DownstreamConnection makes an object survive the entire duration of a downstream connection. + // - Connection makes an object survive the entire duration of a connection. // Any stream within this connection can see the same object. // // Note that order matters in this enum because it's assumed that life span grows as enum number // grows. - enum LifeSpan { - FilterChain, - DownstreamRequest, - DownstreamConnection, - TopSpan = DownstreamConnection - }; + enum LifeSpan { FilterChain, Request, Connection, TopSpan = Connection }; class Object { public: @@ -56,6 +52,13 @@ class FilterState { * logging. nullptr if the filter state cannot be serialized or serialization is not supported. */ virtual ProtobufTypes::MessagePtr serializeAsProto() const { return nullptr; } + + /** + * @return absl::optional a optional string to the serialization of the filter + * state. No value if the filter state cannot be serialized or serialization is not supported. + * This method can be used to get an unstructured serialization result. + */ + virtual absl::optional serializeAsString() const { return absl::nullopt; } }; virtual ~FilterState() = default; @@ -65,7 +68,7 @@ class FilterState { * @param data an owning pointer to the data to be stored. * @param state_type indicates whether the object is mutable or not. * @param life_span indicates the life span of the object: bound to the filter chain, a - * downstream request, or a downstream connection. + * request, or a connection. * * Note that it is an error to call setData() twice with the same * data_name, if the existing object is immutable. Similarly, it is an diff --git a/include/envoy/stream_info/stream_info.h b/include/envoy/stream_info/stream_info.h index 1afda3a2336a..89824f4190f4 100644 --- a/include/envoy/stream_info/stream_info.h +++ b/include/envoy/stream_info/stream_info.h @@ -91,6 +91,10 @@ struct ResponseCodeDetailValues { // Envoy is doing non-streaming proxying, and the request payload exceeded // configured limits. const std::string RequestPayloadTooLarge = "request_payload_too_large"; + // Envoy is doing streaming proxying, but too much data arrived while waiting + // to attempt a retry. + const std::string RequestPayloadExceededRetryBufferLimit = + "request_payload_exceeded_retry_buffer_limit"; // Envoy is doing non-streaming proxying, and the response payload exceeded // configured limits. const std::string ResponsePayloadTooLArge = "response_payload_too_large"; diff --git a/include/envoy/tracing/http_tracer_manager.h b/include/envoy/tracing/http_tracer_manager.h index bab294cd6dc9..a19d384bb34f 100644 --- a/include/envoy/tracing/http_tracer_manager.h +++ b/include/envoy/tracing/http_tracer_manager.h @@ -1,6 +1,6 @@ #pragma once -#include "envoy/config/trace/v3/trace.pb.h" +#include "envoy/config/trace/v3/http_tracer.pb.h" #include "envoy/tracing/http_tracer.h" namespace Envoy { diff --git a/source/common/access_log/access_log_formatter.cc b/source/common/access_log/access_log_formatter.cc index 49d91ab1ca05..6ea62a7dd1a8 100644 --- a/source/common/access_log/access_log_formatter.cc +++ b/source/common/access_log/access_log_formatter.cc @@ -233,6 +233,9 @@ std::vector AccessLogFormatParser::parse(const std::string static constexpr absl::string_view FILTER_STATE_TOKEN{"FILTER_STATE("}; const std::regex command_w_args_regex(R"EOF(%([A-Z]|_)+(\([^\)]*\))?(:[0-9]+)?(%))EOF"); + static constexpr absl::string_view PLAIN_SERIALIZATION{"PLAIN"}; + static constexpr absl::string_view TYPED_SERIALIZATION{"TYPED"}; + for (size_t pos = 0; pos < format.length(); ++pos) { if (format[pos] == '%') { if (!current_token.empty()) { @@ -292,12 +295,21 @@ std::vector AccessLogFormatParser::parse(const std::string std::vector path; const size_t start = FILTER_STATE_TOKEN.size(); - parseCommand(token, start, "", key, path, max_length); + parseCommand(token, start, ":", key, path, max_length); if (key.empty()) { throw EnvoyException("Invalid filter state configuration, key cannot be empty."); } - formatters.push_back(std::make_unique(key, max_length)); + const absl::string_view serialize_type = + !path.empty() ? path[path.size() - 1] : TYPED_SERIALIZATION; + + if (serialize_type != PLAIN_SERIALIZATION && serialize_type != TYPED_SERIALIZATION) { + throw EnvoyException("Invalid filter state serialize type, only support PLAIN/TYPED."); + } + const bool serialize_as_string = serialize_type == PLAIN_SERIALIZATION; + + formatters.push_back( + std::make_unique(key, max_length, serialize_as_string)); } else if (absl::StartsWith(token, "START_TIME")) { const size_t parameters_length = pos + StartTimeParamStart + 1; const size_t parameters_end = command_end_position - parameters_length; @@ -966,25 +978,38 @@ DynamicMetadataFormatter::formatValue(const Http::RequestHeaderMap&, const Http: } FilterStateFormatter::FilterStateFormatter(const std::string& key, - absl::optional max_length) - : key_(key), max_length_(max_length) {} + absl::optional max_length, + bool serialize_as_string) + : key_(key), max_length_(max_length), serialize_as_string_(serialize_as_string) {} -ProtobufTypes::MessagePtr +const Envoy::StreamInfo::FilterState::Object* FilterStateFormatter::filterState(const StreamInfo::StreamInfo& stream_info) const { const StreamInfo::FilterState& filter_state = stream_info.filterState(); if (!filter_state.hasDataWithName(key_)) { return nullptr; } - - const auto& object = filter_state.getDataReadOnly(key_); - return object.serializeAsProto(); + return &filter_state.getDataReadOnly(key_); } std::string FilterStateFormatter::format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo& stream_info) const { - ProtobufTypes::MessagePtr proto = filterState(stream_info); + const Envoy::StreamInfo::FilterState::Object* state = filterState(stream_info); + if (!state) { + return UnspecifiedValueString; + } + + if (serialize_as_string_) { + absl::optional plain_value = state->serializeAsString(); + if (plain_value.has_value()) { + truncate(plain_value.value(), max_length_); + return plain_value.value(); + } + return UnspecifiedValueString; + } + + ProtobufTypes::MessagePtr proto = state->serializeAsProto(); if (proto == nullptr) { return UnspecifiedValueString; } @@ -1005,8 +1030,22 @@ ProtobufWkt::Value FilterStateFormatter::formatValue(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, const Http::ResponseTrailerMap&, const StreamInfo::StreamInfo& stream_info) const { - ProtobufTypes::MessagePtr proto = filterState(stream_info); - if (proto == nullptr) { + const Envoy::StreamInfo::FilterState::Object* state = filterState(stream_info); + if (!state) { + return unspecifiedValue(); + } + + if (serialize_as_string_) { + absl::optional plain_value = state->serializeAsString(); + if (plain_value.has_value()) { + truncate(plain_value.value(), max_length_); + return ValueUtil::stringValue(plain_value.value()); + } + return unspecifiedValue(); + } + + ProtobufTypes::MessagePtr proto = state->serializeAsProto(); + if (!proto) { return unspecifiedValue(); } diff --git a/source/common/access_log/access_log_formatter.h b/source/common/access_log/access_log_formatter.h index 4437462b5d1b..408eb49b3eab 100644 --- a/source/common/access_log/access_log_formatter.h +++ b/source/common/access_log/access_log_formatter.h @@ -293,7 +293,8 @@ class DynamicMetadataFormatter : public FormatterProvider, MetadataFormatter { */ class FilterStateFormatter : public FormatterProvider { public: - FilterStateFormatter(const std::string& key, absl::optional max_length); + FilterStateFormatter(const std::string& key, absl::optional max_length, + bool serialize_as_string); // FormatterProvider std::string format(const Http::RequestHeaderMap&, const Http::ResponseHeaderMap&, @@ -303,10 +304,13 @@ class FilterStateFormatter : public FormatterProvider { const StreamInfo::StreamInfo&) const override; private: - ProtobufTypes::MessagePtr filterState(const StreamInfo::StreamInfo& stream_info) const; + const Envoy::StreamInfo::FilterState::Object* + filterState(const StreamInfo::StreamInfo& stream_info) const; std::string key_; absl::optional max_length_; + + bool serialize_as_string_; }; /** diff --git a/source/common/event/BUILD b/source/common/event/BUILD index 612bcfcf31ec..33fd947b8d77 100644 --- a/source/common/event/BUILD +++ b/source/common/event/BUILD @@ -123,3 +123,12 @@ envoy_cc_library( "//source/common/common:scope_tracker", ], ) + +envoy_cc_library( + name = "deferred_task", + hdrs = ["deferred_task.h"], + deps = [ + "//include/envoy/event:deferred_deletable", + "//include/envoy/event:dispatcher_interface", + ], +) diff --git a/source/common/event/deferred_task.h b/source/common/event/deferred_task.h new file mode 100644 index 000000000000..66ea7b929cc3 --- /dev/null +++ b/source/common/event/deferred_task.h @@ -0,0 +1,35 @@ +#pragma once + +#include + +#include "envoy/event/dispatcher.h" + +namespace Envoy { +namespace Event { + +/** + * A util to schedule a task to run in a future event loop cycle. One of the use cases is to run the + * task after the previously DeferredDeletable objects are destroyed. + */ +class DeferredTaskUtil { +private: + class DeferredTask : public DeferredDeletable { + public: + DeferredTask(std::function&& task) : task_(std::move(task)) {} + ~DeferredTask() override { task_(); } + + private: + std::function task_; + }; + +public: + /** + * Submits an item for run deferred delete. + */ + static void deferredRun(Dispatcher& dispatcher, std::function&& func) { + dispatcher.deferredDelete(std::make_unique(std::move(func))); + } +}; + +} // namespace Event +} // namespace Envoy \ No newline at end of file diff --git a/source/common/grpc/context_impl.cc b/source/common/grpc/context_impl.cc index 04c56ddef1b3..f612f71cc074 100644 --- a/source/common/grpc/context_impl.cc +++ b/source/common/grpc/context_impl.cc @@ -15,7 +15,7 @@ ContextImpl::ContextImpl(Stats::SymbolTable& symbol_table) total_(stat_name_pool_.add("total")), zero_(stat_name_pool_.add("0")), request_message_count_(stat_name_pool_.add("request_message_count")), response_message_count_(stat_name_pool_.add("response_message_count")), - stat_names_(symbol_table) {} + upstream_rq_time_(stat_name_pool_.add("upstream_rq_time")), stat_names_(symbol_table) {} // Makes a stat name from a string, if we don't already have one for it. // This always takes a lock on mutex_, and if we haven't seen the name @@ -114,6 +114,21 @@ void ContextImpl::chargeResponseMessageStat(const Upstream::ClusterInfo& cluster .add(amount); } +void ContextImpl::chargeUpstreamStat(const Upstream::ClusterInfo& cluster, + const absl::optional& request_names, + std::chrono::milliseconds duration) { + auto prefix_and_storage = getPrefix(Protocol::Grpc, request_names); + Stats::StatName prefix = prefix_and_storage.first; + + const Stats::SymbolTable::StoragePtr upstream_rq_time = + symbol_table_.join({prefix, upstream_rq_time_}); + + cluster.statsScope() + .histogramFromStatName(Stats::StatName(upstream_rq_time.get()), + Stats::Histogram::Unit::Milliseconds) + .recordValue(duration.count()); +} + absl::optional ContextImpl::resolveDynamicServiceAndMethod(const Http::HeaderEntry* path) { absl::optional request_names = Common::resolveServiceAndMethod(path); diff --git a/source/common/grpc/context_impl.h b/source/common/grpc/context_impl.h index e220802eefb5..9d3ddc731458 100644 --- a/source/common/grpc/context_impl.h +++ b/source/common/grpc/context_impl.h @@ -38,6 +38,9 @@ class ContextImpl : public Context { void chargeResponseMessageStat(const Upstream::ClusterInfo& cluster, const absl::optional& request_names, uint64_t amount) override; + void chargeUpstreamStat(const Upstream::ClusterInfo& cluster, + const absl::optional& request_names, + std::chrono::milliseconds duration) override; /** * Resolve the gRPC service and method from the HTTP2 :path header. @@ -83,6 +86,7 @@ class ContextImpl : public Context { const Stats::StatName zero_; const Stats::StatName request_message_count_; const Stats::StatName response_message_count_; + const Stats::StatName upstream_rq_time_; StatNames stat_names_; }; diff --git a/source/common/http/BUILD b/source/common/http/BUILD index b2b394ae7d51..283f6e2ef7aa 100644 --- a/source/common/http/BUILD +++ b/source/common/http/BUILD @@ -376,7 +376,7 @@ envoy_cc_library( "//source/common/common:regex_lib", "//source/common/common:utility_lib", "//source/common/protobuf:utility_lib", - "//source/common/runtime:runtime_lib", + "//source/common/runtime:runtime_features_lib", "@envoy_api//envoy/config/route/v3:pkg_cc_proto", "@envoy_api//envoy/type/v3:pkg_cc_proto", ], diff --git a/source/common/http/codec_client.cc b/source/common/http/codec_client.cc index 682d4ea394c3..6d8011e22485 100644 --- a/source/common/http/codec_client.cc +++ b/source/common/http/codec_client.cc @@ -156,7 +156,8 @@ CodecClientProd::CodecClientProd(Type type, Network::ClientConnectionPtr&& conne case Type::HTTP2: { codec_ = std::make_unique( *connection_, *this, host->cluster().statsScope(), host->cluster().http2Options(), - Http::DEFAULT_MAX_REQUEST_HEADERS_KB, host->cluster().maxResponseHeadersCount()); + Http::DEFAULT_MAX_REQUEST_HEADERS_KB, host->cluster().maxResponseHeadersCount(), + Http2::ProdNghttp2SessionFactory::get()); break; } case Type::HTTP3: { diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index 2b3946187846..270d9d92b53b 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -2387,7 +2387,7 @@ bool ConnectionManagerImpl::ActiveStreamDecoderFilter::recreateStream() { // store any objects with a LifeSpan at or above DownstreamRequest. This is to avoid unnecessary // heap allocation. if (parent_.stream_info_.filter_state_->hasDataAtOrAboveLifeSpan( - StreamInfo::FilterState::LifeSpan::DownstreamRequest)) { + StreamInfo::FilterState::LifeSpan::Request)) { (*parent_.connection_manager_.streams_.begin())->stream_info_.filter_state_ = std::make_shared( parent_.stream_info_.filter_state_->parent(), diff --git a/source/common/http/conn_manager_utility.cc b/source/common/http/conn_manager_utility.cc index 2526ac56b114..b4a97bfa8b05 100644 --- a/source/common/http/conn_manager_utility.cc +++ b/source/common/http/conn_manager_utility.cc @@ -15,6 +15,7 @@ #include "common/http/path_utility.h" #include "common/http/utility.h" #include "common/network/utility.h" +#include "common/runtime/runtime_features.h" #include "common/tracing/http_tracer_impl.h" #include "absl/strings/str_cat.h" @@ -373,7 +374,12 @@ void ConnectionManagerUtility::mutateResponseHeaders( // upgrade response it has already passed the protocol checks. const bool no_body = (!response_headers.TransferEncoding() && !response_headers.ContentLength()); - if (no_body) { + + const bool is_1xx = CodeUtility::is1xx(Utility::getResponseStatus(response_headers)); + + // We are explicitly forbidden from setting content-length for 1xx responses + // (RFC7230, Section 3.3.2). We ignore 204 because this is an upgrade. + if (no_body && !is_1xx) { response_headers.setContentLength(uint64_t(0)); } } else { diff --git a/source/common/http/header_utility.cc b/source/common/http/header_utility.cc index 9772f3556683..5da6106261a9 100644 --- a/source/common/http/header_utility.cc +++ b/source/common/http/header_utility.cc @@ -6,7 +6,7 @@ #include "common/common/utility.h" #include "common/http/header_map_impl.h" #include "common/protobuf/utility.h" -#include "common/runtime/runtime_impl.h" +#include "common/runtime/runtime_features.h" #include "absl/strings/match.h" #include "nghttp2/nghttp2.h" diff --git a/source/common/http/http1/BUILD b/source/common/http/http1/BUILD index 5ec7f16ab906..2709b312976e 100644 --- a/source/common/http/http1/BUILD +++ b/source/common/http/http1/BUILD @@ -38,7 +38,7 @@ envoy_cc_library( "//source/common/http:headers_lib", "//source/common/http:utility_lib", "//source/common/http/http1:header_formatter_lib", - "//source/common/runtime:runtime_lib", + "//source/common/runtime:runtime_features_lib", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], ) @@ -66,7 +66,7 @@ envoy_cc_library( "//source/common/http:conn_pool_base_legacy_lib", "//source/common/http:headers_lib", "//source/common/network:utility_lib", - "//source/common/runtime:runtime_lib", + "//source/common/runtime:runtime_features_lib", "//source/common/stats:timespan_lib", "//source/common/upstream:upstream_lib", ], @@ -90,7 +90,7 @@ envoy_cc_library( "//source/common/http:codes_lib", "//source/common/http:conn_pool_base_lib", "//source/common/http:headers_lib", - "//source/common/runtime:runtime_lib", + "//source/common/runtime:runtime_features_lib", "//source/common/upstream:upstream_lib", ], ) diff --git a/source/common/http/http1/codec_impl.cc b/source/common/http/http1/codec_impl.cc index de1258a57ad3..cc2f53d6b3a2 100644 --- a/source/common/http/http1/codec_impl.cc +++ b/source/common/http/http1/codec_impl.cc @@ -16,7 +16,7 @@ #include "common/http/headers.h" #include "common/http/http1/header_formatter.h" #include "common/http/utility.h" -#include "common/runtime/runtime_impl.h" +#include "common/runtime/runtime_features.h" #include "absl/container/fixed_array.h" #include "absl/strings/ascii.h" diff --git a/source/common/http/http1/conn_pool.cc b/source/common/http/http1/conn_pool.cc index 428657bcd8af..9b82d9b4d7e4 100644 --- a/source/common/http/http1/conn_pool.cc +++ b/source/common/http/http1/conn_pool.cc @@ -14,7 +14,7 @@ #include "common/http/codes.h" #include "common/http/headers.h" #include "common/http/http1/conn_pool_legacy.h" -#include "common/runtime/runtime_impl.h" +#include "common/runtime/runtime_features.h" #include "absl/strings/match.h" diff --git a/source/common/http/http1/conn_pool_legacy.cc b/source/common/http/http1/conn_pool_legacy.cc index b0dc47be2528..1c5942b219c4 100644 --- a/source/common/http/http1/conn_pool_legacy.cc +++ b/source/common/http/http1/conn_pool_legacy.cc @@ -15,7 +15,6 @@ #include "common/http/codes.h" #include "common/http/headers.h" #include "common/network/utility.h" -#include "common/runtime/runtime_impl.h" #include "common/stats/timespan_impl.h" #include "common/upstream/upstream_impl.h" diff --git a/source/common/http/http2/BUILD b/source/common/http/http2/BUILD index ffa5191a2675..0790e3ee9cf5 100644 --- a/source/common/http/http2/BUILD +++ b/source/common/http/http2/BUILD @@ -43,7 +43,7 @@ envoy_cc_library( "//source/common/http:header_utility_lib", "//source/common/http:headers_lib", "//source/common/http:utility_lib", - "//source/common/runtime:runtime_lib", + "//source/common/runtime:runtime_features_lib", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], ) diff --git a/source/common/http/http2/codec_impl.cc b/source/common/http/http2/codec_impl.cc index 2d54b0b1a413..14560492e5ae 100644 --- a/source/common/http/http2/codec_impl.cc +++ b/source/common/http/http2/codec_impl.cc @@ -20,7 +20,6 @@ #include "common/http/header_utility.h" #include "common/http/headers.h" #include "common/http/utility.h" -#include "common/runtime/runtime_impl.h" #include "absl/container/fixed_array.h" @@ -71,6 +70,19 @@ bool Utility::reconstituteCrumbledCookies(const HeaderString& key, const HeaderS ConnectionImpl::Http2Callbacks ConnectionImpl::http2_callbacks_; +nghttp2_session* ProdNghttp2SessionFactory::create(const nghttp2_session_callbacks* callbacks, + ConnectionImpl* connection, + const nghttp2_option* options) { + nghttp2_session* session; + nghttp2_session_client_new2(&session, callbacks, connection, options); + return session; +} + +void ProdNghttp2SessionFactory::init(nghttp2_session*, ConnectionImpl* connection, + const envoy::config::core::v3::Http2ProtocolOptions& options) { + connection->sendSettings(options, true); +} + /** * Helper to remove const during a cast. nghttp2 takes non-const pointers for headers even though * it copies them. @@ -820,7 +832,9 @@ int ConnectionImpl::onMetadataFrameComplete(int32_t stream_id, bool end_metadata stream_id, end_metadata); StreamImpl* stream = getStream(stream_id); - ASSERT(stream != nullptr); + if (stream == nullptr) { + return 0; + } bool result = stream->getMetadataDecoder().onMetadataFrameComplete(end_metadata); return result ? 0 : NGHTTP2_ERR_CALLBACK_FAILURE; @@ -830,7 +844,9 @@ ssize_t ConnectionImpl::packMetadata(int32_t stream_id, uint8_t* buf, size_t len ENVOY_CONN_LOG(trace, "pack METADATA frame on stream {}", connection_, stream_id); StreamImpl* stream = getStream(stream_id); - ASSERT(stream != nullptr); + if (stream == nullptr) { + return 0; + } MetadataEncoder& encoder = stream->getMetadataEncoder(); return encoder.packNextFramePayload(buf, len); @@ -1131,14 +1147,15 @@ ConnectionImpl::ClientHttp2Options::ClientHttp2Options( ClientConnectionImpl::ClientConnectionImpl( Network::Connection& connection, Http::ConnectionCallbacks& callbacks, Stats::Scope& stats, const envoy::config::core::v3::Http2ProtocolOptions& http2_options, - const uint32_t max_response_headers_kb, const uint32_t max_response_headers_count) + const uint32_t max_response_headers_kb, const uint32_t max_response_headers_count, + Nghttp2SessionFactory& http2_session_factory) : ConnectionImpl(connection, stats, http2_options, max_response_headers_kb, max_response_headers_count), callbacks_(callbacks) { ClientHttp2Options client_http2_options(http2_options); - nghttp2_session_client_new2(&session_, http2_callbacks_.callbacks(), base(), - client_http2_options.options()); - sendSettings(http2_options, true); + session_ = http2_session_factory.create(http2_callbacks_.callbacks(), base(), + client_http2_options.options()); + http2_session_factory.init(session_, base(), http2_options); allow_metadata_ = http2_options.allow_metadata(); } diff --git a/source/common/http/http2/codec_impl.h b/source/common/http/http2/codec_impl.h index 512f0e65aa73..ed579f6ff7ff 100644 --- a/source/common/http/http2/codec_impl.h +++ b/source/common/http/http2/codec_impl.h @@ -22,7 +22,6 @@ #include "common/http/http2/metadata_decoder.h" #include "common/http/http2/metadata_encoder.h" #include "common/http/utility.h" -#include "common/runtime/runtime_impl.h" #include "absl/types/optional.h" #include "nghttp2/nghttp2.h" @@ -76,6 +75,39 @@ class Utility { HeaderString& cookies); }; +class ConnectionImpl; + +// Abstract nghttp2_session factory. Used to enable injection of factories for testing. +class Nghttp2SessionFactory { +public: + virtual ~Nghttp2SessionFactory() = default; + + // Returns a new nghttp2_session to be used with |connection|. + virtual nghttp2_session* create(const nghttp2_session_callbacks* callbacks, + ConnectionImpl* connection, const nghttp2_option* options) PURE; + + // Initializes the |session|. + virtual void init(nghttp2_session* session, ConnectionImpl* connection, + const envoy::config::core::v3::Http2ProtocolOptions& options) PURE; +}; + +class ProdNghttp2SessionFactory : public Nghttp2SessionFactory { +public: + nghttp2_session* create(const nghttp2_session_callbacks* callbacks, ConnectionImpl* connection, + const nghttp2_option* options) override; + + void init(nghttp2_session* session, ConnectionImpl* connection, + const envoy::config::core::v3::Http2ProtocolOptions& options) override; + + // Returns a global factory instance. Note that this is possible because no internal state is + // maintained; the thread safety of create() and init()'s side effects is guaranteed by Envoy's + // worker based threading model. + static ProdNghttp2SessionFactory& get() { + static ProdNghttp2SessionFactory* instance = new ProdNghttp2SessionFactory(); + return *instance; + } +}; + /** * Base class for HTTP/2 client and server codecs. */ @@ -107,6 +139,8 @@ class ConnectionImpl : public virtual Connection, protected Logger::Loggable; ConnectionImpl* base() { return this; } + // NOTE: Always use non debug nullptr checks against the return value of this function. There are + // edge cases (such as for METADATA frames) where nghttp2 will issue a callback for a stream_id + // that is not associated with an existing stream. StreamImpl* getStream(int32_t stream_id); int saveHeader(const nghttp2_frame* frame, HeaderString&& name, HeaderString&& value); void sendPendingFrames(); @@ -424,6 +461,13 @@ class ConnectionImpl : public virtual Connection, protected Logger::Loggable(NumInternalRedirectsFilterStateName)) { - filter_state.setData(NumInternalRedirectsFilterStateName, - std::make_shared(0), - StreamInfo::FilterState::StateType::Mutable, - StreamInfo::FilterState::LifeSpan::DownstreamRequest); + filter_state.setData( + NumInternalRedirectsFilterStateName, std::make_shared(0), + StreamInfo::FilterState::StateType::Mutable, StreamInfo::FilterState::LifeSpan::Request); } StreamInfo::UInt32Accessor& num_internal_redirect = filter_state.getDataMutable(NumInternalRedirectsFilterStateName); @@ -667,11 +666,12 @@ void Filter::sendNoHealthyUpstreamResponse() { } Http::FilterDataStatus Filter::decodeData(Buffer::Instance& data, bool end_stream) { - // upstream_requests_.size() cannot be 0 because we add to it unconditionally - // in decodeHeaders(). It cannot be > 1 because that only happens when a per + // upstream_requests_.size() cannot be > 1 because that only happens when a per // try timeout occurs with hedge_on_per_try_timeout enabled but the per - // try timeout timer is not started until onUpstreamComplete(). - ASSERT(upstream_requests_.size() == 1); + // try timeout timer is not started until onRequestComplete(). It could be zero + // if the first request attempt has already failed and a retry is waiting for + // a backoff timer. + ASSERT(upstream_requests_.size() <= 1); bool buffering = (retry_state_ && retry_state_->enabled()) || !active_shadow_policies_.empty(); if (buffering && @@ -681,13 +681,31 @@ Http::FilterDataStatus Filter::decodeData(Buffer::Instance& data, bool end_strea retry_state_.reset(); buffering = false; active_shadow_policies_.clear(); + + // If we had to abandon buffering and there's no request in progress, abort the request and + // clean up. This happens if the initial upstream request failed, and we are currently waiting + // for a backoff timer before starting the next upstream attempt. + if (upstream_requests_.empty()) { + cleanup(); + callbacks_->sendLocalReply( + Http::Code::InsufficientStorage, "exceeded request buffer limit while retrying upstream", + modify_headers_, absl::nullopt, + StreamInfo::ResponseCodeDetails::get().RequestPayloadExceededRetryBufferLimit); + return Http::FilterDataStatus::StopIterationNoBuffer; + } } + // If we aren't buffering and there is no active request, an abort should have occurred + // already. + ASSERT(buffering || !upstream_requests_.empty()); + if (buffering) { // If we are going to buffer for retries or shadowing, we need to make a copy before encoding // since it's all moves from here on. - Buffer::OwnedImpl copy(data); - upstream_requests_.front()->encodeData(copy, end_stream); + if (!upstream_requests_.empty()) { + Buffer::OwnedImpl copy(data); + upstream_requests_.front()->encodeData(copy, end_stream); + } // If we are potentially going to retry or shadow this request we need to buffer. // This will not cause the connection manager to 413 because before we hit the @@ -709,11 +727,12 @@ Http::FilterDataStatus Filter::decodeData(Buffer::Instance& data, bool end_strea Http::FilterTrailersStatus Filter::decodeTrailers(Http::RequestTrailerMap& trailers) { ENVOY_STREAM_LOG(debug, "router decoding trailers:\n{}", *callbacks_, trailers); - // upstream_requests_.size() cannot be 0 because we add to it unconditionally - // in decodeHeaders(). It cannot be > 1 because that only happens when a per + // upstream_requests_.size() cannot be > 1 because that only happens when a per // try timeout occurs with hedge_on_per_try_timeout enabled but the per - // try timeout timer is not started until onUpstreamComplete(). - ASSERT(upstream_requests_.size() == 1); + // try timeout timer is not started until onRequestComplete(). It could be zero + // if the first request attempt has already failed and a retry is waiting for + // a backoff timer. + ASSERT(upstream_requests_.size() <= 1); downstream_trailers_ = &trailers; for (auto& upstream_request : upstream_requests_) { upstream_request->encodeTrailers(trailers); @@ -724,8 +743,10 @@ Http::FilterTrailersStatus Filter::decodeTrailers(Http::RequestTrailerMap& trail Http::FilterMetadataStatus Filter::decodeMetadata(Http::MetadataMap& metadata_map) { Http::MetadataMapPtr metadata_map_ptr = std::make_unique(metadata_map); - ASSERT(upstream_requests_.size() == 1); - upstream_requests_.front()->encodeMetadata(std::move(metadata_map_ptr)); + if (!upstream_requests_.empty()) { + // TODO(soya3129): Save metadata for retry, redirect and shadowing case. + upstream_requests_.front()->encodeMetadata(std::move(metadata_map_ptr)); + } return Http::FilterMetadataStatus::Continue; } @@ -869,8 +890,9 @@ void Filter::onSoftPerTryTimeout(UpstreamRequest& upstream_request) { RetryStatus retry_status = retry_state_->shouldHedgeRetryPerTryTimeout([this]() -> void { doRetry(); }); - if (retry_status == RetryStatus::Yes && setupRetry()) { - setupRetry(); + if (retry_status == RetryStatus::Yes) { + pending_retries_++; + // Don't increment upstream_host->stats().rq_error_ here, we'll do that // later if 1) we hit global timeout or 2) we get bad response headers // back. @@ -996,7 +1018,9 @@ bool Filter::maybeRetryReset(Http::StreamResetReason reset_reason, const RetryStatus retry_status = retry_state_->shouldRetryReset(reset_reason, [this]() -> void { doRetry(); }); - if (retry_status == RetryStatus::Yes && setupRetry()) { + if (retry_status == RetryStatus::Yes) { + pending_retries_++; + if (upstream_request.upstreamHost()) { upstream_request.upstreamHost()->stats().rq_error_.inc(); } @@ -1184,19 +1208,18 @@ void Filter::onUpstreamHeaders(uint64_t response_code, Http::ResponseHeaderMapPt } else { const RetryStatus retry_status = retry_state_->shouldRetryHeaders(*headers, [this]() -> void { doRetry(); }); - // Capture upstream_host since setupRetry() in the following line will clear - // upstream_request. - const auto upstream_host = upstream_request.upstreamHost(); - if (retry_status == RetryStatus::Yes && setupRetry()) { + if (retry_status == RetryStatus::Yes) { + pending_retries_++; + upstream_request.upstreamHost()->stats().rq_error_.inc(); + Http::CodeStats& code_stats = httpContext().codeStats(); + code_stats.chargeBasicResponseStat(cluster_->statsScope(), config_.retry_, + static_cast(response_code)); + if (!end_stream) { upstream_request.resetStream(); } upstream_request.removeFromList(upstream_requests_); - Http::CodeStats& code_stats = httpContext().codeStats(); - code_stats.chargeBasicResponseStat(cluster_->statsScope(), config_.retry_, - static_cast(response_code)); - upstream_host->stats().rq_error_.inc(); return; } else if (retry_status == RetryStatus::NoOverflow) { callbacks_->streamInfo().setResponseFlag(StreamInfo::ResponseFlag::UpstreamOverflow); @@ -1375,23 +1398,6 @@ void Filter::onUpstreamComplete(UpstreamRequest& upstream_request) { cleanup(); } -bool Filter::setupRetry() { - // If we responded before the request was complete we don't bother doing a retry. This may not - // catch certain cases where we are in full streaming mode and we have a connect timeout or an - // overflow of some kind. However, in many cases deployments will use the buffer filter before - // this filter which will make this a non-issue. The implementation of supporting retry in cases - // where the request is not complete is more complicated so we will start with this for now. - if (!downstream_end_stream_) { - config_.stats_.rq_retry_skipped_request_not_complete_.inc(); - return false; - } - pending_retries_++; - - ENVOY_STREAM_LOG(debug, "performing retry", *callbacks_); - - return true; -} - bool Filter::setupRedirect(const Http::ResponseHeaderMap& headers, UpstreamRequest& upstream_request) { ENVOY_STREAM_LOG(debug, "attempting internal redirect", *callbacks_); @@ -1412,7 +1418,7 @@ bool Filter::setupRedirect(const Http::ResponseHeaderMap& headers, const StreamInfo::FilterStateSharedPtr& filter_state = callbacks_->streamInfo().filterState(); - // As with setupRetry, redirects are not supported for streaming requests yet. + // Redirects are not supported for streaming requests yet. if (downstream_end_stream_ && !callbacks_->decodingBuffer() && // Redirects with body not yet supported. location != nullptr && @@ -1432,6 +1438,8 @@ bool Filter::setupRedirect(const Http::ResponseHeaderMap& headers, } void Filter::doRetry() { + ENVOY_STREAM_LOG(debug, "performing retry", *callbacks_); + is_retry_ = true; attempt_count_++; ASSERT(pending_retries_ > 0); @@ -1454,10 +1462,10 @@ void Filter::doRetry() { downstream_headers_->setEnvoyAttemptCount(attempt_count_); } - ASSERT(response_timeout_ || timeout_.global_timeout_.count() == 0); UpstreamRequest* upstream_request_tmp = upstream_request.get(); upstream_request->moveIntoList(std::move(upstream_request), upstream_requests_); - upstream_requests_.front()->encodeHeaders(!callbacks_->decodingBuffer() && !downstream_trailers_); + upstream_requests_.front()->encodeHeaders(!callbacks_->decodingBuffer() && + !downstream_trailers_ && downstream_end_stream_); // It's possible we got immediately reset which means the upstream request we just // added to the front of the list might have been removed, so we need to check to make // sure we don't encodeData on the wrong request. @@ -1465,7 +1473,7 @@ void Filter::doRetry() { if (callbacks_->decodingBuffer()) { // If we are doing a retry we need to make a copy. Buffer::OwnedImpl copy(*callbacks_->decodingBuffer()); - upstream_requests_.front()->encodeData(copy, !downstream_trailers_); + upstream_requests_.front()->encodeData(copy, !downstream_trailers_ && downstream_end_stream_); } if (downstream_trailers_) { diff --git a/source/common/router/router.h b/source/common/router/router.h index fb5e52945469..9f532d08fb5f 100644 --- a/source/common/router/router.h +++ b/source/common/router/router.h @@ -46,8 +46,7 @@ namespace Router { COUNTER(rq_redirect) \ COUNTER(rq_direct_response) \ COUNTER(rq_total) \ - COUNTER(rq_reset_after_downstream_response_started) \ - COUNTER(rq_retry_skipped_request_not_complete) + COUNTER(rq_reset_after_downstream_response_started) // clang-format on /** @@ -492,8 +491,6 @@ class Filter : Logger::Loggable, // for the remaining upstream requests to return. void resetOtherUpstreams(UpstreamRequest& upstream_request); void sendNoHealthyUpstreamResponse(); - // TODO(soya3129): Save metadata for retry, redirect and shadowing case. - bool setupRetry(); bool setupRedirect(const Http::ResponseHeaderMap& headers, UpstreamRequest& upstream_request); void updateOutlierDetection(Upstream::Outlier::Result result, UpstreamRequest& upstream_request, absl::optional code); diff --git a/source/common/router/upstream_request.cc b/source/common/router/upstream_request.cc index ac048161c93c..1199e43b239b 100644 --- a/source/common/router/upstream_request.cc +++ b/source/common/router/upstream_request.cc @@ -32,7 +32,6 @@ #include "common/router/config_impl.h" #include "common/router/debug_config.h" #include "common/router/router.h" -#include "common/runtime/runtime_impl.h" #include "common/stream_info/uint32_accessor_impl.h" #include "common/tracing/http_tracer_impl.h" @@ -328,6 +327,8 @@ void UpstreamRequest::onPoolReady( onUpstreamHostSelected(host); + stream_info_.setUpstreamFilterState(std::make_shared( + info.filterState().parent()->parent(), StreamInfo::FilterState::LifeSpan::Request)); stream_info_.setUpstreamLocalAddress(upstream_local_address); parent_.callbacks()->streamInfo().setUpstreamLocalAddress(upstream_local_address); diff --git a/source/common/runtime/BUILD b/source/common/runtime/BUILD index fab542c3dbfa..dbab335cf0a7 100644 --- a/source/common/runtime/BUILD +++ b/source/common/runtime/BUILD @@ -17,8 +17,25 @@ envoy_cc_library( "runtime_features.h", ], deps = [ + # AVOID ADDING TO THESE DEPENDENCIES IF POSSIBLE + # Any code using runtime guards depends on this library, and the more dependencies there are, + # the harder it is to runtime-guard without dependency loops. + "//include/envoy/runtime:runtime_interface", "//source/common/common:hash_lib", "//source/common/singleton:const_singleton", + "//source/common/singleton:threadsafe_singleton", + ], +) + +envoy_cc_library( + name = "runtime_protos_lib", + hdrs = [ + "runtime_protos.h", + ], + deps = [ + "//include/envoy/runtime:runtime_interface", + "@envoy_api//envoy/config/core/v3:pkg_cc_proto", + "@envoy_api//envoy/type/v3:pkg_cc_proto", ], ) @@ -29,11 +46,11 @@ envoy_cc_library( ], hdrs = [ "runtime_impl.h", - "runtime_protos.h", ], external_deps = ["ssl"], deps = [ ":runtime_features_lib", + ":runtime_protos_lib", "//include/envoy/config:subscription_interface", "//include/envoy/event:dispatcher_interface", "//include/envoy/init:manager_interface", diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index 687088335845..5eb9143cc55e 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -1,8 +1,37 @@ #include "common/runtime/runtime_features.h" +#include "absl/strings/match.h" + namespace Envoy { namespace Runtime { +bool isRuntimeFeature(absl::string_view feature) { + return RuntimeFeaturesDefaults::get().enabledByDefault(feature) || + RuntimeFeaturesDefaults::get().existsButDisabled(feature); +} + +bool runtimeFeatureEnabled(absl::string_view feature) { + ASSERT(isRuntimeFeature(feature)); + if (Runtime::LoaderSingleton::getExisting()) { + return Runtime::LoaderSingleton::getExisting()->threadsafeSnapshot()->runtimeFeatureEnabled( + feature); + } + ENVOY_LOG_TO_LOGGER(Envoy::Logger::Registry::getLog(Envoy::Logger::Id::runtime), warn, + "Unable to use runtime singleton for feature {}", feature); + return RuntimeFeaturesDefaults::get().enabledByDefault(feature); +} + +uint64_t getInteger(absl::string_view feature, uint64_t default_value) { + ASSERT(absl::StartsWith(feature, "envoy.")); + if (Runtime::LoaderSingleton::getExisting()) { + return Runtime::LoaderSingleton::getExisting()->threadsafeSnapshot()->getInteger( + std::string(feature), default_value); + } + ENVOY_LOG_TO_LOGGER(Envoy::Logger::Registry::getLog(Envoy::Logger::Id::runtime), warn, + "Unable to use runtime singleton for feature {}", feature); + return default_value; +} + // Add additional features here to enable the new code paths by default. // // Per documentation in CONTRIBUTING.md is expected that new high risk code paths be guarded diff --git a/source/common/runtime/runtime_features.h b/source/common/runtime/runtime_features.h index e8c7b2bc4851..b7d57ded38b7 100644 --- a/source/common/runtime/runtime_features.h +++ b/source/common/runtime/runtime_features.h @@ -2,13 +2,20 @@ #include +#include "envoy/runtime/runtime.h" + #include "common/singleton/const_singleton.h" +#include "common/singleton/threadsafe_singleton.h" #include "absl/container/flat_hash_set.h" namespace Envoy { namespace Runtime { +bool isRuntimeFeature(absl::string_view feature); +bool runtimeFeatureEnabled(absl::string_view feature); +uint64_t getInteger(absl::string_view feature, uint64_t default_value); + class RuntimeFeatures { public: RuntimeFeatures(); diff --git a/source/common/runtime/runtime_impl.cc b/source/common/runtime/runtime_impl.cc index dd9c9182a44d..1ed9faf2c432 100644 --- a/source/common/runtime/runtime_impl.cc +++ b/source/common/runtime/runtime_impl.cc @@ -32,36 +32,6 @@ namespace Envoy { namespace Runtime { -namespace { - -bool isRuntimeFeature(absl::string_view feature) { - return RuntimeFeaturesDefaults::get().enabledByDefault(feature) || - RuntimeFeaturesDefaults::get().existsButDisabled(feature); -} - -} // namespace - -bool runtimeFeatureEnabled(absl::string_view feature) { - ASSERT(isRuntimeFeature(feature)); - if (Runtime::LoaderSingleton::getExisting()) { - return Runtime::LoaderSingleton::getExisting()->threadsafeSnapshot()->runtimeFeatureEnabled( - feature); - } - ENVOY_LOG_TO_LOGGER(Envoy::Logger::Registry::getLog(Envoy::Logger::Id::runtime), warn, - "Unable to use runtime singleton for feature {}", feature); - return RuntimeFeaturesDefaults::get().enabledByDefault(feature); -} - -uint64_t getInteger(absl::string_view feature, uint64_t default_value) { - ASSERT(absl::StartsWith(feature, "envoy.")); - if (Runtime::LoaderSingleton::getExisting()) { - return Runtime::LoaderSingleton::getExisting()->threadsafeSnapshot()->getInteger( - std::string(feature), default_value); - } - ENVOY_LOG_TO_LOGGER(Envoy::Logger::Registry::getLog(Envoy::Logger::Id::runtime), warn, - "Unable to use runtime singleton for feature {}", feature); - return default_value; -} const size_t RandomGeneratorImpl::UUID_LENGTH = 36; diff --git a/source/common/runtime/runtime_impl.h b/source/common/runtime/runtime_impl.h index 7e3443b21daf..8fa838e88cd8 100644 --- a/source/common/runtime/runtime_impl.h +++ b/source/common/runtime/runtime_impl.h @@ -32,9 +32,6 @@ namespace Envoy { namespace Runtime { -bool runtimeFeatureEnabled(absl::string_view feature); -uint64_t getInteger(absl::string_view feature, uint64_t default_value); - using RuntimeSingleton = ThreadSafeSingleton; /** diff --git a/source/common/stats/BUILD b/source/common/stats/BUILD index c720e2ef92a1..256074df9cbf 100644 --- a/source/common/stats/BUILD +++ b/source/common/stats/BUILD @@ -48,6 +48,7 @@ envoy_cc_library( ":histogram_lib", ":null_counter_lib", ":null_gauge_lib", + ":null_text_readout_lib", ":scope_prefixer_lib", ":stats_lib", ":store_impl_lib", @@ -100,6 +101,16 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "null_text_readout_lib", + hdrs = ["null_text_readout.h"], + deps = [ + ":metric_impl_lib", + ":symbol_table_lib", + "//include/envoy/stats:stats_interface", + ], +) + envoy_cc_library( name = "recent_lookups_lib", srcs = ["recent_lookups.cc"], @@ -240,6 +251,7 @@ envoy_cc_library( ":allocator_lib", ":null_counter_lib", ":null_gauge_lib", + ":null_text_readout_lib", ":scope_prefixer_lib", ":stats_lib", ":stats_matcher_lib", diff --git a/source/common/stats/allocator_impl.cc b/source/common/stats/allocator_impl.cc index 3286aeb7834e..06db3ee37f52 100644 --- a/source/common/stats/allocator_impl.cc +++ b/source/common/stats/allocator_impl.cc @@ -39,7 +39,7 @@ void AllocatorImpl::debugPrint() { } #endif -// Counter and Gauge both inherit from from RefcountInterface and +// Counter, Gauge and TextReadout inherit from RefcountInterface and // Metric. MetricImpl takes care of most of the Metric API, but we need to cover // symbolTable() here, which we don't store directly, but get it via the alloc, // which we need in order to clean up the counter and gauge maps in that class @@ -101,10 +101,6 @@ template class StatsSharedImpl : public MetricImpl protected: AllocatorImpl& alloc_; - // Holds backing store shared by both CounterImpl and GaugeImpl. CounterImpl - // adds another field, pending_increment_, that is not used in Gauge. - std::atomic value_{0}; - // ref_count_ can be incremented as an atomic, without taking a new lock, as // the critical 0->1 transition occurs in makeCounter and makeGauge, which // already hold the lock. Increment also occurs when copying shared pointers, @@ -144,6 +140,7 @@ class CounterImpl : public StatsSharedImpl { uint64_t value() const override { return value_; } private: + std::atomic value_{0}; std::atomic pending_increment_{0}; }; @@ -226,12 +223,42 @@ class GaugeImpl : public StatsSharedImpl { break; } } + +private: + std::atomic value_{0}; +}; + +class TextReadoutImpl : public StatsSharedImpl { +public: + TextReadoutImpl(StatName name, AllocatorImpl& alloc, StatName tag_extracted_name, + const StatNameTagVector& stat_name_tags) + : StatsSharedImpl(name, alloc, tag_extracted_name, stat_name_tags) {} + + void removeFromSetLockHeld() EXCLUSIVE_LOCKS_REQUIRED(alloc_.mutex_) override { + const size_t count = alloc_.text_readouts_.erase(statName()); + ASSERT(count == 1); + } + + // Stats::TextReadout + void set(std::string&& value) override { + absl::MutexLock lock(&mutex_); + value_ = std::move(value); + } + std::string value() const override { + absl::MutexLock lock(&mutex_); + return value_; + } + +private: + mutable absl::Mutex mutex_; + std::string value_ ABSL_GUARDED_BY(mutex_); }; CounterSharedPtr AllocatorImpl::makeCounter(StatName name, StatName tag_extracted_name, const StatNameTagVector& stat_name_tags) { Thread::LockGuard lock(mutex_); ASSERT(gauges_.find(name) == gauges_.end()); + ASSERT(text_readouts_.find(name) == text_readouts_.end()); auto iter = counters_.find(name); if (iter != counters_.end()) { return CounterSharedPtr(*iter); @@ -246,6 +273,7 @@ GaugeSharedPtr AllocatorImpl::makeGauge(StatName name, StatName tag_extracted_na Gauge::ImportMode import_mode) { Thread::LockGuard lock(mutex_); ASSERT(counters_.find(name) == counters_.end()); + ASSERT(text_readouts_.find(name) == text_readouts_.end()); auto iter = gauges_.find(name); if (iter != gauges_.end()) { return GaugeSharedPtr(*iter); @@ -256,6 +284,21 @@ GaugeSharedPtr AllocatorImpl::makeGauge(StatName name, StatName tag_extracted_na return gauge; } +TextReadoutSharedPtr AllocatorImpl::makeTextReadout(StatName name, StatName tag_extracted_name, + const StatNameTagVector& stat_name_tags) { + Thread::LockGuard lock(mutex_); + ASSERT(counters_.find(name) == counters_.end()); + ASSERT(gauges_.find(name) == gauges_.end()); + auto iter = text_readouts_.find(name); + if (iter != text_readouts_.end()) { + return TextReadoutSharedPtr(*iter); + } + auto text_readout = + TextReadoutSharedPtr(new TextReadoutImpl(name, *this, tag_extracted_name, stat_name_tags)); + text_readouts_.insert(text_readout.get()); + return text_readout; +} + bool AllocatorImpl::isMutexLockedForTest() { bool locked = mutex_.tryLock(); if (locked) { diff --git a/source/common/stats/allocator_impl.h b/source/common/stats/allocator_impl.h index 2c4ba307f797..02e926529358 100644 --- a/source/common/stats/allocator_impl.h +++ b/source/common/stats/allocator_impl.h @@ -28,6 +28,8 @@ class AllocatorImpl : public Allocator { GaugeSharedPtr makeGauge(StatName name, StatName tag_extracted_name, const StatNameTagVector& stat_name_tags, Gauge::ImportMode import_mode) override; + TextReadoutSharedPtr makeTextReadout(StatName name, StatName tag_extracted_name, + const StatNameTagVector& stat_name_tags) override; SymbolTable& symbolTable() override { return symbol_table_; } const SymbolTable& constSymbolTable() const override { return symbol_table_; } @@ -49,6 +51,7 @@ class AllocatorImpl : public Allocator { template friend class StatsSharedImpl; friend class CounterImpl; friend class GaugeImpl; + friend class TextReadoutImpl; struct HeapStatHash { using is_transparent = void; // NOLINT(readability-identifier-naming) @@ -66,6 +69,7 @@ class AllocatorImpl : public Allocator { void removeCounterFromSetLockHeld(Counter* counter) EXCLUSIVE_LOCKS_REQUIRED(mutex_); void removeGaugeFromSetLockHeld(Gauge* gauge) EXCLUSIVE_LOCKS_REQUIRED(mutex_); + void removeTextReadoutFromSetLockHeld(Counter* counter) EXCLUSIVE_LOCKS_REQUIRED(mutex_); // An unordered set of HeapStatData pointers which keys off the key() // field in each object. This necessitates a custom comparator and hasher, which key off of the @@ -74,6 +78,7 @@ class AllocatorImpl : public Allocator { using StatSet = absl::flat_hash_set; StatSet counters_ GUARDED_BY(mutex_); StatSet gauges_ GUARDED_BY(mutex_); + StatSet text_readouts_ GUARDED_BY(mutex_); SymbolTable& symbol_table_; diff --git a/source/common/stats/isolated_store_impl.cc b/source/common/stats/isolated_store_impl.cc index c3600c98d815..d9511916e844 100644 --- a/source/common/stats/isolated_store_impl.cc +++ b/source/common/stats/isolated_store_impl.cc @@ -32,6 +32,9 @@ IsolatedStoreImpl::IsolatedStoreImpl(SymbolTable& symbol_table) histograms_([this](StatName name, Histogram::Unit unit) -> HistogramSharedPtr { return HistogramSharedPtr(new HistogramImpl(name, unit, *this, name, StatNameTagVector{})); }), + text_readouts_([this](StatName name, TextReadout::Type) -> TextReadoutSharedPtr { + return alloc_.makeTextReadout(name, name, StatNameTagVector{}); + }), null_counter_(new NullCounterImpl(symbol_table)), null_gauge_(new NullGaugeImpl(symbol_table)) {} diff --git a/source/common/stats/isolated_store_impl.h b/source/common/stats/isolated_store_impl.h index 987617a9fe90..2427e71a1b31 100644 --- a/source/common/stats/isolated_store_impl.h +++ b/source/common/stats/isolated_store_impl.h @@ -29,11 +29,13 @@ template class IsolatedStatsCache { using CounterAllocator = std::function(StatName name)>; using GaugeAllocator = std::function(StatName, Gauge::ImportMode)>; using HistogramAllocator = std::function(StatName, Histogram::Unit)>; + using TextReadoutAllocator = std::function(StatName name, TextReadout::Type)>; using BaseOptConstRef = absl::optional>; IsolatedStatsCache(CounterAllocator alloc) : counter_alloc_(alloc) {} IsolatedStatsCache(GaugeAllocator alloc) : gauge_alloc_(alloc) {} IsolatedStatsCache(HistogramAllocator alloc) : histogram_alloc_(alloc) {} + IsolatedStatsCache(TextReadoutAllocator alloc) : text_readout_alloc_(alloc) {} Base& get(StatName name) { auto stat = stats_.find(name); @@ -68,6 +70,17 @@ template class IsolatedStatsCache { return *new_stat; } + Base& get(StatName name, TextReadout::Type type) { + auto stat = stats_.find(name); + if (stat != stats_.end()) { + return *stat->second; + } + + RefcountPtr new_stat = text_readout_alloc_(name, type); + stats_.emplace(new_stat->statName(), new_stat); + return *new_stat; + } + std::vector> toVector() const { std::vector> vec; vec.reserve(stats_.size()); @@ -93,6 +106,7 @@ template class IsolatedStatsCache { CounterAllocator counter_alloc_; GaugeAllocator gauge_alloc_; HistogramAllocator histogram_alloc_; + TextReadoutAllocator text_readout_alloc_; }; class IsolatedStoreImpl : public StoreImpl { @@ -124,11 +138,21 @@ class IsolatedStoreImpl : public StoreImpl { Histogram& histogram = histograms_.get(joiner.nameWithTags(), unit); return histogram; } + TextReadout& textReadoutFromStatNameWithTags(const StatName& name, + StatNameTagVectorOptConstRef tags) override { + TagUtility::TagStatNameJoiner joiner(name, tags, symbolTable()); + TextReadout& text_readout = + text_readouts_.get(joiner.nameWithTags(), TextReadout::Type::Default); + return text_readout; + } CounterOptConstRef findCounter(StatName name) const override { return counters_.find(name); } GaugeOptConstRef findGauge(StatName name) const override { return gauges_.find(name); } HistogramOptConstRef findHistogram(StatName name) const override { return histograms_.find(name); } + TextReadoutOptConstRef findTextReadout(StatName name) const override { + return text_readouts_.find(name); + } // Stats::Store std::vector counters() const override { return counters_.toVector(); } @@ -143,6 +167,9 @@ class IsolatedStoreImpl : public StoreImpl { std::vector histograms() const override { return std::vector{}; } + std::vector textReadouts() const override { + return text_readouts_.toVector(); + } Counter& counterFromString(const std::string& name) override { StatNameManagedStorage storage(name, symbolTable()); @@ -156,6 +183,10 @@ class IsolatedStoreImpl : public StoreImpl { StatNameManagedStorage storage(name, symbolTable()); return histogramFromStatName(storage.statName(), unit); } + TextReadout& textReadoutFromString(const std::string& name) override { + StatNameManagedStorage storage(name, symbolTable()); + return textReadoutFromStatName(storage.statName()); + } private: IsolatedStoreImpl(std::unique_ptr&& symbol_table); @@ -165,6 +196,7 @@ class IsolatedStoreImpl : public StoreImpl { IsolatedStatsCache counters_; IsolatedStatsCache gauges_; IsolatedStatsCache histograms_; + IsolatedStatsCache text_readouts_; RefcountPtr null_counter_; RefcountPtr null_gauge_; }; diff --git a/source/common/stats/null_text_readout.h b/source/common/stats/null_text_readout.h new file mode 100644 index 000000000000..d3e9cc832e6b --- /dev/null +++ b/source/common/stats/null_text_readout.h @@ -0,0 +1,44 @@ +#pragma once + +#include "envoy/stats/stats.h" + +#include "common/stats/metric_impl.h" + +namespace Envoy { +namespace Stats { + +/** + * Null text readout implementation. + * No-ops on all calls and requires no underlying metric or data. + */ +class NullTextReadoutImpl : public MetricImpl { +public: + explicit NullTextReadoutImpl(SymbolTable& symbol_table) + : MetricImpl(symbol_table), symbol_table_(symbol_table) {} + ~NullTextReadoutImpl() override { + // MetricImpl must be explicitly cleared() before destruction, otherwise it + // will not be able to access the SymbolTable& to free the symbols. An RAII + // alternative would be to store the SymbolTable reference in the + // MetricImpl, costing 8 bytes per stat. + MetricImpl::clear(symbol_table_); + } + + void set(std::string&&) override {} + std::string value() const override { return std::string(); } + + // Metric + bool used() const override { return false; } + SymbolTable& symbolTable() override { return symbol_table_; } + + // RefcountInterface + void incRefCount() override { refcount_helper_.incRefCount(); } + bool decRefCount() override { return refcount_helper_.decRefCount(); } + uint32_t use_count() const override { return refcount_helper_.use_count(); } + +private: + RefcountHelper refcount_helper_; + SymbolTable& symbol_table_; +}; + +} // namespace Stats +} // namespace Envoy diff --git a/source/common/stats/scope_prefixer.cc b/source/common/stats/scope_prefixer.cc index fc687c7b2341..5da4bb3686e4 100644 --- a/source/common/stats/scope_prefixer.cc +++ b/source/common/stats/scope_prefixer.cc @@ -49,6 +49,13 @@ Histogram& ScopePrefixer::histogramFromStatNameWithTags(const StatName& name, return scope_.histogramFromStatNameWithTags(StatName(stat_name_storage.get()), tags, unit); } +TextReadout& ScopePrefixer::textReadoutFromStatNameWithTags(const StatName& name, + StatNameTagVectorOptConstRef tags) { + Stats::SymbolTable::StoragePtr stat_name_storage = + scope_.symbolTable().join({prefix_.statName(), name}); + return scope_.textReadoutFromStatNameWithTags(StatName(stat_name_storage.get()), tags); +} + CounterOptConstRef ScopePrefixer::findCounter(StatName name) const { return scope_.findCounter(name); } @@ -59,6 +66,10 @@ HistogramOptConstRef ScopePrefixer::findHistogram(StatName name) const { return scope_.findHistogram(name); } +TextReadoutOptConstRef ScopePrefixer::findTextReadout(StatName name) const { + return scope_.findTextReadout(name); +} + void ScopePrefixer::deliverHistogramToSinks(const Histogram& histograms, uint64_t val) { scope_.deliverHistogramToSinks(histograms, val); } diff --git a/source/common/stats/scope_prefixer.h b/source/common/stats/scope_prefixer.h index 948de23fba2d..b7c874375620 100644 --- a/source/common/stats/scope_prefixer.h +++ b/source/common/stats/scope_prefixer.h @@ -23,6 +23,8 @@ class ScopePrefixer : public Scope { Gauge::ImportMode import_mode) override; Histogram& histogramFromStatNameWithTags(const StatName& name, StatNameTagVectorOptConstRef tags, Histogram::Unit unit) override; + TextReadout& textReadoutFromStatNameWithTags(const StatName& name, + StatNameTagVectorOptConstRef tags) override; void deliverHistogramToSinks(const Histogram& histograms, uint64_t val) override; Counter& counterFromString(const std::string& name) override { @@ -37,10 +39,15 @@ class ScopePrefixer : public Scope { StatNameManagedStorage storage(name, symbolTable()); return Scope::histogramFromStatName(storage.statName(), unit); } + TextReadout& textReadoutFromString(const std::string& name) override { + StatNameManagedStorage storage(name, symbolTable()); + return Scope::textReadoutFromStatName(storage.statName()); + } CounterOptConstRef findCounter(StatName name) const override; GaugeOptConstRef findGauge(StatName name) const override; HistogramOptConstRef findHistogram(StatName name) const override; + TextReadoutOptConstRef findTextReadout(StatName name) const override; const SymbolTable& constSymbolTable() const override { return scope_.constSymbolTable(); } SymbolTable& symbolTable() override { return scope_.symbolTable(); } diff --git a/source/common/stats/thread_local_store.cc b/source/common/stats/thread_local_store.cc index 2d3c9645cd55..5a9a47f912fa 100644 --- a/source/common/stats/thread_local_store.cc +++ b/source/common/stats/thread_local_store.cc @@ -28,7 +28,7 @@ ThreadLocalStoreImpl::ThreadLocalStoreImpl(Allocator& alloc) tag_producer_(std::make_unique()), stats_matcher_(std::make_unique()), heap_allocator_(alloc.symbolTable()), null_counter_(alloc.symbolTable()), null_gauge_(alloc.symbolTable()), - null_histogram_(alloc.symbolTable()) {} + null_histogram_(alloc.symbolTable()), null_text_readout_(alloc.symbolTable()) {} ThreadLocalStoreImpl::~ThreadLocalStoreImpl() { ASSERT(shutting_down_ || !threading_ever_initialized_); @@ -51,6 +51,7 @@ void ThreadLocalStoreImpl::setStatsMatcher(StatsMatcherPtr&& stats_matcher) { removeRejectedStats(scope->central_cache_->counters_, deleted_counters_); removeRejectedStats(scope->central_cache_->gauges_, deleted_gauges_); removeRejectedStats(scope->central_cache_->histograms_, deleted_histograms_); + removeRejectedStats(scope->central_cache_->text_readouts_, deleted_text_readouts_); } } @@ -129,6 +130,22 @@ std::vector ThreadLocalStoreImpl::gauges() const { return ret; } +std::vector ThreadLocalStoreImpl::textReadouts() const { + // Handle de-dup due to overlapping scopes. + std::vector ret; + StatNameHashSet names; + Thread::LockGuard lock(lock_); + for (ScopeImpl* scope : scopes_) { + for (auto& text_readout : scope->central_cache_->text_readouts_) { + if (names.insert(text_readout.first).second) { + ret.push_back(text_readout.second); + } + } + } + + return ret; +} + std::vector ThreadLocalStoreImpl::histograms() const { std::vector ret; Thread::LockGuard lock(lock_); @@ -538,6 +555,44 @@ Histogram& ThreadLocalStoreImpl::ScopeImpl::histogramFromStatNameWithTags( return **central_ref; } +TextReadout& ThreadLocalStoreImpl::ScopeImpl::textReadoutFromStatNameWithTags( + const StatName& name, StatNameTagVectorOptConstRef stat_name_tags) { + if (parent_.rejectsAll()) { + return parent_.null_text_readout_; + } + + // Determine the final name based on the prefix and the passed name. + // + // Note that we can do map.find(final_name.c_str()), but we cannot do + // map[final_name.c_str()] as the char*-keyed maps would then save the pointer + // to a temporary, and address sanitization errors would follow. Instead we + // must do a find() first, using the value if it succeeds. If it fails, then + // after we construct the stat we can insert it into the required maps. This + // strategy costs an extra hash lookup for each miss, but saves time + // re-copying the string and significant memory overhead. + TagUtility::TagStatNameJoiner joiner(prefix_.statName(), name, stat_name_tags, symbolTable()); + Stats::StatName final_stat_name = joiner.nameWithTags(); + + // We now find the TLS cache. This might remain null if we don't have TLS + // initialized currently. + StatRefMap* tls_cache = nullptr; + StatNameHashSet* tls_rejected_stats = nullptr; + if (!parent_.shutting_down_ && parent_.tls_) { + TlsCacheEntry& entry = parent_.tls_->getTyped().insertScope(this->scope_id_); + tls_cache = &entry.text_readouts_; + tls_rejected_stats = &entry.rejected_stats_; + } + + return safeMakeStat( + final_stat_name, joiner.tagExtractedName(), stat_name_tags, central_cache_->text_readouts_, + central_cache_->rejected_stats_, + [](Allocator& allocator, StatName name, StatName tag_extracted_name, + const StatNameTagVector& tags) -> TextReadoutSharedPtr { + return allocator.makeTextReadout(name, tag_extracted_name, tags); + }, + tls_cache, tls_rejected_stats, parent_.null_text_readout_); +} + CounterOptConstRef ThreadLocalStoreImpl::ScopeImpl::findCounter(StatName name) const { return findStatLockHeld(name, central_cache_->counters_); } @@ -556,6 +611,10 @@ HistogramOptConstRef ThreadLocalStoreImpl::ScopeImpl::findHistogram(StatName nam return std::cref(*histogram_ref); } +TextReadoutOptConstRef ThreadLocalStoreImpl::ScopeImpl::findTextReadout(StatName name) const { + return findStatLockHeld(name, central_cache_->text_readouts_); +} + Histogram& ThreadLocalStoreImpl::ScopeImpl::tlsHistogram(StatName name, ParentHistogramImpl& parent) { // tlsHistogram() is generally not called for a histogram that is rejected by diff --git a/source/common/stats/thread_local_store.h b/source/common/stats/thread_local_store.h index d7ba7a5d346f..135abeb257e4 100644 --- a/source/common/stats/thread_local_store.h +++ b/source/common/stats/thread_local_store.h @@ -15,6 +15,7 @@ #include "common/stats/histogram_impl.h" #include "common/stats/null_counter.h" #include "common/stats/null_gauge.h" +#include "common/stats/null_text_readout.h" #include "common/stats/symbol_table_impl.h" #include "common/stats/utility.h" @@ -184,6 +185,13 @@ class ThreadLocalStoreImpl : Logger::Loggable, public StoreRo Histogram& histogramFromString(const std::string& name, Histogram::Unit unit) override { return default_scope_->histogramFromString(name, unit); } + TextReadout& textReadoutFromStatNameWithTags(const StatName& name, + StatNameTagVectorOptConstRef tags) override { + return default_scope_->textReadoutFromStatNameWithTags(name, tags); + } + TextReadout& textReadoutFromString(const std::string& name) override { + return default_scope_->textReadoutFromString(name); + } NullGaugeImpl& nullGauge(const std::string&) override { return null_gauge_; } const SymbolTable& constSymbolTable() const override { return alloc_.constSymbolTable(); } SymbolTable& symbolTable() override { return alloc_.symbolTable(); } @@ -221,9 +229,22 @@ class ThreadLocalStoreImpl : Logger::Loggable, public StoreRo } return absl::nullopt; } + TextReadoutOptConstRef findTextReadout(StatName name) const override { + TextReadoutOptConstRef found_text_readout; + Thread::LockGuard lock(lock_); + for (ScopeImpl* scope : scopes_) { + found_text_readout = scope->findTextReadout(name); + if (found_text_readout.has_value()) { + return found_text_readout; + } + } + return absl::nullopt; + } + // Stats::Store std::vector counters() const override; std::vector gauges() const override; + std::vector textReadouts() const override; std::vector histograms() const override; // Stats::StoreRoot @@ -246,12 +267,13 @@ class ThreadLocalStoreImpl : Logger::Loggable, public StoreRo template using StatRefMap = StatNameHashMap>; struct TlsCacheEntry { - // The counters and gauges in the TLS cache are stored by reference, + // The counters, gauges and text readouts in the TLS cache are stored by reference, // depending on the CentralCache for backing store. This avoids a potential // contention-storm when destructing a scope, as the counter/gauge ref-count // decrement in allocator_impl.cc needs to hold the single allocator mutex. StatRefMap counters_; StatRefMap gauges_; + StatRefMap text_readouts_; // The histogram objects are not shared with the central cache, and don't // require taking a lock when decrementing their ref-count. @@ -274,6 +296,7 @@ class ThreadLocalStoreImpl : Logger::Loggable, public StoreRo StatNameHashMap counters_; StatNameHashMap gauges_; StatNameHashMap histograms_; + StatNameHashMap text_readouts_; StatNameStorageSet rejected_stats_; SymbolTable& symbol_table_; }; @@ -293,6 +316,8 @@ class ThreadLocalStoreImpl : Logger::Loggable, public StoreRo StatNameTagVectorOptConstRef tags, Histogram::Unit unit) override; Histogram& tlsHistogram(StatName name, ParentHistogramImpl& parent) override; + TextReadout& textReadoutFromStatNameWithTags(const StatName& name, + StatNameTagVectorOptConstRef tags) override; ScopePtr createScope(const std::string& name) override { return parent_.createScope(symbolTable().toString(prefix_.statName()) + "." + name); } @@ -311,6 +336,10 @@ class ThreadLocalStoreImpl : Logger::Loggable, public StoreRo StatNameManagedStorage storage(name, symbolTable()); return histogramFromStatName(storage.statName(), unit); } + TextReadout& textReadoutFromString(const std::string& name) override { + StatNameManagedStorage storage(name, symbolTable()); + return textReadoutFromStatName(storage.statName()); + } NullGaugeImpl& nullGauge(const std::string&) override { return parent_.null_gauge_; } @@ -319,6 +348,7 @@ class ThreadLocalStoreImpl : Logger::Loggable, public StoreRo CounterOptConstRef findCounter(StatName name) const override; GaugeOptConstRef findGauge(StatName name) const override; HistogramOptConstRef findHistogram(StatName name) const override; + TextReadoutOptConstRef findTextReadout(StatName name) const override; template using MakeStatFn = std::function( @@ -407,6 +437,7 @@ class ThreadLocalStoreImpl : Logger::Loggable, public StoreRo NullCounterImpl null_counter_; NullGaugeImpl null_gauge_; NullHistogramImpl null_histogram_; + NullTextReadoutImpl null_text_readout_; // Retain storage for deleted stats; these are no longer in maps because the // matcher-pattern was established after they were created. Since the stats @@ -419,6 +450,7 @@ class ThreadLocalStoreImpl : Logger::Loggable, public StoreRo std::vector deleted_counters_; std::vector deleted_gauges_; std::vector deleted_histograms_; + std::vector deleted_text_readouts_; Thread::ThreadSynchronizer sync_; std::atomic next_scope_id_{}; diff --git a/source/common/stream_info/stream_info_impl.h b/source/common/stream_info/stream_info_impl.h index d181153f9a11..eb9a672a9e36 100644 --- a/source/common/stream_info/stream_info_impl.h +++ b/source/common/stream_info/stream_info_impl.h @@ -36,7 +36,7 @@ struct StreamInfoImpl : public StreamInfo { start_time_monotonic_(time_source.monotonicTime()), protocol_(protocol), filter_state_(std::make_shared( FilterStateImpl::LazyCreateAncestor(parent_filter_state, - FilterState::LifeSpan::DownstreamConnection), + FilterState::LifeSpan::Connection), FilterState::LifeSpan::FilterChain)), request_id_extension_(Http::RequestIDExtensionFactory::noopInstance()) {} diff --git a/source/docs/stats.md b/source/docs/stats.md index 023e865a0a3a..43be6992146c 100644 --- a/source/docs/stats.md +++ b/source/docs/stats.md @@ -8,6 +8,8 @@ binary program restarts. The metrics are tracked as: * Histograms: mapping ranges of values to frequency. The ranges are auto-adjusted as data accumulates. Unlike counters and gauges, histogram data is not retained across binary program restarts. + * TextReadouts: Unicode strings. Unlike counters and gauges, text readout data + is not retained across binary program restarts. In order to support restarting the Envoy binary program without losing counter and gauge values, they are passed from parent to child in an RPC protocol. @@ -79,7 +81,7 @@ followed. Stat names are replicated in several places in various forms. - * Held with the stat values, in `CounterImpl` and `GaugeImpl`, which are defined in + * Held with the stat values, in `CounterImpl`, `GaugeImpl` and `TextReadoutImpl`, which are defined in [allocator_impl.cc](https://github.com/envoyproxy/envoy/blob/master/source/common/stats/allocator_impl.cc) * In [MetricImpl](https://github.com/envoyproxy/envoy/blob/master/source/common/stats/metric_impl.h) in a transformed state, with tags extracted into vectors of name/value strings. diff --git a/source/extensions/clusters/dynamic_forward_proxy/cluster.cc b/source/extensions/clusters/dynamic_forward_proxy/cluster.cc index 657bb45753a6..c671aeb3c795 100644 --- a/source/extensions/clusters/dynamic_forward_proxy/cluster.cc +++ b/source/extensions/clusters/dynamic_forward_proxy/cluster.cc @@ -165,12 +165,22 @@ void Cluster::onDnsHostRemove(const std::string& host) { Upstream::HostConstSharedPtr Cluster::LoadBalancer::chooseHost(Upstream::LoadBalancerContext* context) { - if (!context || !context->downstreamHeaders()) { + if (!context) { return nullptr; } - const auto host_it = - host_map_->find(context->downstreamHeaders()->Host()->value().getStringView()); + absl::string_view host; + if (context->downstreamHeaders()) { + host = context->downstreamHeaders()->Host()->value().getStringView(); + } else if (context->downstreamConnection()) { + host = context->downstreamConnection()->requestedServerName(); + } + + if (host.empty()) { + return nullptr; + } + + const auto host_it = host_map_->find(host); if (host_it == host_map_->end()) { return nullptr; } else { diff --git a/source/extensions/common/tap/extension_config_base.cc b/source/extensions/common/tap/extension_config_base.cc index 67d12e2d1193..fda84e29fbeb 100644 --- a/source/extensions/common/tap/extension_config_base.cc +++ b/source/extensions/common/tap/extension_config_base.cc @@ -28,7 +28,15 @@ ExtensionConfigBase::ExtensionConfigBase( break; } case envoy::extensions::common::tap::v3::CommonExtensionConfig::ConfigTypeCase::kStaticConfig: { - newTapConfig(envoy::config::tap::v3::TapConfig(proto_config_.static_config()), nullptr); + // Right now only one sink is supported. + ASSERT(proto_config_.static_config().output_config().sinks().size() == 1); + if (proto_config_.static_config().output_config().sinks()[0].output_sink_type_case() == + envoy::config::tap::v3::OutputSink::OutputSinkTypeCase::kStreamingAdmin) { + // Require that users do not specify a streaming admin with static configuration. + throw EnvoyException( + fmt::format("Error: Specifying admin streaming output without configuring admin.")); + } + installNewTap(envoy::config::tap::v3::TapConfig(proto_config_.static_config()), nullptr); ENVOY_LOG(debug, "initializing tap extension with static config"); break; } @@ -58,14 +66,19 @@ void ExtensionConfigBase::clearTapConfig() { tls_slot_->runOnAllThreads([this] { tls_slot_->getTyped().config_ = nullptr; }); } -void ExtensionConfigBase::newTapConfig(envoy::config::tap::v3::TapConfig&& proto_config, - Sink* admin_streamer) { +void ExtensionConfigBase::installNewTap(envoy::config::tap::v3::TapConfig&& proto_config, + Sink* admin_streamer) { TapConfigSharedPtr new_config = config_factory_->createConfigFromProto(std::move(proto_config), admin_streamer); tls_slot_->runOnAllThreads( [this, new_config] { tls_slot_->getTyped().config_ = new_config; }); } +void ExtensionConfigBase::newTapConfig(envoy::config::tap::v3::TapConfig&& proto_config, + Sink* admin_streamer) { + installNewTap(envoy::config::tap::v3::TapConfig(proto_config), admin_streamer); +} + } // namespace Tap } // namespace Common } // namespace Extensions diff --git a/source/extensions/common/tap/extension_config_base.h b/source/extensions/common/tap/extension_config_base.h index 191483ebcd40..d2745a6a9193 100644 --- a/source/extensions/common/tap/extension_config_base.h +++ b/source/extensions/common/tap/extension_config_base.h @@ -38,6 +38,10 @@ class ExtensionConfigBase : public ExtensionConfig, Logger::Loggableget(header_name_); + if (header == nullptr) { + return percentage_; + } + + uint32_t header_numerator; + if (!absl::SimpleAtoi(header->value().getStringView(), &header_numerator)) { + return percentage_; + } + + envoy::type::v3::FractionalPercent result; + result.set_numerator(std::min(header_numerator, percentage_.numerator())); + result.set_denominator(percentage_.denominator()); + return result; +} + FaultAbortConfig::FaultAbortConfig( - const envoy::extensions::filters::http::fault::v3::FaultAbort& abort_config) - : percentage_(abort_config.percentage()) { + const envoy::extensions::filters::http::fault::v3::FaultAbort& abort_config) { switch (abort_config.error_type_case()) { case envoy::extensions::filters::http::fault::v3::FaultAbort::ErrorTypeCase::kHttpStatus: - provider_ = std::make_unique(abort_config.http_status()); + provider_ = + std::make_unique(abort_config.http_status(), abort_config.percentage()); break; case envoy::extensions::filters::http::fault::v3::FaultAbort::ErrorTypeCase::kHeaderAbort: - provider_ = std::make_unique(); + provider_ = std::make_unique(abort_config.percentage()); break; case envoy::extensions::filters::http::fault::v3::FaultAbort::ErrorTypeCase::ERROR_TYPE_NOT_SET: NOT_REACHED_GCOVR_EXCL_LINE; } } -absl::optional -FaultAbortConfig::HeaderAbortProvider::statusCode(const Http::HeaderEntry* header) const { +absl::optional FaultAbortConfig::HeaderAbortProvider::statusCode( + const Http::RequestHeaderMap* request_headers) const { absl::optional ret; + const auto header = request_headers->get(HeaderNames::get().AbortRequest); if (header == nullptr) { return ret; } @@ -46,17 +65,17 @@ FaultAbortConfig::HeaderAbortProvider::statusCode(const Http::HeaderEntry* heade } FaultDelayConfig::FaultDelayConfig( - const envoy::extensions::filters::common::fault::v3::FaultDelay& delay_config) - : percentage_(delay_config.percentage()) { + const envoy::extensions::filters::common::fault::v3::FaultDelay& delay_config) { switch (delay_config.fault_delay_secifier_case()) { case envoy::extensions::filters::common::fault::v3::FaultDelay::FaultDelaySecifierCase:: kFixedDelay: provider_ = std::make_unique( - std::chrono::milliseconds(PROTOBUF_GET_MS_REQUIRED(delay_config, fixed_delay))); + std::chrono::milliseconds(PROTOBUF_GET_MS_REQUIRED(delay_config, fixed_delay)), + delay_config.percentage()); break; case envoy::extensions::filters::common::fault::v3::FaultDelay::FaultDelaySecifierCase:: kHeaderDelay: - provider_ = std::make_unique(); + provider_ = std::make_unique(delay_config.percentage()); break; case envoy::extensions::filters::common::fault::v3::FaultDelay::FaultDelaySecifierCase:: FAULT_DELAY_SECIFIER_NOT_SET: @@ -64,8 +83,9 @@ FaultDelayConfig::FaultDelayConfig( } } -absl::optional -FaultDelayConfig::HeaderDelayProvider::duration(const Http::HeaderEntry* header) const { +absl::optional FaultDelayConfig::HeaderDelayProvider::duration( + const Http::RequestHeaderMap* request_headers) const { + const auto header = request_headers->get(HeaderNames::get().DelayRequest); if (header == nullptr) { return absl::nullopt; } @@ -79,15 +99,14 @@ FaultDelayConfig::HeaderDelayProvider::duration(const Http::HeaderEntry* header) } FaultRateLimitConfig::FaultRateLimitConfig( - const envoy::extensions::filters::common::fault::v3::FaultRateLimit& rate_limit_config) - : percentage_(rate_limit_config.percentage()) { + const envoy::extensions::filters::common::fault::v3::FaultRateLimit& rate_limit_config) { switch (rate_limit_config.limit_type_case()) { case envoy::extensions::filters::common::fault::v3::FaultRateLimit::LimitTypeCase::kFixedLimit: - provider_ = - std::make_unique(rate_limit_config.fixed_limit().limit_kbps()); + provider_ = std::make_unique( + rate_limit_config.fixed_limit().limit_kbps(), rate_limit_config.percentage()); break; case envoy::extensions::filters::common::fault::v3::FaultRateLimit::LimitTypeCase::kHeaderLimit: - provider_ = std::make_unique(); + provider_ = std::make_unique(rate_limit_config.percentage()); break; case envoy::extensions::filters::common::fault::v3::FaultRateLimit::LimitTypeCase:: LIMIT_TYPE_NOT_SET: @@ -95,8 +114,9 @@ FaultRateLimitConfig::FaultRateLimitConfig( } } -absl::optional -FaultRateLimitConfig::HeaderRateLimitProvider::rateKbps(const Http::HeaderEntry* header) const { +absl::optional FaultRateLimitConfig::HeaderRateLimitProvider::rateKbps( + const Http::RequestHeaderMap* request_headers) const { + const auto header = request_headers->get(HeaderNames::get().ThroughputResponse); if (header == nullptr) { return absl::nullopt; } diff --git a/source/extensions/filters/common/fault/fault_config.h b/source/extensions/filters/common/fault/fault_config.h index 985fae12e0d1..2bf80a1e67d2 100644 --- a/source/extensions/filters/common/fault/fault_config.h +++ b/source/extensions/filters/common/fault/fault_config.h @@ -19,21 +19,47 @@ class HeaderNameValues { public: const char* prefix() { return ThreadSafeSingleton::get().prefix(); } + const Http::LowerCaseString AbortRequest{absl::StrCat(prefix(), "-fault-abort-request")}; + const Http::LowerCaseString AbortRequestPercentage{ + absl::StrCat(prefix(), "-fault-abort-request-percentage")}; const Http::LowerCaseString DelayRequest{absl::StrCat(prefix(), "-fault-delay-request")}; + const Http::LowerCaseString DelayRequestPercentage{ + absl::StrCat(prefix(), "-fault-delay-request-percentage")}; const Http::LowerCaseString ThroughputResponse{ absl::StrCat(prefix(), "-fault-throughput-response")}; - const Http::LowerCaseString AbortRequest{absl::StrCat(prefix(), "-fault-abort-request")}; + const Http::LowerCaseString ThroughputResponsePercentage{ + absl::StrCat(prefix(), "-fault-throughput-response-percentage")}; }; using HeaderNames = ConstSingleton; +class HeaderPercentageProvider { +public: + HeaderPercentageProvider(const Http::LowerCaseString& header_name, + const envoy::type::v3::FractionalPercent& percentage) + : header_name_(header_name), percentage_(percentage) {} + + // Return the percentage. Optionally passed HTTP headers that may contain the percentage number, + // otherwise the percentage passed at the initialized time is returned. + envoy::type::v3::FractionalPercent + percentage(const Http::RequestHeaderMap* request_headers) const; + +private: + const Http::LowerCaseString header_name_; + const envoy::type::v3::FractionalPercent percentage_; +}; + class FaultAbortConfig { public: FaultAbortConfig(const envoy::extensions::filters::http::fault::v3::FaultAbort& abort_config); - const envoy::type::v3::FractionalPercent& percentage() const { return percentage_; } - absl::optional statusCode(const Http::HeaderEntry* header) const { - return provider_->statusCode(header); + absl::optional statusCode(const Http::RequestHeaderMap* request_headers) const { + return provider_->statusCode(request_headers); + } + + envoy::type::v3::FractionalPercent + percentage(const Http::RequestHeaderMap* request_headers) const { + return provider_->percentage(request_headers); } private: @@ -42,36 +68,54 @@ class FaultAbortConfig { public: virtual ~AbortProvider() = default; - // Return the HTTP status code to use. Optionally passed an HTTP header that may contain the + // Return the HTTP status code to use. Optionally passed HTTP headers that may contain the // HTTP status code depending on the provider implementation. - virtual absl::optional statusCode(const Http::HeaderEntry* header) const PURE; + virtual absl::optional + statusCode(const Http::RequestHeaderMap* request_headers) const PURE; + // Return what percentage of requests abort faults should be applied to. Optionally passed + // HTTP headers that may contain the percentage depending on the provider implementation. + virtual envoy::type::v3::FractionalPercent + percentage(const Http::RequestHeaderMap* request_headers) const PURE; }; // Delay provider that uses a fixed abort status code. class FixedAbortProvider : public AbortProvider { public: - FixedAbortProvider(uint64_t status_code) : status_code_(status_code) {} + FixedAbortProvider(uint64_t status_code, const envoy::type::v3::FractionalPercent& percentage) + : status_code_(status_code), percentage_(percentage) {} // AbortProvider - absl::optional statusCode(const Http::HeaderEntry*) const override { + absl::optional statusCode(const Http::RequestHeaderMap*) const override { return static_cast(status_code_); } + envoy::type::v3::FractionalPercent percentage(const Http::RequestHeaderMap*) const override { + return percentage_; + } + private: const uint64_t status_code_; + const envoy::type::v3::FractionalPercent percentage_; }; // Abort provider the reads a status code from an HTTP header. - class HeaderAbortProvider : public AbortProvider { + class HeaderAbortProvider : public AbortProvider, public HeaderPercentageProvider { public: + HeaderAbortProvider(const envoy::type::v3::FractionalPercent& percentage) + : HeaderPercentageProvider(HeaderNames::get().AbortRequestPercentage, percentage) {} // AbortProvider - absl::optional statusCode(const Http::HeaderEntry* header) const override; + absl::optional + statusCode(const Http::RequestHeaderMap* request_headers) const override; + + envoy::type::v3::FractionalPercent + percentage(const Http::RequestHeaderMap* request_headers) const override { + return HeaderPercentageProvider::percentage(request_headers); + } }; using AbortProviderPtr = std::unique_ptr; AbortProviderPtr provider_; - const envoy::type::v3::FractionalPercent percentage_; }; using FaultAbortConfigPtr = std::unique_ptr; @@ -83,9 +127,14 @@ class FaultDelayConfig { public: FaultDelayConfig(const envoy::extensions::filters::common::fault::v3::FaultDelay& delay_config); - const envoy::type::v3::FractionalPercent& percentage() const { return percentage_; } - absl::optional duration(const Http::HeaderEntry* header) const { - return provider_->duration(header); + absl::optional + duration(const Http::RequestHeaderMap* request_headers) const { + return provider_->duration(request_headers); + } + + envoy::type::v3::FractionalPercent + percentage(const Http::RequestHeaderMap* request_headers) const { + return provider_->percentage(request_headers); } private: @@ -94,38 +143,56 @@ class FaultDelayConfig { public: virtual ~DelayProvider() = default; - // Return the duration to use. Optionally passed an HTTP header that may contain the delay + // Return the duration to use. Optionally passed HTTP headers that may contain the delay // depending on the provider implementation. virtual absl::optional - duration(const Http::HeaderEntry* header) const PURE; + duration(const Http::RequestHeaderMap* request_headers) const PURE; + // Return what percentage of requests request faults should be applied to. Optionally passed + // HTTP headers that may contain the percentage depending on the provider implementation. + virtual envoy::type::v3::FractionalPercent + percentage(const Http::RequestHeaderMap* request_headers) const PURE; }; // Delay provider that uses a fixed delay. class FixedDelayProvider : public DelayProvider { public: - FixedDelayProvider(std::chrono::milliseconds delay) : delay_(delay) {} + FixedDelayProvider(std::chrono::milliseconds delay, + const envoy::type::v3::FractionalPercent& percentage) + : delay_(delay), percentage_(percentage) {} // DelayProvider - absl::optional duration(const Http::HeaderEntry*) const override { + absl::optional + duration(const Http::RequestHeaderMap*) const override { return delay_; } + envoy::type::v3::FractionalPercent percentage(const Http::RequestHeaderMap*) const override { + return percentage_; + } + private: const std::chrono::milliseconds delay_; + const envoy::type::v3::FractionalPercent percentage_; }; // Delay provider the reads a delay from an HTTP header. - class HeaderDelayProvider : public DelayProvider { + class HeaderDelayProvider : public DelayProvider, public HeaderPercentageProvider { public: + HeaderDelayProvider(const envoy::type::v3::FractionalPercent& percentage) + : HeaderPercentageProvider(HeaderNames::get().DelayRequestPercentage, percentage) {} // DelayProvider absl::optional - duration(const Http::HeaderEntry* header) const override; + duration(const Http::RequestHeaderMap* request_headers) const override; + + envoy::type::v3::FractionalPercent + percentage(const Http::RequestHeaderMap* request_headers) const override { + return HeaderPercentageProvider::percentage(request_headers); + } }; using DelayProviderPtr = std::unique_ptr; DelayProviderPtr provider_; - const envoy::type::v3::FractionalPercent percentage_; }; using FaultDelayConfigPtr = std::unique_ptr; @@ -139,9 +206,13 @@ class FaultRateLimitConfig { FaultRateLimitConfig( const envoy::extensions::filters::common::fault::v3::FaultRateLimit& rate_limit_config); - const envoy::type::v3::FractionalPercent& percentage() const { return percentage_; } - absl::optional rateKbps(const Http::HeaderEntry* header) const { - return provider_->rateKbps(header); + absl::optional rateKbps(const Http::RequestHeaderMap* request_headers) const { + return provider_->rateKbps(request_headers); + } + + envoy::type::v3::FractionalPercent + percentage(const Http::RequestHeaderMap* request_headers) const { + return provider_->percentage(request_headers); } private: @@ -150,33 +221,52 @@ class FaultRateLimitConfig { public: virtual ~RateLimitProvider() = default; - // Return the rate limit to use in KiB/s. Optionally passed an HTTP header that may contain the + // Return the rate limit to use in KiB/s. Optionally passed HTTP headers that may contain the // rate limit depending on the provider implementation. - virtual absl::optional rateKbps(const Http::HeaderEntry* header) const PURE; + virtual absl::optional + rateKbps(const Http::RequestHeaderMap* request_headers) const PURE; + // Return what percentage of requests response rate limit faults should be applied to. + // Optionally passed HTTP headers that may contain the percentage depending on the provider + // implementation. + virtual envoy::type::v3::FractionalPercent + percentage(const Http::RequestHeaderMap* request_headers) const PURE; }; // Rate limit provider that uses a fixed rate limit. class FixedRateLimitProvider : public RateLimitProvider { public: - FixedRateLimitProvider(uint64_t fixed_rate_kbps) : fixed_rate_kbps_(fixed_rate_kbps) {} - absl::optional rateKbps(const Http::HeaderEntry*) const override { + FixedRateLimitProvider(uint64_t fixed_rate_kbps, + const envoy::type::v3::FractionalPercent& percentage) + : fixed_rate_kbps_(fixed_rate_kbps), percentage_(percentage) {} + absl::optional rateKbps(const Http::RequestHeaderMap*) const override { return fixed_rate_kbps_; } + envoy::type::v3::FractionalPercent percentage(const Http::RequestHeaderMap*) const override { + return percentage_; + } + private: const uint64_t fixed_rate_kbps_; + const envoy::type::v3::FractionalPercent percentage_; }; // Rate limit provider that reads the rate limit from an HTTP header. - class HeaderRateLimitProvider : public RateLimitProvider { + class HeaderRateLimitProvider : public RateLimitProvider, public HeaderPercentageProvider { public: - absl::optional rateKbps(const Http::HeaderEntry* header) const override; + HeaderRateLimitProvider(const envoy::type::v3::FractionalPercent& percentage) + : HeaderPercentageProvider(HeaderNames::get().ThroughputResponsePercentage, percentage) {} + // RateLimitProvider + absl::optional rateKbps(const Http::RequestHeaderMap* request_headers) const override; + envoy::type::v3::FractionalPercent + percentage(const Http::RequestHeaderMap* request_headers) const override { + return HeaderPercentageProvider::percentage(request_headers); + } }; using RateLimitProviderPtr = std::unique_ptr; RateLimitProviderPtr provider_; - const envoy::type::v3::FractionalPercent percentage_; }; using FaultRateLimitConfigPtr = std::unique_ptr; diff --git a/source/extensions/filters/common/lua/lua.cc b/source/extensions/filters/common/lua/lua.cc index 73aeff96754a..3f8b5cd7ec01 100644 --- a/source/extensions/filters/common/lua/lua.cc +++ b/source/extensions/filters/common/lua/lua.cc @@ -51,6 +51,7 @@ ThreadLocalState::ThreadLocalState(const std::string& code, ThreadLocal::SlotAll // First verify that the supplied code can be parsed. CSmartPtr state(lua_open()); + ASSERT(state.get() != nullptr, "unable to create new lua state object"); luaL_openlibs(state.get()); if (0 != luaL_dostring(state.get(), code.c_str())) { @@ -91,6 +92,7 @@ CoroutinePtr ThreadLocalState::createCoroutine() { } ThreadLocalState::LuaThreadLocal::LuaThreadLocal(const std::string& code) : state_(lua_open()) { + ASSERT(state_.get() != nullptr, "unable to create new lua state object"); luaL_openlibs(state_.get()); int rc = luaL_dostring(state_.get(), code.c_str()); ASSERT(rc == 0); diff --git a/source/extensions/filters/http/common/compressor/compressor.cc b/source/extensions/filters/http/common/compressor/compressor.cc index 5242ee31f419..7c6ac05893b3 100644 --- a/source/extensions/filters/http/common/compressor/compressor.cc +++ b/source/extensions/filters/http/common/compressor/compressor.cc @@ -242,7 +242,7 @@ CompressorFilter::chooseEncoding(const Http::ResponseHeaderMap& headers) const { // Find intersection of encodings accepted by the user agent and provided // by the allowed compressors and choose the one with the highest q-value. EncPair choice{Http::Headers::get().AcceptEncodingValues.Identity, static_cast(0)}; - for (const auto pair : pairs) { + for (const auto& pair : pairs) { if ((pair.second > choice.second) && (allowed_compressors.count(std::string(pair.first)) || pair.first == Http::Headers::get().AcceptEncodingValues.Identity || diff --git a/source/extensions/filters/http/fault/fault_filter.cc b/source/extensions/filters/http/fault/fault_filter.cc index f396f5a12ff6..f3e277edfe6b 100644 --- a/source/extensions/filters/http/fault/fault_filter.cc +++ b/source/extensions/filters/http/fault/fault_filter.cc @@ -174,23 +174,16 @@ Http::FilterHeadersStatus FaultFilter::decodeHeaders(Http::RequestHeaderMap& hea } void FaultFilter::maybeSetupResponseRateLimit(const Http::RequestHeaderMap& request_headers) { - if (fault_settings_->responseRateLimit() == nullptr) { + if (!isResponseRateLimitEnabled(request_headers)) { return; } - absl::optional rate_kbps = fault_settings_->responseRateLimit()->rateKbps( - request_headers.get(Filters::Common::Fault::HeaderNames::get().ThroughputResponse)); + absl::optional rate_kbps = + fault_settings_->responseRateLimit()->rateKbps(&request_headers); if (!rate_kbps.has_value()) { return; } - // TODO(mattklein123): Allow runtime override via downstream cluster similar to the other keys. - if (!config_->runtime().snapshot().featureEnabled( - fault_settings_->responseRateLimitPercentRuntime(), - fault_settings_->responseRateLimit()->percentage())) { - return; - } - // General stats. All injected faults are considered a single aggregate active fault. maybeIncActiveFaults(); config_->stats().response_rl_injected_.inc(); @@ -221,46 +214,56 @@ bool FaultFilter::faultOverflow() { return false; } -bool FaultFilter::isDelayEnabled() { +bool FaultFilter::isDelayEnabled(const Http::RequestHeaderMap& request_headers) { const auto request_delay = fault_settings_->requestDelay(); if (request_delay == nullptr) { return false; } if (!downstream_cluster_delay_percent_key_.empty()) { - return config_->runtime().snapshot().featureEnabled(downstream_cluster_delay_percent_key_, - request_delay->percentage()); + return config_->runtime().snapshot().featureEnabled( + downstream_cluster_delay_percent_key_, request_delay->percentage(&request_headers)); } return config_->runtime().snapshot().featureEnabled(fault_settings_->delayPercentRuntime(), - request_delay->percentage()); + request_delay->percentage(&request_headers)); } -bool FaultFilter::isAbortEnabled() { +bool FaultFilter::isAbortEnabled(const Http::RequestHeaderMap& request_headers) { const auto request_abort = fault_settings_->requestAbort(); if (request_abort == nullptr) { return false; } if (!downstream_cluster_abort_percent_key_.empty()) { - return config_->runtime().snapshot().featureEnabled(downstream_cluster_abort_percent_key_, - request_abort->percentage()); + return config_->runtime().snapshot().featureEnabled( + downstream_cluster_abort_percent_key_, request_abort->percentage(&request_headers)); } return config_->runtime().snapshot().featureEnabled(fault_settings_->abortPercentRuntime(), - request_abort->percentage()); + request_abort->percentage(&request_headers)); +} + +bool FaultFilter::isResponseRateLimitEnabled(const Http::RequestHeaderMap& request_headers) { + if (fault_settings_->responseRateLimit() == nullptr) { + return false; + } + + // TODO(mattklein123): Allow runtime override via downstream cluster similar to the other keys. + return config_->runtime().snapshot().featureEnabled( + fault_settings_->responseRateLimitPercentRuntime(), + fault_settings_->responseRateLimit()->percentage(&request_headers)); } absl::optional FaultFilter::delayDuration(const Http::RequestHeaderMap& request_headers) { absl::optional ret; - if (!isDelayEnabled()) { + if (!isDelayEnabled(request_headers)) { return ret; } // See if the configured delay provider has a default delay, if not there is no delay (e.g., // header configuration and no/invalid header). - auto config_duration = fault_settings_->requestDelay()->duration( - request_headers.get(Filters::Common::Fault::HeaderNames::get().DelayRequest)); + auto config_duration = fault_settings_->requestDelay()->duration(&request_headers); if (!config_duration.has_value()) { return ret; } @@ -283,14 +286,13 @@ FaultFilter::delayDuration(const Http::RequestHeaderMap& request_headers) { absl::optional FaultFilter::abortHttpStatus(const Http::RequestHeaderMap& request_headers) { - if (!isAbortEnabled()) { + if (!isAbortEnabled(request_headers)) { return absl::nullopt; } // See if the configured abort provider has a default status code, if not there is no abort status // code (e.g., header configuration and no/invalid header). - const auto config_abort = fault_settings_->requestAbort()->statusCode( - request_headers.get(Filters::Common::Fault::HeaderNames::get().AbortRequest)); + const auto config_abort = fault_settings_->requestAbort()->statusCode(&request_headers); if (!config_abort.has_value()) { return absl::nullopt; } diff --git a/source/extensions/filters/http/fault/fault_filter.h b/source/extensions/filters/http/fault/fault_filter.h index 9572909a882f..bdbcbd975282 100644 --- a/source/extensions/filters/http/fault/fault_filter.h +++ b/source/extensions/filters/http/fault/fault_filter.h @@ -248,8 +248,9 @@ class FaultFilter : public Http::StreamFilter, Logger::Loggable delayDuration(const Http::RequestHeaderMap& request_headers); absl::optional abortHttpStatus(const Http::RequestHeaderMap& request_headers); diff --git a/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.cc b/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.cc index ae54456db528..1e0b67aa654f 100644 --- a/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.cc +++ b/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.cc @@ -361,8 +361,7 @@ Http::FilterHeadersStatus JsonTranscoderFilter::decodeHeaders(Http::RequestHeade // just pass-through the request to upstream. return Http::FilterHeadersStatus::Continue; } - has_http_body_response_ = - !method_->descriptor_->server_streaming() && method_->response_type_is_http_body_; + if (method_->request_type_is_http_body_) { if (headers.ContentType() != nullptr) { absl::string_view content_type = headers.ContentType()->value().getStringView(); @@ -502,11 +501,14 @@ Http::FilterHeadersStatus JsonTranscoderFilter::encodeHeaders(Http::ResponseHead } headers.setReferenceContentType(Http::Headers::get().ContentTypeValues.Json); - if (!method_->descriptor_->server_streaming()) { - return Http::FilterHeadersStatus::StopIteration; - } - return Http::FilterHeadersStatus::Continue; + // In case of HttpBody in response - content type is unknown at this moment. + // So "Continue" only for regular streaming use case and StopIteration for + // all other cases (non streaming, streaming + httpBody) + if (method_->descriptor_->server_streaming() && !method_->response_type_is_http_body_) { + return Http::FilterHeadersStatus::Continue; + } + return Http::FilterHeadersStatus::StopIteration; } Http::FilterDataStatus JsonTranscoderFilter::encodeData(Buffer::Instance& data, bool end_stream) { @@ -516,10 +518,12 @@ Http::FilterDataStatus JsonTranscoderFilter::encodeData(Buffer::Instance& data, has_body_ = true; - // TODO(dio): Add support for streaming case. - if (has_http_body_response_) { + if (method_->response_type_is_http_body_) { buildResponseFromHttpBodyOutput(*response_headers_, data); - return Http::FilterDataStatus::StopIterationAndBuffer; + if (!method_->descriptor_->server_streaming()) { + return Http::FilterDataStatus::StopIterationAndBuffer; + } + return Http::FilterDataStatus::Continue; } response_in_.move(data); @@ -539,6 +543,13 @@ Http::FilterDataStatus JsonTranscoderFilter::encodeData(Buffer::Instance& data, return Http::FilterDataStatus::Continue; } +Http::FilterTrailersStatus +JsonTranscoderFilter::encodeTrailers(Http::ResponseTrailerMap& trailers) { + doTrailers(trailers); + + return Http::FilterTrailersStatus::Continue; +} + void JsonTranscoderFilter::doTrailers(Http::ResponseHeaderOrTrailerMap& headers_or_trailers) { if (error_ || !transcoder_) { return; @@ -552,6 +563,12 @@ void JsonTranscoderFilter::doTrailers(Http::ResponseHeaderOrTrailerMap& headers_ return; } + if (method_->response_type_is_http_body_ && method_->descriptor_->server_streaming()) { + // Do not add empty json when HttpBody + streaming + // Also, headers already sent, just continue. + return; + } + Buffer::OwnedImpl data; readToBuffer(*transcoder_->ResponseOutput(), data); @@ -667,8 +684,15 @@ void JsonTranscoderFilter::buildResponseFromHttpBodyOutput( data.add(body); - response_headers.setContentType(http_body.content_type()); - response_headers.setContentLength(body.size()); + if (!method_->descriptor_->server_streaming()) { + // Non streaming case: single message with content type / length + response_headers.setContentType(http_body.content_type()); + response_headers.setContentLength(body.size()); + } else if (!http_body_response_headers_set_) { + // Streaming case: set content type only once from first HttpBody message + response_headers.setContentType(http_body.content_type()); + http_body_response_headers_set_ = true; + } return; } } diff --git a/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.h b/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.h index 15ce11054706..5c271dafde24 100644 --- a/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.h +++ b/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.h @@ -149,10 +149,7 @@ class JsonTranscoderFilter : public Http::StreamFilter, public Logger::Loggable< Http::FilterHeadersStatus encodeHeaders(Http::ResponseHeaderMap& headers, bool end_stream) override; Http::FilterDataStatus encodeData(Buffer::Instance& data, bool end_stream) override; - Http::FilterTrailersStatus encodeTrailers(Http::ResponseTrailerMap& trailers) override { - doTrailers(trailers); - return Http::FilterTrailersStatus::Continue; - } + Http::FilterTrailersStatus encodeTrailers(Http::ResponseTrailerMap& trailers) override; Http::FilterMetadataStatus encodeMetadata(Http::MetadataMap&) override { return Http::FilterMetadataStatus::Continue; } @@ -189,8 +186,8 @@ class JsonTranscoderFilter : public Http::StreamFilter, public Logger::Loggable< std::string content_type_; bool error_{false}; - bool has_http_body_response_{false}; bool has_body_{false}; + bool http_body_response_headers_set_{false}; }; } // namespace GrpcJsonTranscoder diff --git a/source/extensions/filters/http/grpc_stats/grpc_stats_filter.cc b/source/extensions/filters/http/grpc_stats/grpc_stats_filter.cc index a67f17d62cf5..e155135a4090 100644 --- a/source/extensions/filters/http/grpc_stats/grpc_stats_filter.cc +++ b/source/extensions/filters/http/grpc_stats/grpc_stats_filter.cc @@ -92,7 +92,8 @@ class GrpcServiceMethodToRequestNamesMap { struct Config { Config(const envoy::extensions::filters::http::grpc_stats::v3::FilterConfig& proto_config, Server::Configuration::FactoryContext& context) - : context_(context.grpcContext()), emit_filter_state_(proto_config.emit_filter_state()) { + : context_(context.grpcContext()), emit_filter_state_(proto_config.emit_filter_state()), + enable_upstream_stats_(proto_config.enable_upstream_stats()) { switch (proto_config.per_method_stat_specifier_case()) { case envoy::extensions::filters::http::grpc_stats::v3::FilterConfig:: @@ -134,7 +135,8 @@ struct Config { } } Grpc::Context& context_; - bool emit_filter_state_; + const bool emit_filter_state_; + const bool enable_upstream_stats_; bool stats_for_all_methods_{false}; absl::optional allowlist_; }; @@ -226,6 +228,16 @@ class GrpcStatsFilter : public Http::PassThroughFilter { if (doStatTracking()) { config_->context_.chargeStat(*cluster_, Grpc::Context::Protocol::Grpc, request_names_, trailers.GrpcStatus()); + + if (config_->enable_upstream_stats_ && + decoder_callbacks_->streamInfo().lastUpstreamTxByteSent().has_value() && + decoder_callbacks_->streamInfo().lastUpstreamRxByteReceived().has_value()) { + std::chrono::milliseconds chrono_duration = + std::chrono::duration_cast( + decoder_callbacks_->streamInfo().lastUpstreamRxByteReceived().value() - + decoder_callbacks_->streamInfo().lastUpstreamTxByteSent().value()); + config_->context_.chargeUpstreamStat(*cluster_, request_names_, chrono_duration); + } } return Http::FilterTrailersStatus::Continue; } diff --git a/source/extensions/filters/http/header_to_metadata/config.cc b/source/extensions/filters/http/header_to_metadata/config.cc index 831c40cf4594..e51104c658da 100644 --- a/source/extensions/filters/http/header_to_metadata/config.cc +++ b/source/extensions/filters/http/header_to_metadata/config.cc @@ -26,6 +26,13 @@ Http::FilterFactoryCb HeaderToMetadataConfig::createFilterFactoryFromProtoTyped( }; } +Router::RouteSpecificFilterConfigConstSharedPtr +HeaderToMetadataConfig::createRouteSpecificFilterConfigTyped( + const envoy::extensions::filters::http::header_to_metadata::v3::Config& config, + Server::Configuration::ServerFactoryContext&, ProtobufMessage::ValidationVisitor&) { + return std::make_shared(config, true); +} + /** * Static registration for the header-to-metadata filter. @see RegisterFactory. */ diff --git a/source/extensions/filters/http/header_to_metadata/config.h b/source/extensions/filters/http/header_to_metadata/config.h index 5e56388844ac..247fa1c535e3 100644 --- a/source/extensions/filters/http/header_to_metadata/config.h +++ b/source/extensions/filters/http/header_to_metadata/config.h @@ -23,6 +23,9 @@ class HeaderToMetadataConfig Http::FilterFactoryCb createFilterFactoryFromProtoTyped( const envoy::extensions::filters::http::header_to_metadata::v3::Config& proto_config, const std::string& stats_prefix, Server::Configuration::FactoryContext& context) override; + Router::RouteSpecificFilterConfigConstSharedPtr createRouteSpecificFilterConfigTyped( + const envoy::extensions::filters::http::header_to_metadata::v3::Config& config, + Server::Configuration::ServerFactoryContext&, ProtobufMessage::ValidationVisitor&) override; }; } // namespace HeaderToMetadataFilter diff --git a/source/extensions/filters/http/header_to_metadata/header_to_metadata_filter.cc b/source/extensions/filters/http/header_to_metadata/header_to_metadata_filter.cc index b171d6b4b615..3a5d2dc6725f 100644 --- a/source/extensions/filters/http/header_to_metadata/header_to_metadata_filter.cc +++ b/source/extensions/filters/http/header_to_metadata/header_to_metadata_filter.cc @@ -16,13 +16,18 @@ namespace Extensions { namespace HttpFilters { namespace HeaderToMetadataFilter { -Config::Config(const envoy::extensions::filters::http::header_to_metadata::v3::Config config) { +Config::Config(const envoy::extensions::filters::http::header_to_metadata::v3::Config config, + const bool per_route) { request_set_ = Config::configToVector(config.request_rules(), request_rules_); response_set_ = Config::configToVector(config.response_rules(), response_rules_); - // don't allow an empty configuration - if (!response_set_ && !request_set_) { - throw EnvoyException("Must at least specify either response or request config"); + // Note: empty configs are fine for the global config, which would be the case for enabling + // the filter globally without rules and then applying them at the virtual host or + // route level. At the virtual or route level, it makes no sense to have an empty + // config so we throw an error. + if (per_route && !response_set_ && !request_set_) { + throw EnvoyException("header_to_metadata_filter: Per filter configs must at least specify " + "either request or response rules"); } } @@ -56,8 +61,9 @@ HeaderToMetadataFilter::~HeaderToMetadataFilter() = default; Http::FilterHeadersStatus HeaderToMetadataFilter::decodeHeaders(Http::RequestHeaderMap& headers, bool) { - if (config_->doRequest()) { - writeHeaderToMetadata(headers, config_->requestRules(), *decoder_callbacks_); + const auto* config = getConfig(); + if (config->doRequest()) { + writeHeaderToMetadata(headers, config->requestRules(), *decoder_callbacks_); } return Http::FilterHeadersStatus::Continue; @@ -70,8 +76,9 @@ void HeaderToMetadataFilter::setDecoderFilterCallbacks( Http::FilterHeadersStatus HeaderToMetadataFilter::encodeHeaders(Http::ResponseHeaderMap& headers, bool) { - if (config_->doResponse()) { - writeHeaderToMetadata(headers, config_->responseRules(), *encoder_callbacks_); + const auto* config = getConfig(); + if (config->doResponse()) { + writeHeaderToMetadata(headers, config->responseRules(), *encoder_callbacks_); } return Http::FilterHeadersStatus::Continue; } @@ -199,6 +206,34 @@ void HeaderToMetadataFilter::writeHeaderToMetadata(Http::HeaderMap& headers, } } +const Config* HeaderToMetadataFilter::getRouteConfig() const { + if (!decoder_callbacks_->route() || !decoder_callbacks_->route()->routeEntry()) { + return nullptr; + } + + const auto* entry = decoder_callbacks_->route()->routeEntry(); + const auto* per_filter_config = + entry->virtualHost().perFilterConfig(HttpFilterNames::get().HeaderToMetadata); + + return dynamic_cast(per_filter_config); +} + +// TODO(rgs1): this belongs in one of the filter interfaces, see issue #10164. +const Config* HeaderToMetadataFilter::getConfig() const { + // Cached config pointer. + if (effective_config_) { + return effective_config_; + } + + effective_config_ = getRouteConfig(); + if (effective_config_) { + return effective_config_; + } + + effective_config_ = config_.get(); + return effective_config_; +} + } // namespace HeaderToMetadataFilter } // namespace HttpFilters } // namespace Extensions diff --git a/source/extensions/filters/http/header_to_metadata/header_to_metadata_filter.h b/source/extensions/filters/http/header_to_metadata/header_to_metadata_filter.h index bd1f1222286b..6c02dfb75b07 100644 --- a/source/extensions/filters/http/header_to_metadata/header_to_metadata_filter.h +++ b/source/extensions/filters/http/header_to_metadata/header_to_metadata_filter.h @@ -28,9 +28,11 @@ const uint32_t MAX_HEADER_VALUE_LEN = 8 * 1024; * Encapsulates the filter configuration with STL containers and provides an area for any custom * configuration logic. */ -class Config : public Logger::Loggable { +class Config : public ::Envoy::Router::RouteSpecificFilterConfig, + public Logger::Loggable { public: - Config(const envoy::extensions::filters::http::header_to_metadata::v3::Config config); + Config(const envoy::extensions::filters::http::header_to_metadata::v3::Config config, + bool per_route = false); HeaderToMetadataRules requestRules() const { return request_rules_; } HeaderToMetadataRules responseRules() const { return response_rules_; } @@ -102,9 +104,12 @@ class HeaderToMetadataFilter : public Http::StreamFilter, void setEncoderFilterCallbacks(Http::StreamEncoderFilterCallbacks& callbacks) override; private: + friend class HeaderToMetadataTest; + using StructMap = std::map; const ConfigSharedPtr config_; + mutable const Config* effective_config_{nullptr}; Http::StreamDecoderFilterCallbacks* decoder_callbacks_{}; Http::StreamEncoderFilterCallbacks* encoder_callbacks_{}; @@ -123,6 +128,8 @@ class HeaderToMetadataFilter : public Http::StreamFilter, bool addMetadata(StructMap&, const std::string&, const std::string&, absl::string_view, ValueType, ValueEncode) const; const std::string& decideNamespace(const std::string& nspace) const; + const Config* getConfig() const; + const Config* getRouteConfig() const; }; } // namespace HeaderToMetadataFilter diff --git a/source/extensions/filters/network/dubbo_proxy/hessian_utils.cc b/source/extensions/filters/network/dubbo_proxy/hessian_utils.cc index 337afc66059d..c06ca93dd2c4 100644 --- a/source/extensions/filters/network/dubbo_proxy/hessian_utils.cc +++ b/source/extensions/filters/network/dubbo_proxy/hessian_utils.cc @@ -29,7 +29,7 @@ typename std::enable_if::value, T>::type leftShift(T left, uin inline void addByte(Buffer::Instance& buffer, const uint8_t value) { buffer.add(&value, 1); } void addSeq(Buffer::Instance& buffer, const std::initializer_list& values) { - for (const int8_t& value : values) { + for (const uint8_t& value : values) { buffer.add(&value, 1); } } diff --git a/source/extensions/filters/network/mongo_proxy/proxy.cc b/source/extensions/filters/network/mongo_proxy/proxy.cc index 0ec6c178291b..fcbd3f6c52bb 100644 --- a/source/extensions/filters/network/mongo_proxy/proxy.cc +++ b/source/extensions/filters/network/mongo_proxy/proxy.cc @@ -401,8 +401,10 @@ absl::optional ProxyFilter::delayDuration() { return result; } + // Use a default percentage + const auto percentage = fault_config_->percentage(nullptr); if (!runtime_.snapshot().featureEnabled(MongoRuntimeConfig::get().FixedDelayPercent, - fault_config_->percentage())) { + percentage)) { return result; } diff --git a/source/extensions/filters/network/sni_cluster/sni_cluster.cc b/source/extensions/filters/network/sni_cluster/sni_cluster.cc index eb1694647b75..876eca758a94 100644 --- a/source/extensions/filters/network/sni_cluster/sni_cluster.cc +++ b/source/extensions/filters/network/sni_cluster/sni_cluster.cc @@ -21,8 +21,7 @@ Network::FilterStatus SniClusterFilter::onNewConnection() { read_callbacks_->connection().streamInfo().filterState()->setData( TcpProxy::PerConnectionCluster::key(), std::make_unique(sni), - StreamInfo::FilterState::StateType::Mutable, - StreamInfo::FilterState::LifeSpan::DownstreamConnection); + StreamInfo::FilterState::StateType::Mutable, StreamInfo::FilterState::LifeSpan::Connection); } return Network::FilterStatus::Continue; diff --git a/source/extensions/filters/network/sni_dynamic_forward_proxy/BUILD b/source/extensions/filters/network/sni_dynamic_forward_proxy/BUILD new file mode 100644 index 000000000000..b4a08a55260c --- /dev/null +++ b/source/extensions/filters/network/sni_dynamic_forward_proxy/BUILD @@ -0,0 +1,40 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_extension", + "envoy_cc_library", + "envoy_package", +) + +envoy_package() + +envoy_cc_library( + name = "proxy_filter_lib", + srcs = ["proxy_filter.cc"], + hdrs = ["proxy_filter.h"], + deps = [ + "//include/envoy/network:connection_interface", + "//include/envoy/network:filter_interface", + "//source/common/common:assert_lib", + "//source/common/common:minimal_logger_lib", + "//source/common/tcp_proxy", + "//source/extensions/common/dynamic_forward_proxy:dns_cache_interface", + "@envoy_api//envoy/extensions/filters/network/sni_dynamic_forward_proxy/v3alpha:pkg_cc_proto", + ], +) + +envoy_cc_extension( + name = "config", + srcs = ["config.cc"], + hdrs = ["config.h"], + security_posture = "unknown", + status = "alpha", + deps = [ + ":proxy_filter_lib", + "//source/extensions/common/dynamic_forward_proxy:dns_cache_manager_impl", + "//source/extensions/filters/network:well_known_names", + "//source/extensions/filters/network/common:factory_base_lib", + "@envoy_api//envoy/extensions/filters/network/sni_dynamic_forward_proxy/v3alpha:pkg_cc_proto", + ], +) diff --git a/source/extensions/filters/network/sni_dynamic_forward_proxy/config.cc b/source/extensions/filters/network/sni_dynamic_forward_proxy/config.cc new file mode 100644 index 000000000000..a5f9fa9e1819 --- /dev/null +++ b/source/extensions/filters/network/sni_dynamic_forward_proxy/config.cc @@ -0,0 +1,41 @@ +#include "extensions/filters/network/sni_dynamic_forward_proxy/config.h" + +#include "envoy/registry/registry.h" +#include "envoy/server/filter_config.h" + +#include "extensions/common/dynamic_forward_proxy/dns_cache_manager_impl.h" +#include "extensions/filters/network/sni_dynamic_forward_proxy/proxy_filter.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace SniDynamicForwardProxy { + +SniDynamicForwardProxyNetworkFilterConfigFactory::SniDynamicForwardProxyNetworkFilterConfigFactory() + : FactoryBase(NetworkFilterNames::get().SniDynamicForwardProxy) {} + +Network::FilterFactoryCb +SniDynamicForwardProxyNetworkFilterConfigFactory::createFilterFactoryFromProtoTyped( + const FilterConfig& proto_config, Server::Configuration::FactoryContext& context) { + + Extensions::Common::DynamicForwardProxy::DnsCacheManagerFactoryImpl cache_manager_factory( + context.singletonManager(), context.dispatcher(), context.threadLocal(), context.random(), + context.scope()); + ProxyFilterConfigSharedPtr filter_config(std::make_shared( + proto_config, cache_manager_factory, context.clusterManager())); + + return [filter_config](Network::FilterManager& filter_manager) -> void { + filter_manager.addReadFilter(std::make_shared(filter_config)); + }; +} + +/** + * Static registration for the sni_dynamic_forward_proxy filter. @see RegisterFactory. + */ +REGISTER_FACTORY(SniDynamicForwardProxyNetworkFilterConfigFactory, + Server::Configuration::NamedNetworkFilterConfigFactory); + +} // namespace SniDynamicForwardProxy +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/network/sni_dynamic_forward_proxy/config.h b/source/extensions/filters/network/sni_dynamic_forward_proxy/config.h new file mode 100644 index 000000000000..c54dc6d56937 --- /dev/null +++ b/source/extensions/filters/network/sni_dynamic_forward_proxy/config.h @@ -0,0 +1,34 @@ +#pragma once + +#include "envoy/extensions/filters/network/sni_dynamic_forward_proxy/v3alpha/sni_dynamic_forward_proxy.pb.h" +#include "envoy/extensions/filters/network/sni_dynamic_forward_proxy/v3alpha/sni_dynamic_forward_proxy.pb.validate.h" + +#include "extensions/filters/network/common/factory_base.h" +#include "extensions/filters/network/well_known_names.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace SniDynamicForwardProxy { + +using FilterConfig = + envoy::extensions::filters::network::sni_dynamic_forward_proxy::v3alpha::FilterConfig; + +/** + * Config registration for the sni_dynamic_forward_proxy filter. @see + * NamedNetworkFilterConfigFactory. + */ +class SniDynamicForwardProxyNetworkFilterConfigFactory : public Common::FactoryBase { +public: + SniDynamicForwardProxyNetworkFilterConfigFactory(); + +private: + Network::FilterFactoryCb + createFilterFactoryFromProtoTyped(const FilterConfig& proto_config, + Server::Configuration::FactoryContext& context) override; +}; + +} // namespace SniDynamicForwardProxy +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/network/sni_dynamic_forward_proxy/proxy_filter.cc b/source/extensions/filters/network/sni_dynamic_forward_proxy/proxy_filter.cc new file mode 100644 index 000000000000..d9eeceb12098 --- /dev/null +++ b/source/extensions/filters/network/sni_dynamic_forward_proxy/proxy_filter.cc @@ -0,0 +1,76 @@ +#include "extensions/filters/network/sni_dynamic_forward_proxy/proxy_filter.h" + +#include "envoy/network/connection.h" +#include "envoy/network/filter.h" +#include "envoy/upstream/thread_local_cluster.h" + +#include "common/common/assert.h" +#include "common/tcp_proxy/tcp_proxy.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace SniDynamicForwardProxy { + +ProxyFilterConfig::ProxyFilterConfig( + const FilterConfig& proto_config, + Extensions::Common::DynamicForwardProxy::DnsCacheManagerFactory& cache_manager_factory, + Upstream::ClusterManager& cluster_manager) + : port_(static_cast(proto_config.port_value())), + dns_cache_manager_(cache_manager_factory.get()), + dns_cache_(dns_cache_manager_->getCache(proto_config.dns_cache_config())), + cluster_manager_(cluster_manager) {} + +ProxyFilter::ProxyFilter(ProxyFilterConfigSharedPtr config) : config_(std::move(config)) {} + +using LoadDnsCacheEntryStatus = Common::DynamicForwardProxy::DnsCache::LoadDnsCacheEntryStatus; + +Network::FilterStatus ProxyFilter::onNewConnection() { + absl::string_view sni = read_callbacks_->connection().requestedServerName(); + ENVOY_CONN_LOG(trace, "sni_dynamic_forward_proxy: new connection with server name '{}'", + read_callbacks_->connection(), sni); + + if (sni.empty()) { + return Network::FilterStatus::Continue; + } + + // TODO(lizan): implement circuit breaker in SNI dynamic forward proxy like it is in HTTP: + // https://github.com/envoyproxy/envoy/blob/master/source/extensions/filters/http/dynamic_forward_proxy/proxy_filter.cc#L65 + + uint32_t default_port = config_->port(); + + auto result = config_->cache().loadDnsCacheEntry(sni, default_port, *this); + + cache_load_handle_ = std::move(result.handle_); + switch (result.status_) { + case LoadDnsCacheEntryStatus::InCache: { + ASSERT(cache_load_handle_ == nullptr); + ENVOY_CONN_LOG(debug, "DNS cache entry already loaded, continuing", + read_callbacks_->connection()); + return Network::FilterStatus::Continue; + } + case LoadDnsCacheEntryStatus::Loading: { + ASSERT(cache_load_handle_ != nullptr); + ENVOY_CONN_LOG(debug, "waiting to load DNS cache entry", read_callbacks_->connection()); + return Network::FilterStatus::StopIteration; + } + case LoadDnsCacheEntryStatus::Overflow: { + ASSERT(cache_load_handle_ == nullptr); + ENVOY_CONN_LOG(debug, "DNS cache overflow", read_callbacks_->connection()); + read_callbacks_->connection().close(Network::ConnectionCloseType::NoFlush); + return Network::FilterStatus::StopIteration; + } + } + + NOT_REACHED_GCOVR_EXCL_LINE; +} + +void ProxyFilter::onLoadDnsCacheComplete() { + ENVOY_CONN_LOG(debug, "load DNS cache complete, continuing", read_callbacks_->connection()); + read_callbacks_->continueReading(); +} + +} // namespace SniDynamicForwardProxy +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/network/sni_dynamic_forward_proxy/proxy_filter.h b/source/extensions/filters/network/sni_dynamic_forward_proxy/proxy_filter.h new file mode 100644 index 000000000000..49f66aba3fb6 --- /dev/null +++ b/source/extensions/filters/network/sni_dynamic_forward_proxy/proxy_filter.h @@ -0,0 +1,66 @@ +#pragma once + +#include "envoy/extensions/filters/network/sni_dynamic_forward_proxy/v3alpha/sni_dynamic_forward_proxy.pb.h" +#include "envoy/network/filter.h" +#include "envoy/upstream/cluster_manager.h" + +#include "common/common/logger.h" + +#include "extensions/common/dynamic_forward_proxy/dns_cache.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace SniDynamicForwardProxy { + +using FilterConfig = + envoy::extensions::filters::network::sni_dynamic_forward_proxy::v3alpha::FilterConfig; + +class ProxyFilterConfig { +public: + ProxyFilterConfig( + const FilterConfig& proto_config, + Extensions::Common::DynamicForwardProxy::DnsCacheManagerFactory& cache_manager_factory, + Upstream::ClusterManager& cluster_manager); + + Extensions::Common::DynamicForwardProxy::DnsCache& cache() { return *dns_cache_; } + Upstream::ClusterManager& clusterManager() { return cluster_manager_; } + uint32_t port() { return port_; } + +private: + const uint32_t port_; + const Extensions::Common::DynamicForwardProxy::DnsCacheManagerSharedPtr dns_cache_manager_; + const Extensions::Common::DynamicForwardProxy::DnsCacheSharedPtr dns_cache_; + Upstream::ClusterManager& cluster_manager_; +}; + +using ProxyFilterConfigSharedPtr = std::shared_ptr; + +class ProxyFilter + : public Network::ReadFilter, + public Extensions::Common::DynamicForwardProxy::DnsCache::LoadDnsCacheEntryCallbacks, + Logger::Loggable { +public: + ProxyFilter(ProxyFilterConfigSharedPtr config); + // Network::ReadFilter + Network::FilterStatus onData(Buffer::Instance&, bool) override { + return Network::FilterStatus::Continue; + } + Network::FilterStatus onNewConnection() override; + void initializeReadFilterCallbacks(Network::ReadFilterCallbacks& callbacks) override { + read_callbacks_ = &callbacks; + } + + // Extensions::Common::DynamicForwardProxy::DnsCache::LoadDnsCacheEntryCallbacks + void onLoadDnsCacheComplete() override; + +private: + const ProxyFilterConfigSharedPtr config_; + Extensions::Common::DynamicForwardProxy::DnsCache::LoadDnsCacheEntryHandlePtr cache_load_handle_; + Network::ReadFilterCallbacks* read_callbacks_{}; +}; + +} // namespace SniDynamicForwardProxy +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/network/well_known_names.h b/source/extensions/filters/network/well_known_names.h index 2c3c7d4c7559..a7577b8ffd2c 100644 --- a/source/extensions/filters/network/well_known_names.h +++ b/source/extensions/filters/network/well_known_names.h @@ -46,6 +46,8 @@ class NetworkFilterNameValues { const std::string Rbac = "envoy.filters.network.rbac"; // SNI Cluster filter const std::string SniCluster = "envoy.filters.network.sni_cluster"; + // SNI Dynamic forward proxy filter + const std::string SniDynamicForwardProxy = "envoy.filters.network.sni_dynamic_forward_proxy"; // ZooKeeper proxy filter const std::string ZooKeeperProxy = "envoy.filters.network.zookeeper_proxy"; }; diff --git a/source/extensions/quic_listeners/quiche/active_quic_listener.cc b/source/extensions/quic_listeners/quiche/active_quic_listener.cc index 954441465cc5..8ab780021d62 100644 --- a/source/extensions/quic_listeners/quiche/active_quic_listener.cc +++ b/source/extensions/quic_listeners/quiche/active_quic_listener.cc @@ -31,7 +31,7 @@ ActiveQuicListener::ActiveQuicListener(Event::Dispatcher& dispatcher, Network::ListenerConfig& listener_config, const quic::QuicConfig& quic_config, Network::Socket::OptionsSharedPtr options) - : Server::ConnectionHandlerImpl::ActiveListenerImplBase(parent, listener_config), + : Server::ConnectionHandlerImpl::ActiveListenerImplBase(parent, &listener_config), dispatcher_(dispatcher), version_manager_(quic::CurrentSupportedVersions()), listen_socket_(*listen_socket) { if (options != nullptr) { @@ -59,7 +59,7 @@ ActiveQuicListener::ActiveQuicListener(Event::Dispatcher& dispatcher, std::make_unique(dispatcher_, *connection_helper->GetClock()); quic_dispatcher_ = std::make_unique( crypto_config_.get(), quic_config, &version_manager_, std::move(connection_helper), - std::move(alarm_factory), quic::kQuicDefaultConnectionIdLength, parent, config_, stats_, + std::move(alarm_factory), quic::kQuicDefaultConnectionIdLength, parent, *config_, stats_, per_worker_stats_, dispatcher, listen_socket_); quic_dispatcher_->InitializeWithWriter(new EnvoyQuicPacketWriter(listen_socket_)); } @@ -67,7 +67,7 @@ ActiveQuicListener::ActiveQuicListener(Event::Dispatcher& dispatcher, ActiveQuicListener::~ActiveQuicListener() { onListenerShutdown(); } void ActiveQuicListener::onListenerShutdown() { - ENVOY_LOG(info, "Quic listener {} shutdown.", config_.name()); + ENVOY_LOG(info, "Quic listener {} shutdown.", config_->name()); quic_dispatcher_->Shutdown(); udp_listener_.reset(); } diff --git a/source/extensions/stat_sinks/common/statsd/statsd.cc b/source/extensions/stat_sinks/common/statsd/statsd.cc index 5782935fb751..b4676d290155 100644 --- a/source/extensions/stat_sinks/common/statsd/statsd.cc +++ b/source/extensions/stat_sinks/common/statsd/statsd.cc @@ -62,6 +62,7 @@ void UdpStatsdSink::flush(Stats::MetricSnapshot& snapshot) { buildTagStr(gauge.get().tags()))); } } + // TODO(efimki): Add support of text readouts stats. } void UdpStatsdSink::onHistogramComplete(const Stats::Histogram& histogram, uint64_t value) { @@ -128,6 +129,7 @@ void TcpStatsdSink::flush(Stats::MetricSnapshot& snapshot) { tls_sink.flushGauge(gauge.get().name(), gauge.get().value()); } } + // TODO(efimki): Add support of text readouts stats. tls_sink.endFlush(true); } diff --git a/source/extensions/stat_sinks/hystrix/hystrix.cc b/source/extensions/stat_sinks/hystrix/hystrix.cc index e04fa22ae0d1..0596dd4cda41 100644 --- a/source/extensions/stat_sinks/hystrix/hystrix.cc +++ b/source/extensions/stat_sinks/hystrix/hystrix.cc @@ -50,7 +50,7 @@ void HystrixSink::addHistogramToStream(const QuantileLatencyMap& latency_map, ab // TODO: Consider if we better use join here ss << ", \"" << key << "\": {"; bool is_first = true; - for (const std::pair& element : latency_map) { + for (const auto& element : latency_map) { const std::string quantile = fmt::sprintf("%g", element.first * 100); HystrixSink::addDoubleToStream(quantile, element.second, ss, is_first); is_first = false; diff --git a/source/extensions/tracers/datadog/config.cc b/source/extensions/tracers/datadog/config.cc index 5b8908ed3ccf..e03a6d82573c 100644 --- a/source/extensions/tracers/datadog/config.cc +++ b/source/extensions/tracers/datadog/config.cc @@ -1,7 +1,7 @@ #include "extensions/tracers/datadog/config.h" -#include "envoy/config/trace/v3/trace.pb.h" -#include "envoy/config/trace/v3/trace.pb.validate.h" +#include "envoy/config/trace/v3/datadog.pb.h" +#include "envoy/config/trace/v3/datadog.pb.validate.h" #include "envoy/registry/registry.h" #include "common/common/utility.h" diff --git a/source/extensions/tracers/datadog/config.h b/source/extensions/tracers/datadog/config.h index 2453546405b0..324c82e85f9b 100644 --- a/source/extensions/tracers/datadog/config.h +++ b/source/extensions/tracers/datadog/config.h @@ -2,8 +2,8 @@ #include -#include "envoy/config/trace/v3/trace.pb.h" -#include "envoy/config/trace/v3/trace.pb.validate.h" +#include "envoy/config/trace/v3/datadog.pb.h" +#include "envoy/config/trace/v3/datadog.pb.validate.h" #include "extensions/tracers/common/factory_base.h" diff --git a/source/extensions/tracers/datadog/datadog_tracer_impl.cc b/source/extensions/tracers/datadog/datadog_tracer_impl.cc index 4b529d5bf459..ef99f50664e7 100644 --- a/source/extensions/tracers/datadog/datadog_tracer_impl.cc +++ b/source/extensions/tracers/datadog/datadog_tracer_impl.cc @@ -1,6 +1,6 @@ #include "extensions/tracers/datadog/datadog_tracer_impl.h" -#include "envoy/config/trace/v3/trace.pb.h" +#include "envoy/config/trace/v3/datadog.pb.h" #include "common/common/enum_to_int.h" #include "common/common/fmt.h" diff --git a/source/extensions/tracers/datadog/datadog_tracer_impl.h b/source/extensions/tracers/datadog/datadog_tracer_impl.h index 774e34665a85..5cdb482543bc 100644 --- a/source/extensions/tracers/datadog/datadog_tracer_impl.h +++ b/source/extensions/tracers/datadog/datadog_tracer_impl.h @@ -2,7 +2,7 @@ #include -#include "envoy/config/trace/v3/trace.pb.h" +#include "envoy/config/trace/v3/datadog.pb.h" #include "envoy/local_info/local_info.h" #include "envoy/runtime/runtime.h" #include "envoy/thread_local/thread_local.h" diff --git a/source/extensions/tracers/dynamic_ot/config.cc b/source/extensions/tracers/dynamic_ot/config.cc index 54736ec04107..c9667ac2a5d6 100644 --- a/source/extensions/tracers/dynamic_ot/config.cc +++ b/source/extensions/tracers/dynamic_ot/config.cc @@ -1,7 +1,7 @@ #include "extensions/tracers/dynamic_ot/config.h" -#include "envoy/config/trace/v3/trace.pb.h" -#include "envoy/config/trace/v3/trace.pb.validate.h" +#include "envoy/config/trace/v3/dynamic_ot.pb.h" +#include "envoy/config/trace/v3/dynamic_ot.pb.validate.h" #include "envoy/registry/registry.h" #include "common/common/utility.h" diff --git a/source/extensions/tracers/dynamic_ot/config.h b/source/extensions/tracers/dynamic_ot/config.h index 58fff56e4243..05fd2873132b 100644 --- a/source/extensions/tracers/dynamic_ot/config.h +++ b/source/extensions/tracers/dynamic_ot/config.h @@ -1,7 +1,7 @@ #pragma once -#include "envoy/config/trace/v3/trace.pb.h" -#include "envoy/config/trace/v3/trace.pb.validate.h" +#include "envoy/config/trace/v3/dynamic_ot.pb.h" +#include "envoy/config/trace/v3/dynamic_ot.pb.validate.h" #include "extensions/tracers/common/factory_base.h" diff --git a/source/extensions/tracers/lightstep/config.cc b/source/extensions/tracers/lightstep/config.cc index 52ecaec58c33..52509819dcaf 100644 --- a/source/extensions/tracers/lightstep/config.cc +++ b/source/extensions/tracers/lightstep/config.cc @@ -1,7 +1,7 @@ #include "extensions/tracers/lightstep/config.h" -#include "envoy/config/trace/v3/trace.pb.h" -#include "envoy/config/trace/v3/trace.pb.validate.h" +#include "envoy/config/trace/v3/lightstep.pb.h" +#include "envoy/config/trace/v3/lightstep.pb.validate.h" #include "envoy/registry/registry.h" #include "common/common/utility.h" diff --git a/source/extensions/tracers/lightstep/config.h b/source/extensions/tracers/lightstep/config.h index 771986998cd1..f14a45470ecf 100644 --- a/source/extensions/tracers/lightstep/config.h +++ b/source/extensions/tracers/lightstep/config.h @@ -1,7 +1,7 @@ #pragma once -#include "envoy/config/trace/v3/trace.pb.h" -#include "envoy/config/trace/v3/trace.pb.validate.h" +#include "envoy/config/trace/v3/lightstep.pb.h" +#include "envoy/config/trace/v3/lightstep.pb.validate.h" #include "extensions/tracers/common/factory_base.h" diff --git a/source/extensions/tracers/lightstep/lightstep_tracer_impl.cc b/source/extensions/tracers/lightstep/lightstep_tracer_impl.cc index 0f4b1a89e378..9cafe7e8a9fa 100644 --- a/source/extensions/tracers/lightstep/lightstep_tracer_impl.cc +++ b/source/extensions/tracers/lightstep/lightstep_tracer_impl.cc @@ -5,7 +5,7 @@ #include #include -#include "envoy/config/trace/v3/trace.pb.h" +#include "envoy/config/trace/v3/lightstep.pb.h" #include "common/buffer/buffer_impl.h" #include "common/buffer/zero_copy_input_stream_impl.h" diff --git a/source/extensions/tracers/lightstep/lightstep_tracer_impl.h b/source/extensions/tracers/lightstep/lightstep_tracer_impl.h index ee572f47f6eb..5a67bc8575b8 100644 --- a/source/extensions/tracers/lightstep/lightstep_tracer_impl.h +++ b/source/extensions/tracers/lightstep/lightstep_tracer_impl.h @@ -4,7 +4,7 @@ #include #include -#include "envoy/config/trace/v3/trace.pb.h" +#include "envoy/config/trace/v3/lightstep.pb.h" #include "envoy/runtime/runtime.h" #include "envoy/thread_local/thread_local.h" #include "envoy/tracing/http_tracer.h" diff --git a/source/extensions/tracers/opencensus/config.cc b/source/extensions/tracers/opencensus/config.cc index 27f90ad21069..af778ad04f7c 100644 --- a/source/extensions/tracers/opencensus/config.cc +++ b/source/extensions/tracers/opencensus/config.cc @@ -1,7 +1,7 @@ #include "extensions/tracers/opencensus/config.h" -#include "envoy/config/trace/v3/trace.pb.h" -#include "envoy/config/trace/v3/trace.pb.validate.h" +#include "envoy/config/trace/v3/opencensus.pb.h" +#include "envoy/config/trace/v3/opencensus.pb.validate.h" #include "envoy/registry/registry.h" #include "common/tracing/http_tracer_impl.h" diff --git a/source/extensions/tracers/opencensus/config.h b/source/extensions/tracers/opencensus/config.h index 14fcf0176ae6..a5270a45cb86 100644 --- a/source/extensions/tracers/opencensus/config.h +++ b/source/extensions/tracers/opencensus/config.h @@ -2,8 +2,8 @@ #include -#include "envoy/config/trace/v3/trace.pb.h" -#include "envoy/config/trace/v3/trace.pb.validate.h" +#include "envoy/config/trace/v3/opencensus.pb.h" +#include "envoy/config/trace/v3/opencensus.pb.validate.h" #include "extensions/tracers/common/factory_base.h" diff --git a/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc b/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc index 00a13fb0e69c..c39d4ebddd56 100644 --- a/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc +++ b/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc @@ -2,7 +2,7 @@ #include -#include "envoy/config/trace/v3/trace.pb.h" +#include "envoy/config/trace/v3/opencensus.pb.h" #include "envoy/http/header_map.h" #include "common/common/base64.h" diff --git a/source/extensions/tracers/opencensus/opencensus_tracer_impl.h b/source/extensions/tracers/opencensus/opencensus_tracer_impl.h index 75e21b7a742f..2c06d0c49a5d 100644 --- a/source/extensions/tracers/opencensus/opencensus_tracer_impl.h +++ b/source/extensions/tracers/opencensus/opencensus_tracer_impl.h @@ -1,7 +1,7 @@ #pragma once #include "envoy/api/api.h" -#include "envoy/config/trace/v3/trace.pb.h" +#include "envoy/config/trace/v3/opencensus.pb.h" #include "envoy/local_info/local_info.h" #include "envoy/tracing/http_tracer.h" diff --git a/source/extensions/tracers/zipkin/config.cc b/source/extensions/tracers/zipkin/config.cc index 663b8f950d04..0fca39dd4a31 100644 --- a/source/extensions/tracers/zipkin/config.cc +++ b/source/extensions/tracers/zipkin/config.cc @@ -1,7 +1,7 @@ #include "extensions/tracers/zipkin/config.h" -#include "envoy/config/trace/v3/trace.pb.h" -#include "envoy/config/trace/v3/trace.pb.validate.h" +#include "envoy/config/trace/v3/zipkin.pb.h" +#include "envoy/config/trace/v3/zipkin.pb.validate.h" #include "envoy/registry/registry.h" #include "common/common/utility.h" diff --git a/source/extensions/tracers/zipkin/config.h b/source/extensions/tracers/zipkin/config.h index 7ae4337a07aa..b91ef7cb7f35 100644 --- a/source/extensions/tracers/zipkin/config.h +++ b/source/extensions/tracers/zipkin/config.h @@ -1,7 +1,7 @@ #pragma once -#include "envoy/config/trace/v3/trace.pb.h" -#include "envoy/config/trace/v3/trace.pb.validate.h" +#include "envoy/config/trace/v3/zipkin.pb.h" +#include "envoy/config/trace/v3/zipkin.pb.validate.h" #include "extensions/tracers/common/factory_base.h" diff --git a/source/extensions/tracers/zipkin/span_buffer.cc b/source/extensions/tracers/zipkin/span_buffer.cc index 2e7c9bab5ad3..a0803fe080bd 100644 --- a/source/extensions/tracers/zipkin/span_buffer.cc +++ b/source/extensions/tracers/zipkin/span_buffer.cc @@ -1,6 +1,6 @@ #include "extensions/tracers/zipkin/span_buffer.h" -#include "envoy/config/trace/v3/trace.pb.h" +#include "envoy/config/trace/v3/zipkin.pb.h" #include "common/protobuf/utility.h" @@ -9,6 +9,7 @@ #include "extensions/tracers/zipkin/zipkin_json_field_names.h" #include "absl/strings/str_join.h" +#include "absl/strings/str_replace.h" namespace Envoy { namespace Extensions { @@ -61,7 +62,7 @@ SerializerPtr SpanBuffer::makeSerializer( std::string JsonV1Serializer::serialize(const std::vector& zipkin_spans) { const std::string serialized_elements = - absl::StrJoin(zipkin_spans, ",", [](std::string* element, Span zipkin_span) { + absl::StrJoin(zipkin_spans, ",", [](std::string* element, const Span& zipkin_span) { absl::StrAppend(element, zipkin_span.toJson()); }); return absl::StrCat("[", serialized_elements, "]"); @@ -71,20 +72,46 @@ JsonV2Serializer::JsonV2Serializer(const bool shared_span_context) : shared_span_context_{shared_span_context} {} std::string JsonV2Serializer::serialize(const std::vector& zipkin_spans) { - const std::string serialized_elements = - absl::StrJoin(zipkin_spans, ",", [this](std::string* out, const Span& zipkin_span) { + Util::Replacements replacements; + const std::string serialized_elements = absl::StrJoin( + zipkin_spans, ",", [this, &replacements](std::string* out, const Span& zipkin_span) { + const auto& replacement_values = replacements; absl::StrAppend( - out, absl::StrJoin(toListOfSpans(zipkin_span), ",", - [](std::string* element, const ProtobufWkt::Struct& span) { - absl::StrAppend(element, MessageUtil::getJsonStringFromMessage( - span, false, true)); - })); + out, absl::StrJoin( + toListOfSpans(zipkin_span, replacements), ",", + [&replacement_values](std::string* element, const ProtobufWkt::Struct& span) { + const std::string json = MessageUtil::getJsonStringFromMessage( + span, /* pretty_print */ false, + /* always_print_primitive_fields */ true); + + // The Zipkin API V2 specification mandates to store timestamp value as int64 + // https://github.com/openzipkin/zipkin-api/blob/228fabe660f1b5d1e28eac9df41f7d1deed4a1c2/zipkin2-api.yaml#L447-L463 + // (often translated as uint64 in some of the official implementations: + // https://github.com/openzipkin/zipkin-go/blob/62dc8b26c05e0e8b88eb7536eff92498e65bbfc3/model/span.go#L114, + // and see the discussion here: + // https://github.com/openzipkin/zipkin-go/pull/161#issuecomment-598558072). + // However, when the timestamp is stored as number value in a protobuf + // struct, it is stored as a double. Because of how protobuf serializes + // doubles, there is a possibility that the value will be rendered as a + // number with scientific notation as reported in: + // https://github.com/envoyproxy/envoy/issues/9341#issuecomment-566912973. To + // deal with that issue, here we do a workaround by storing the timestamp as + // string and keeping track of that with the corresponding integer + // replacements, and do the replacement here so we can meet the Zipkin API V2 + // requirements. + // + // TODO(dio): The right fix for this is to introduce additional knob when + // serializing double in protobuf DoubleToBuffer function, and make it + // available to be controlled at caller site. + // https://github.com/envoyproxy/envoy/issues/10411). + absl::StrAppend(element, absl::StrReplaceAll(json, replacement_values)); + })); }); return absl::StrCat("[", serialized_elements, "]"); } const std::vector -JsonV2Serializer::toListOfSpans(const Span& zipkin_span) const { +JsonV2Serializer::toListOfSpans(const Span& zipkin_span, Util::Replacements& replacements) const { std::vector spans; spans.reserve(zipkin_span.annotations().size()); for (const auto& annotation : zipkin_span.annotations()) { @@ -103,7 +130,14 @@ JsonV2Serializer::toListOfSpans(const Span& zipkin_span) const { } if (annotation.isSetEndpoint()) { - (*fields)[SPAN_TIMESTAMP] = ValueUtil::numberValue(annotation.timestamp()); + // Usually we store number to a ProtobufWkt::Struct object via ValueUtil::numberValue. + // However, due to the possibility of rendering that to a number with scientific notation, we + // chose to store it as a string and keeping track the corresponding replacement. For example, + // we have 1584324295476870 if we stored it as a double value, MessageToJsonString gives + // us 1.58432429547687e+15. Instead we store it as the string of 1584324295476870 (when it is + // serialized: "1584324295476870"), and replace it post MessageToJsonString serialization with + // integer (1584324295476870 without `"`), see: JsonV2Serializer::serialize. + (*fields)[SPAN_TIMESTAMP] = Util::uint64Value(annotation.timestamp(), replacements); (*fields)[SPAN_LOCAL_ENDPOINT] = ValueUtil::structValue(toProtoEndpoint(annotation.endpoint())); } @@ -121,7 +155,9 @@ JsonV2Serializer::toListOfSpans(const Span& zipkin_span) const { } if (zipkin_span.isSetDuration()) { - (*fields)[SPAN_DURATION] = ValueUtil::numberValue(zipkin_span.duration()); + // Since SPAN_DURATION has the same data type with SPAN_TIMESTAMP, we use Util::uint64Value to + // store it. + (*fields)[SPAN_DURATION] = Util::uint64Value(zipkin_span.duration(), replacements); } const auto& binary_annotations = zipkin_span.binaryAnnotations(); diff --git a/source/extensions/tracers/zipkin/span_buffer.h b/source/extensions/tracers/zipkin/span_buffer.h index 107e09f82efb..a2267c7b724f 100644 --- a/source/extensions/tracers/zipkin/span_buffer.h +++ b/source/extensions/tracers/zipkin/span_buffer.h @@ -1,6 +1,6 @@ #pragma once -#include "envoy/config/trace/v3/trace.pb.h" +#include "envoy/config/trace/v3/zipkin.pb.h" #include "common/protobuf/protobuf.h" @@ -123,7 +123,8 @@ class JsonV2Serializer : public Serializer { std::string serialize(const std::vector& pending_spans) override; private: - const std::vector toListOfSpans(const Span& zipkin_span) const; + const std::vector toListOfSpans(const Span& zipkin_span, + Util::Replacements& replacements) const; const ProtobufWkt::Struct toProtoEndpoint(const Endpoint& zipkin_endpoint) const; const bool shared_span_context_; diff --git a/source/extensions/tracers/zipkin/util.cc b/source/extensions/tracers/zipkin/util.cc index a406f91349da..3d4ff6913f53 100644 --- a/source/extensions/tracers/zipkin/util.cc +++ b/source/extensions/tracers/zipkin/util.cc @@ -7,6 +7,7 @@ #include "common/common/hex.h" #include "common/common/utility.h" +#include "absl/strings/str_cat.h" #include "absl/strings/str_join.h" namespace Envoy { @@ -22,6 +23,12 @@ uint64_t Util::generateRandom64(TimeSource& time_source) { return rand_64(); } +ProtobufWkt::Value Util::uint64Value(uint64_t value, Replacements& replacements) { + const std::string string_value = std::to_string(value); + replacements.push_back({absl::StrCat("\"", string_value, "\""), string_value}); + return ValueUtil::stringValue(string_value); +} + } // namespace Zipkin } // namespace Tracers } // namespace Extensions diff --git a/source/extensions/tracers/zipkin/util.h b/source/extensions/tracers/zipkin/util.h index 0206f82155e7..6f1a93374484 100644 --- a/source/extensions/tracers/zipkin/util.h +++ b/source/extensions/tracers/zipkin/util.h @@ -6,6 +6,7 @@ #include "envoy/common/time.h" #include "common/common/byte_order.h" +#include "common/protobuf/utility.h" namespace Envoy { namespace Extensions { @@ -43,6 +44,18 @@ class Util { auto bytes = toEndianness(value); return std::string(reinterpret_cast(&bytes), sizeof(Type)); } + + using Replacements = std::vector>; + + /** + * Returns a wrapped uint64_t value as a string. In addition to that, it also pushes back a + * replacement to the given replacements vector. + * + * @param value unt64_t number that will be represented in string. + * @param replacements a container to hold the required replacements when serializing this value. + * @return ProtobufWkt::Value wrapped uint64_t as a string. + */ + static ProtobufWkt::Value uint64Value(uint64_t value, Replacements& replacements); }; } // namespace Zipkin diff --git a/source/extensions/tracers/zipkin/zipkin_core_types.cc b/source/extensions/tracers/zipkin/zipkin_core_types.cc index 7c5818b59f5f..19db113b4997 100644 --- a/source/extensions/tracers/zipkin/zipkin_core_types.cc +++ b/source/extensions/tracers/zipkin/zipkin_core_types.cc @@ -25,7 +25,7 @@ Endpoint& Endpoint::operator=(const Endpoint& ep) { return *this; } -const ProtobufWkt::Struct Endpoint::toStruct() const { +const ProtobufWkt::Struct Endpoint::toStruct(Util::Replacements&) const { ProtobufWkt::Struct endpoint; auto* fields = endpoint.mutable_fields(); if (!address_) { @@ -66,14 +66,14 @@ void Annotation::changeEndpointServiceName(const std::string& service_name) { } } -const ProtobufWkt::Struct Annotation::toStruct() const { +const ProtobufWkt::Struct Annotation::toStruct(Util::Replacements& replacements) const { ProtobufWkt::Struct annotation; auto* fields = annotation.mutable_fields(); - (*fields)[ANNOTATION_TIMESTAMP] = ValueUtil::numberValue(timestamp_); + (*fields)[ANNOTATION_TIMESTAMP] = Util::uint64Value(timestamp_, replacements); (*fields)[ANNOTATION_VALUE] = ValueUtil::stringValue(value_); if (endpoint_.has_value()) { (*fields)[ANNOTATION_ENDPOINT] = - ValueUtil::structValue(static_cast(endpoint_.value()).toStruct()); + ValueUtil::structValue(static_cast(endpoint_.value()).toStruct(replacements)); } return annotation; } @@ -98,7 +98,7 @@ BinaryAnnotation& BinaryAnnotation::operator=(const BinaryAnnotation& ann) { return *this; } -const ProtobufWkt::Struct BinaryAnnotation::toStruct() const { +const ProtobufWkt::Struct BinaryAnnotation::toStruct(Util::Replacements& replacements) const { ProtobufWkt::Struct binary_annotation; auto* fields = binary_annotation.mutable_fields(); (*fields)[BINARY_ANNOTATION_KEY] = ValueUtil::stringValue(key_); @@ -106,7 +106,7 @@ const ProtobufWkt::Struct BinaryAnnotation::toStruct() const { if (endpoint_) { (*fields)[BINARY_ANNOTATION_ENDPOINT] = - ValueUtil::structValue(static_cast(endpoint_.value()).toStruct()); + ValueUtil::structValue(static_cast(endpoint_.value()).toStruct(replacements)); } return binary_annotation; @@ -144,7 +144,7 @@ void Span::setServiceName(const std::string& service_name) { } } -const ProtobufWkt::Struct Span::toStruct() const { +const ProtobufWkt::Struct Span::toStruct(Util::Replacements& replacements) const { ProtobufWkt::Struct span; auto* fields = span.mutable_fields(); (*fields)[SPAN_TRACE_ID] = ValueUtil::stringValue(traceIdAsHexString()); @@ -156,17 +156,22 @@ const ProtobufWkt::Struct Span::toStruct() const { } if (timestamp_.has_value()) { - (*fields)[SPAN_TIMESTAMP] = ValueUtil::numberValue(timestamp_.value()); + // Usually we store number to a ProtobufWkt::Struct object via ValueUtil::numberValue. + // However, due to the possibility of rendering that to a number with scientific notation, we + // chose to store it as a string and keeping track the corresponding replacement. + (*fields)[SPAN_TIMESTAMP] = Util::uint64Value(timestamp_.value(), replacements); } if (duration_.has_value()) { - (*fields)[SPAN_DURATION] = ValueUtil::numberValue(duration_.value()); + // Since SPAN_DURATION has the same data type with SPAN_TIMESTAMP, we use Util::uint64Value to + // store it. + (*fields)[SPAN_DURATION] = Util::uint64Value(duration_.value(), replacements); } if (!annotations_.empty()) { std::vector annotation_list; for (auto& annotation : annotations_) { - annotation_list.push_back(ValueUtil::structValue(annotation.toStruct())); + annotation_list.push_back(ValueUtil::structValue(annotation.toStruct(replacements))); } (*fields)[SPAN_ANNOTATIONS] = ValueUtil::listValue(annotation_list); } @@ -174,7 +179,8 @@ const ProtobufWkt::Struct Span::toStruct() const { if (!binary_annotations_.empty()) { std::vector binary_annotation_list; for (auto& binary_annotation : binary_annotations_) { - binary_annotation_list.push_back(ValueUtil::structValue(binary_annotation.toStruct())); + binary_annotation_list.push_back( + ValueUtil::structValue(binary_annotation.toStruct(replacements))); } (*fields)[SPAN_BINARY_ANNOTATIONS] = ValueUtil::listValue(binary_annotation_list); } diff --git a/source/extensions/tracers/zipkin/zipkin_core_types.h b/source/extensions/tracers/zipkin/zipkin_core_types.h index 4edbfed6c696..02f302301b52 100644 --- a/source/extensions/tracers/zipkin/zipkin_core_types.h +++ b/source/extensions/tracers/zipkin/zipkin_core_types.h @@ -14,6 +14,7 @@ #include "extensions/tracers/zipkin/util.h" #include "absl/strings/str_cat.h" +#include "absl/strings/str_replace.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" @@ -36,8 +37,10 @@ class ZipkinBase { /** * All classes defining Zipkin abstractions need to implement this method to convert * the corresponding abstraction to a ProtobufWkt::Struct. + * @param replacements A container that is used to hold the required replacements when this object + * is serialized. */ - virtual const ProtobufWkt::Struct toStruct() const PURE; + virtual const ProtobufWkt::Struct toStruct(Util::Replacements& replacements) const PURE; /** * Serializes the a type as a Zipkin-compliant JSON representation as a string. @@ -45,7 +48,11 @@ class ZipkinBase { * @return a stringified JSON. */ const std::string toJson() const { - return MessageUtil::getJsonStringFromMessage(toStruct(), false, true); + Util::Replacements replacements; + return absl::StrReplaceAll( + MessageUtil::getJsonStringFromMessage(toStruct(replacements), /* pretty_print */ false, + /* always_print_primitive_fields */ true), + replacements); }; }; @@ -104,7 +111,7 @@ class Endpoint : public ZipkinBase { * * @return a protobuf struct. */ - const ProtobufWkt::Struct toStruct() const override; + const ProtobufWkt::Struct toStruct(Util::Replacements& replacements) const override; private: std::string service_name_; @@ -196,7 +203,7 @@ class Annotation : public ZipkinBase { * * @return a protobuf struct. */ - const ProtobufWkt::Struct toStruct() const override; + const ProtobufWkt::Struct toStruct(Util::Replacements& replacements) const override; private: uint64_t timestamp_{0}; @@ -291,10 +298,10 @@ class BinaryAnnotation : public ZipkinBase { /** * Represents the binary annotation as a protobuf struct. - * + * @param replacements Used to hold the required replacements on serialization step. * @return a protobuf struct. */ - const ProtobufWkt::Struct toStruct() const override; + const ProtobufWkt::Struct toStruct(Util::Replacements& replacements) const override; private: std::string key_; @@ -550,7 +557,7 @@ class Span : public ZipkinBase { * * @return a protobuf struct. */ - const ProtobufWkt::Struct toStruct() const override; + const ProtobufWkt::Struct toStruct(Util::Replacements& replacements) const override; /** * Associates a Tracer object with the span. The tracer's reportSpan() method is invoked diff --git a/source/extensions/tracers/zipkin/zipkin_tracer_impl.cc b/source/extensions/tracers/zipkin/zipkin_tracer_impl.cc index 6b0fc0240666..2cb4338c0ad3 100644 --- a/source/extensions/tracers/zipkin/zipkin_tracer_impl.cc +++ b/source/extensions/tracers/zipkin/zipkin_tracer_impl.cc @@ -1,6 +1,6 @@ #include "extensions/tracers/zipkin/zipkin_tracer_impl.h" -#include "envoy/config/trace/v3/trace.pb.h" +#include "envoy/config/trace/v3/zipkin.pb.h" #include "common/common/enum_to_int.h" #include "common/common/fmt.h" diff --git a/source/extensions/tracers/zipkin/zipkin_tracer_impl.h b/source/extensions/tracers/zipkin/zipkin_tracer_impl.h index 36866fd52b9e..5968a4464bbf 100644 --- a/source/extensions/tracers/zipkin/zipkin_tracer_impl.h +++ b/source/extensions/tracers/zipkin/zipkin_tracer_impl.h @@ -1,6 +1,6 @@ #pragma once -#include "envoy/config/trace/v3/trace.pb.h" +#include "envoy/config/trace/v3/zipkin.pb.h" #include "envoy/local_info/local_info.h" #include "envoy/runtime/runtime.h" #include "envoy/thread_local/thread_local.h" diff --git a/source/server/BUILD b/source/server/BUILD index da1d96e1f266..6dacea5f0f39 100644 --- a/source/server/BUILD +++ b/source/server/BUILD @@ -74,6 +74,7 @@ envoy_cc_library( "//include/envoy/stats:timespan_interface", "//source/common/common:linked_object", "//source/common/common:non_copyable", + "//source/common/event:deferred_task", "//source/common/network:connection_lib", "//source/common/stats:timespan_lib", "//source/common/stream_info:stream_info_lib", diff --git a/source/server/configuration_impl.cc b/source/server/configuration_impl.cc index 2952e03cc309..533bbdfef866 100644 --- a/source/server/configuration_impl.cc +++ b/source/server/configuration_impl.cc @@ -8,7 +8,7 @@ #include "envoy/config/bootstrap/v3/bootstrap.pb.h" #include "envoy/config/metrics/v3/stats.pb.h" -#include "envoy/config/trace/v3/trace.pb.h" +#include "envoy/config/trace/v3/http_tracer.pb.h" #include "envoy/network/connection.h" #include "envoy/runtime/runtime.h" #include "envoy/server/instance.h" diff --git a/source/server/configuration_impl.h b/source/server/configuration_impl.h index 6b9f4ef51270..5faf632f8959 100644 --- a/source/server/configuration_impl.h +++ b/source/server/configuration_impl.h @@ -10,7 +10,7 @@ #include #include "envoy/config/bootstrap/v3/bootstrap.pb.h" -#include "envoy/config/trace/v3/trace.pb.h" +#include "envoy/config/trace/v3/http_tracer.pb.h" #include "envoy/config/typed_config.h" #include "envoy/http/filter.h" #include "envoy/network/filter.h" diff --git a/source/server/connection_handler_impl.cc b/source/server/connection_handler_impl.cc index 37549c6b90de..2547ede9f35d 100644 --- a/source/server/connection_handler_impl.cc +++ b/source/server/connection_handler_impl.cc @@ -6,6 +6,7 @@ #include "envoy/stats/scope.h" #include "envoy/stats/timespan.h" +#include "common/event/deferred_task.h" #include "common/network/connection_impl.h" #include "common/network/utility.h" #include "common/stats/timespan_impl.h" @@ -26,9 +27,19 @@ void ConnectionHandlerImpl::decNumConnections() { --num_handler_connections_; } -void ConnectionHandlerImpl::addListener(Network::ListenerConfig& config) { +void ConnectionHandlerImpl::addListener(absl::optional overridden_listener, + Network::ListenerConfig& config) { ActiveListenerDetails details; if (config.listenSocketFactory().socketType() == Network::Address::SocketType::Stream) { + if (overridden_listener.has_value()) { + for (auto& listener : listeners_) { + if (listener.second.listener_->listenerTag() == overridden_listener) { + listener.second.tcp_listener_->get().updateListenerConfig(config); + return; + } + } + NOT_REACHED_GCOVR_EXCL_LINE; + } auto tcp_listener = std::make_unique(*this, config); details.tcp_listener_ = *tcp_listener; details.listener_ = std::move(tcp_listener); @@ -53,6 +64,32 @@ void ConnectionHandlerImpl::removeListeners(uint64_t listener_tag) { } } +void ConnectionHandlerImpl::removeFilterChains( + uint64_t listener_tag, const std::list& filter_chains, + std::function completion) { + // TODO(lambdai): Merge the optimistic path and the pessimistic path. + for (auto& listener : listeners_) { + // Optimistic path: The listener tag provided by arg is not stale. + if (listener.second.listener_->listenerTag() == listener_tag) { + listener.second.tcp_listener_->get().deferredRemoveFilterChains(filter_chains); + // Completion is deferred because the above removeFilterChains() may defer delete connection. + Event::DeferredTaskUtil::deferredRun(dispatcher_, std::move(completion)); + return; + } + } + // Fallback to iterate over all listeners. The reason is that the target listener might have began + // another update and the previous tag is lost. + // TODO(lambdai): Remove this once we decide to use the same listener tag during intelligent + // update. + for (auto& listener : listeners_) { + if (listener.second.tcp_listener_.has_value()) { + listener.second.tcp_listener_->get().deferredRemoveFilterChains(filter_chains); + } + } + // Completion is deferred because the above removeFilterChains() may defer delete connection. + Event::DeferredTaskUtil::deferredRun(dispatcher_, std::move(completion)); +} + void ConnectionHandlerImpl::stopListeners(uint64_t listener_tag) { for (auto& listener : listeners_) { if (listener.second.listener_->listenerTag() == listener_tag) { @@ -101,13 +138,13 @@ void ConnectionHandlerImpl::ActiveTcpListener::removeConnection(ActiveTcpConnect } ConnectionHandlerImpl::ActiveListenerImplBase::ActiveListenerImplBase( - Network::ConnectionHandler& parent, Network::ListenerConfig& config) - : stats_({ALL_LISTENER_STATS(POOL_COUNTER(config.listenerScope()), - POOL_GAUGE(config.listenerScope()), - POOL_HISTOGRAM(config.listenerScope()))}), + Network::ConnectionHandler& parent, Network::ListenerConfig* config) + : stats_({ALL_LISTENER_STATS(POOL_COUNTER(config->listenerScope()), + POOL_GAUGE(config->listenerScope()), + POOL_HISTOGRAM(config->listenerScope()))}), per_worker_stats_({ALL_PER_HANDLER_LISTENER_STATS( - POOL_COUNTER_PREFIX(config.listenerScope(), parent.statPrefix()), - POOL_GAUGE_PREFIX(config.listenerScope(), parent.statPrefix()))}), + POOL_COUNTER_PREFIX(config->listenerScope(), parent.statPrefix()), + POOL_GAUGE_PREFIX(config->listenerScope(), parent.statPrefix()))}), config_(config) {} ConnectionHandlerImpl::ActiveTcpListener::ActiveTcpListener(ConnectionHandlerImpl& parent, @@ -121,15 +158,21 @@ ConnectionHandlerImpl::ActiveTcpListener::ActiveTcpListener(ConnectionHandlerImp ConnectionHandlerImpl::ActiveTcpListener::ActiveTcpListener(ConnectionHandlerImpl& parent, Network::ListenerPtr&& listener, Network::ListenerConfig& config) - : ConnectionHandlerImpl::ActiveListenerImplBase(parent, config), parent_(parent), + : ConnectionHandlerImpl::ActiveListenerImplBase(parent, &config), parent_(parent), listener_(std::move(listener)), listener_filters_timeout_(config.listenerFiltersTimeout()), continue_on_listener_filters_timeout_(config.continueOnListenerFiltersTimeout()) { config.connectionBalancer().registerHandler(*this); } +void ConnectionHandlerImpl::ActiveTcpListener::updateListenerConfig( + Network::ListenerConfig& config) { + ENVOY_LOG(trace, "replacing listener ", config_->listenerTag(), " by ", config.listenerTag()); + config_ = &config; +} + ConnectionHandlerImpl::ActiveTcpListener::~ActiveTcpListener() { is_deleting_ = true; - config_.connectionBalancer().unregisterHandler(*this); + config_->connectionBalancer().unregisterHandler(*this); // Purge sockets that have not progressed to connections. This should only happen when // a listener filter stops iteration and never resumes. @@ -291,7 +334,7 @@ void ConnectionHandlerImpl::ActiveTcpSocket::newConnection() { } void ConnectionHandlerImpl::ActiveTcpListener::onAccept(Network::ConnectionSocketPtr&& socket) { - onAcceptWorker(std::move(socket), config_.handOffRestoredDestinationConnections(), false); + onAcceptWorker(std::move(socket), config_->handOffRestoredDestinationConnections(), false); } void ConnectionHandlerImpl::ActiveTcpListener::onAcceptWorker( @@ -299,7 +342,7 @@ void ConnectionHandlerImpl::ActiveTcpListener::onAcceptWorker( bool rebalanced) { if (!rebalanced) { Network::BalancedConnectionHandler& target_handler = - config_.connectionBalancer().pickTargetHandler(*this); + config_->connectionBalancer().pickTargetHandler(*this); if (&target_handler != this) { target_handler.post(std::move(socket)); return; @@ -310,7 +353,7 @@ void ConnectionHandlerImpl::ActiveTcpListener::onAcceptWorker( hand_off_restored_destination_connections); // Create and run the filters - config_.filterChainFactory().createListenerFilterChain(*active_socket); + config_->filterChainFactory().createListenerFilterChain(*active_socket); active_socket->continueFilterChain(true); // Move active_socket to the sockets_ list if filter iteration needs to continue later. @@ -338,13 +381,13 @@ void ConnectionHandlerImpl::ActiveTcpListener::newConnection( stream_info->setDownstreamDirectRemoteAddress(socket->directRemoteAddress()); // Find matching filter chain. - const auto filter_chain = config_.filterChainManager().findFilterChain(*socket); + const auto filter_chain = config_->filterChainManager().findFilterChain(*socket); if (filter_chain == nullptr) { ENVOY_LOG(debug, "closing connection: no matching filter chain found"); stats_.no_filter_chain_match_.inc(); stream_info->setResponseFlag(StreamInfo::ResponseFlag::NoRouteFound); stream_info->setResponseCodeDetails(StreamInfo::ResponseCodeDetails::get().FilterChainNotFound); - emitLogs(config_, *stream_info); + emitLogs(*config_, *stream_info); socket->close(); return; } @@ -356,10 +399,10 @@ void ConnectionHandlerImpl::ActiveTcpListener::newConnection( std::move(socket), std::move(transport_socket), *stream_info); ActiveTcpConnectionPtr active_connection( new ActiveTcpConnection(active_connections, std::move(server_conn_ptr), - parent_.dispatcher_.timeSource(), config_, std::move(stream_info))); - active_connection->connection_->setBufferLimits(config_.perConnectionBufferLimitBytes()); + parent_.dispatcher_.timeSource(), *config_, std::move(stream_info))); + active_connection->connection_->setBufferLimits(config_->perConnectionBufferLimitBytes()); - const bool empty_filter_chain = !config_.filterChainFactory().createNetworkFilterChain( + const bool empty_filter_chain = !config_->filterChainFactory().createNetworkFilterChain( *active_connection->connection_, filter_chain->networkFilterFactories()); if (empty_filter_chain) { ENVOY_CONN_LOG(debug, "closing connection: no filters", *active_connection->connection_); @@ -372,7 +415,6 @@ void ConnectionHandlerImpl::ActiveTcpListener::newConnection( active_connection->connection_->addConnectionCallbacks(*active_connection); active_connection->moveIntoList(std::move(active_connection), active_connections.connections_); } - // TODO(lambdai): defer delete active_connections when supporting per tag drain } ConnectionHandlerImpl::ActiveConnections& @@ -385,6 +427,29 @@ ConnectionHandlerImpl::ActiveTcpListener::getOrCreateActiveConnections( return *connections; } +void ConnectionHandlerImpl::ActiveTcpListener::deferredRemoveFilterChains( + const std::list& draining_filter_chains) { + // Need to recover the original deleting state. + const bool was_deleting = is_deleting_; + is_deleting_ = true; + for (const auto* filter_chain : draining_filter_chains) { + auto iter = connections_by_context_.find(filter_chain); + if (iter == connections_by_context_.end()) { + // It is possible when listener is stopping. + } else { + auto& connections = iter->second->connections_; + while (!connections.empty()) { + connections.front()->connection_->close(Network::ConnectionCloseType::NoFlush); + } + // Since is_deleting_ is on, we need to manually remove the map value and drive the iterator. + // Defer delete connection container to avoid race condition in destroying connection. + parent_.dispatcher_.deferredDelete(std::move(iter->second)); + iter = connections_by_context_.erase(iter); + } + } + is_deleting_ = was_deleting; +} + namespace { // Structure used to allow a unique_ptr to be captured in a posted lambda. See below. struct RebalancedSocket { @@ -401,24 +466,25 @@ void ConnectionHandlerImpl::ActiveTcpListener::post(Network::ConnectionSocketPtr RebalancedSocketSharedPtr socket_to_rebalance = std::make_shared(); socket_to_rebalance->socket = std::move(socket); - parent_.dispatcher_.post([socket_to_rebalance, tag = config_.listenerTag(), &parent = parent_]() { - // TODO(mattklein123): We should probably use a hash table here to lookup the tag instead of - // iterating through the listener list. - for (const auto& listener : parent.listeners_) { - if (listener.second.listener_->listener() != nullptr && - listener.second.listener_->listenerTag() == tag) { - // If the tag matches this must be a TCP listener. - ASSERT(listener.second.tcp_listener_.has_value()); - listener.second.tcp_listener_.value().get().onAcceptWorker( - std::move(socket_to_rebalance->socket), - listener.second.tcp_listener_.value() - .get() - .config_.handOffRestoredDestinationConnections(), - true); - return; - } - } - }); + parent_.dispatcher_.post( + [socket_to_rebalance, tag = config_->listenerTag(), &parent = parent_]() { + // TODO(mattklein123): We should probably use a hash table here to lookup the tag instead of + // iterating through the listener list. + for (const auto& listener : parent.listeners_) { + if (listener.second.listener_->listener() != nullptr && + listener.second.listener_->listenerTag() == tag) { + // If the tag matches this must be a TCP listener. + ASSERT(listener.second.tcp_listener_.has_value()); + listener.second.tcp_listener_.value().get().onAcceptWorker( + std::move(socket_to_rebalance->socket), + listener.second.tcp_listener_.value() + .get() + .config_->handOffRestoredDestinationConnections(), + true); + return; + } + } + }); } ConnectionHandlerImpl::ActiveConnections::ActiveConnections( @@ -479,16 +545,16 @@ ActiveUdpListener::ActiveUdpListener(Network::ConnectionHandler& parent, ActiveUdpListener::ActiveUdpListener(Network::ConnectionHandler& parent, Network::UdpListenerPtr&& listener, Network::ListenerConfig& config) - : ConnectionHandlerImpl::ActiveListenerImplBase(parent, config), + : ConnectionHandlerImpl::ActiveListenerImplBase(parent, &config), udp_listener_(std::move(listener)), read_filter_(nullptr) { // Create the filter chain on creating a new udp listener - config_.filterChainFactory().createUdpListenerFilterChain(*this, *this); + config_->filterChainFactory().createUdpListenerFilterChain(*this, *this); // If filter is nullptr, fail the creation of the listener if (read_filter_ == nullptr) { throw Network::CreateListenerException( fmt::format("Cannot create listener as no read filter registered for the udp listener: {} ", - config_.name())); + config_->name())); } } diff --git a/source/server/connection_handler_impl.h b/source/server/connection_handler_impl.h index bf1e07002112..c65ddf397da3 100644 --- a/source/server/connection_handler_impl.h +++ b/source/server/connection_handler_impl.h @@ -67,8 +67,12 @@ class ConnectionHandlerImpl : public Network::ConnectionHandler, uint64_t numConnections() const override { return num_handler_connections_; } void incNumConnections() override; void decNumConnections() override; - void addListener(Network::ListenerConfig& config) override; + void addListener(absl::optional overridden_listener, + Network::ListenerConfig& config) override; void removeListeners(uint64_t listener_tag) override; + void removeFilterChains(uint64_t listener_tag, + const std::list& filter_chains, + std::function completion) override; void stopListeners(uint64_t listener_tag) override; void stopListeners() override; void disableListeners() override; @@ -80,14 +84,14 @@ class ConnectionHandlerImpl : public Network::ConnectionHandler, */ class ActiveListenerImplBase : public Network::ConnectionHandler::ActiveListener { public: - ActiveListenerImplBase(Network::ConnectionHandler& parent, Network::ListenerConfig& config); + ActiveListenerImplBase(Network::ConnectionHandler& parent, Network::ListenerConfig* config); // Network::ConnectionHandler::ActiveListener. - uint64_t listenerTag() override { return config_.listenerTag(); } + uint64_t listenerTag() override { return config_->listenerTag(); } ListenerStats stats_; PerHandlerListenerStats per_worker_stats_; - Network::ListenerConfig& config_; + Network::ListenerConfig* config_{}; }; private: @@ -141,8 +145,24 @@ class ConnectionHandlerImpl : public Network::ConnectionHandler, */ void newConnection(Network::ConnectionSocketPtr&& socket); + /** + * Return the active connections container attached with the given filter chain. + */ ActiveConnections& getOrCreateActiveConnections(const Network::FilterChain& filter_chain); + /** + * Schedule to remove and destroy the active connections which are not tracked by listener + * config. Caution: The connection are not destroyed yet when function returns. + */ + void deferredRemoveFilterChains( + const std::list& draining_filter_chains); + + /** + * Update the listener config. The follow up connections will see the new config. The existing + * connections are not impacted. + */ + void updateListenerConfig(Network::ListenerConfig& config); + ConnectionHandlerImpl& parent_; Network::ListenerPtr listener_; const std::chrono::milliseconds listener_filters_timeout_; diff --git a/source/server/filter_chain_manager_impl.h b/source/server/filter_chain_manager_impl.h index 195897ca11a3..681876a5cb1e 100644 --- a/source/server/filter_chain_manager_impl.h +++ b/source/server/filter_chain_manager_impl.h @@ -282,10 +282,7 @@ class FilterChainManagerImpl : public Network::FilterChainManager, findFilterChainForSourceIpAndPort(const SourceIPsTrie& source_ips_trie, const Network::ConnectionSocket& socket) const; - const FilterChainManagerImpl* getOriginFilterChainManager() { - ASSERT(origin_.has_value()); - return origin_.value(); - } + const FilterChainManagerImpl* getOriginFilterChainManager() { return origin_.value(); } // Duplicate the inherent factory context if any. std::shared_ptr findExistingFilterChain(const envoy::config::listener::v3::FilterChain& filter_chain_message); diff --git a/source/server/http/BUILD b/source/server/http/BUILD index d477a211afd3..dbbb828c8930 100644 --- a/source/server/http/BUILD +++ b/source/server/http/BUILD @@ -15,6 +15,7 @@ envoy_cc_library( deps = [ ":admin_filter_lib", ":config_tracker_lib", + ":stats_handler_lib", ":utils_lib", "//include/envoy/filesystem:filesystem_interface", "//include/envoy/http:filter_interface", @@ -27,7 +28,6 @@ envoy_cc_library( "//include/envoy/server:instance_interface", "//include/envoy/server:listener_manager_interface", "//include/envoy/server:options_interface", - "//include/envoy/stats:stats_interface", "//include/envoy/upstream:cluster_manager_interface", "//include/envoy/upstream:resource_manager_interface", "//include/envoy/upstream:upstream_interface", @@ -59,9 +59,7 @@ envoy_cc_library( "//source/common/profiler:profiler_lib", "//source/common/router:config_lib", "//source/common/router:scoped_config_lib", - "//source/common/stats:histogram_lib", "//source/common/stats:isolated_store_lib", - "//source/common/stats:stats_lib", "//source/common/upstream:host_utility_lib", "//source/extensions/access_loggers/file:file_access_log_lib", "@envoy_api//envoy/admin/v3:pkg_cc_proto", @@ -87,6 +85,23 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "stats_handler_lib", + srcs = ["stats_handler.cc"], + hdrs = ["stats_handler.h"], + deps = [ + ":utils_lib", + "//include/envoy/http:codes_interface", + "//include/envoy/server:admin_interface", + "//include/envoy/server:instance_interface", + "//source/common/buffer:buffer_lib", + "//source/common/html:utility_lib", + "//source/common/http:codes_lib", + "//source/common/http:header_map_lib", + "//source/common/stats:histogram_lib", + ], +) + envoy_cc_library( name = "utils_lib", srcs = ["utils.cc"], diff --git a/source/server/http/admin.cc b/source/server/http/admin.cc index bea7e6c9716c..3a8f7a563ce1 100644 --- a/source/server/http/admin.cc +++ b/source/server/http/admin.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include @@ -24,8 +23,6 @@ #include "envoy/server/hot_restart.h" #include "envoy/server/instance.h" #include "envoy/server/options.h" -#include "envoy/stats/scope.h" -#include "envoy/stats/stats.h" #include "envoy/upstream/cluster_manager.h" #include "envoy/upstream/upstream.h" @@ -43,7 +40,6 @@ #include "common/http/conn_manager_utility.h" #include "common/http/header_map_impl.h" #include "common/http/headers.h" -#include "common/json/json_loader.h" #include "common/memory/stats.h" #include "common/memory/utils.h" #include "common/network/listen_socket_impl.h" @@ -52,9 +48,9 @@ #include "common/protobuf/protobuf.h" #include "common/protobuf/utility.h" #include "common/router/config_impl.h" -#include "common/stats/histogram_impl.h" #include "common/upstream/host_utility.h" +#include "server/http/stats_handler.h" #include "server/http/utils.h" #include "extensions/access_loggers/file/file_access_log_impl.h" @@ -133,48 +129,14 @@ const char AdminHtmlEnd[] = R"( )"; -const std::regex PromRegex("[^a-zA-Z0-9_]"); - -const uint64_t RecentLookupsCapacity = 100; - -// Helper method to get filter parameter, or report an error for an invalid regex. -bool filterParam(Http::Utility::QueryParams params, Buffer::Instance& response, - absl::optional& regex) { - auto p = params.find("filter"); - if (p != params.end()) { - const std::string& pattern = p->second; - try { - regex = std::regex(pattern); - } catch (std::regex_error& error) { - // Include the offending pattern in the log, but not the error message. - response.add(fmt::format("Invalid regex: \"{}\"\n", error.what())); - ENVOY_LOG_MISC(error, "admin: Invalid regex: \"{}\": {}", error.what(), pattern); - return false; - } - } - return true; -} - -// Helper method to get a query parameter. -absl::optional queryParam(const Http::Utility::QueryParams& params, - const std::string& key) { - return (params.find(key) != params.end()) ? absl::optional{params.at(key)} - : absl::nullopt; -} - -// Helper method to get the format parameter. -absl::optional formatParam(const Http::Utility::QueryParams& params) { - return queryParam(params, "format"); -} - // Helper method to get the resource parameter. absl::optional resourceParam(const Http::Utility::QueryParams& params) { - return queryParam(params, "resource"); + return Utility::queryParam(params, "resource"); } // Helper method to get the mask parameter. absl::optional maskParam(const Http::Utility::QueryParams& params) { - return queryParam(params, "mask"); + return Utility::queryParam(params, "mask"); } // Helper method that ensures that we've setting flags based on all the health flag values on the @@ -384,6 +346,7 @@ void AdminImpl::addCircuitSettings(const std::string& cluster_name, const std::s resource_manager.retries().max())); } +// TODO(efimki): Add support of text readouts stats. void AdminImpl::writeClustersAsJson(Buffer::Instance& response) { envoy::admin::v3::Clusters clusters; for (auto& cluster_pair : server_.clusterManager().clusters()) { @@ -461,6 +424,7 @@ void AdminImpl::writeClustersAsJson(Buffer::Instance& response) { response.add(MessageUtil::getJsonStringFromMessage(clusters, true)); // pretty-print } +// TODO(efimki): Add support of text readouts stats. void AdminImpl::writeClustersAsText(Buffer::Instance& response) { for (auto& cluster : server_.clusterManager().clusters()) { addOutlierInfo(cluster.second.get().info()->name(), cluster.second.get().outlierDetector(), @@ -546,7 +510,7 @@ Http::Code AdminImpl::handlerClusters(absl::string_view url, Http::ResponseHeaderMap& response_headers, Buffer::Instance& response, AdminStream&) { Http::Utility::QueryParams query_params = Http::Utility::parseQueryString(url); - const auto format_value = formatParam(query_params); + const auto format_value = Utility::formatParam(query_params); if (format_value.has_value() && format_value.value() == "json") { writeClustersAsJson(response); @@ -804,54 +768,6 @@ Http::Code AdminImpl::handlerDrainListeners(absl::string_view url, Http::Respons return Http::Code::OK; } -Http::Code AdminImpl::handlerResetCounters(absl::string_view, Http::ResponseHeaderMap&, - Buffer::Instance& response, AdminStream&) { - for (const Stats::CounterSharedPtr& counter : server_.stats().counters()) { - counter->reset(); - } - server_.stats().symbolTable().clearRecentLookups(); - response.add("OK\n"); - return Http::Code::OK; -} - -Http::Code AdminImpl::handlerStatsRecentLookups(absl::string_view, Http::ResponseHeaderMap&, - Buffer::Instance& response, AdminStream&) { - Stats::SymbolTable& symbol_table = server_.stats().symbolTable(); - std::string table; - const uint64_t total = - symbol_table.getRecentLookups([&table](absl::string_view name, uint64_t count) { - table += fmt::format("{:8d} {}\n", count, name); - }); - if (table.empty() && symbol_table.recentLookupCapacity() == 0) { - table = "Lookup tracking is not enabled. Use /stats/recentlookups/enable to enable.\n"; - } else { - response.add(" Count Lookup\n"); - } - response.add(absl::StrCat(table, "\ntotal: ", total, "\n")); - return Http::Code::OK; -} - -Http::Code AdminImpl::handlerStatsRecentLookupsClear(absl::string_view, Http::ResponseHeaderMap&, - Buffer::Instance& response, AdminStream&) { - server_.stats().symbolTable().clearRecentLookups(); - response.add("OK\n"); - return Http::Code::OK; -} - -Http::Code AdminImpl::handlerStatsRecentLookupsDisable(absl::string_view, Http::ResponseHeaderMap&, - Buffer::Instance& response, AdminStream&) { - server_.stats().symbolTable().setRecentLookupCapacity(0); - response.add("OK\n"); - return Http::Code::OK; -} - -Http::Code AdminImpl::handlerStatsRecentLookupsEnable(absl::string_view, Http::ResponseHeaderMap&, - Buffer::Instance& response, AdminStream&) { - server_.stats().symbolTable().setRecentLookupCapacity(RecentLookupsCapacity); - response.add("OK\n"); - return Http::Code::OK; -} - Http::Code AdminImpl::handlerServerInfo(absl::string_view, Http::ResponseHeaderMap& headers, Buffer::Instance& response, AdminStream&) { const std::time_t current_time = @@ -889,250 +805,6 @@ Http::Code AdminImpl::handlerReady(absl::string_view, Http::ResponseHeaderMap&, return code; } -Http::Code AdminImpl::handlerStats(absl::string_view url, Http::ResponseHeaderMap& response_headers, - Buffer::Instance& response, AdminStream& admin_stream) { - Http::Code rc = Http::Code::OK; - const Http::Utility::QueryParams params = Http::Utility::parseQueryString(url); - - const bool used_only = params.find("usedonly") != params.end(); - absl::optional regex; - if (!filterParam(params, response, regex)) { - return Http::Code::BadRequest; - } - - std::map all_stats; - for (const Stats::CounterSharedPtr& counter : server_.stats().counters()) { - if (shouldShowMetric(*counter, used_only, regex)) { - all_stats.emplace(counter->name(), counter->value()); - } - } - - for (const Stats::GaugeSharedPtr& gauge : server_.stats().gauges()) { - if (shouldShowMetric(*gauge, used_only, regex)) { - ASSERT(gauge->importMode() != Stats::Gauge::ImportMode::Uninitialized); - all_stats.emplace(gauge->name(), gauge->value()); - } - } - - if (const auto format_value = formatParam(params)) { - if (format_value.value() == "json") { - response_headers.setReferenceContentType(Http::Headers::get().ContentTypeValues.Json); - response.add( - AdminImpl::statsAsJson(all_stats, server_.stats().histograms(), used_only, regex)); - } else if (format_value.value() == "prometheus") { - return handlerPrometheusStats(url, response_headers, response, admin_stream); - } else { - response.add("usage: /stats?format=json or /stats?format=prometheus \n"); - response.add("\n"); - rc = Http::Code::NotFound; - } - } else { // Display plain stats if format query param is not there. - for (const auto& stat : all_stats) { - response.add(fmt::format("{}: {}\n", stat.first, stat.second)); - } - // TODO(ramaraochavali): See the comment in ThreadLocalStoreImpl::histograms() for why we use a - // multimap here. This makes sure that duplicate histograms get output. When shared storage is - // implemented this can be switched back to a normal map. - std::multimap all_histograms; - for (const Stats::ParentHistogramSharedPtr& histogram : server_.stats().histograms()) { - if (shouldShowMetric(*histogram, used_only, regex)) { - all_histograms.emplace(histogram->name(), histogram->quantileSummary()); - } - } - for (const auto& histogram : all_histograms) { - response.add(fmt::format("{}: {}\n", histogram.first, histogram.second)); - } - } - return rc; -} - -Http::Code AdminImpl::handlerPrometheusStats(absl::string_view path_and_query, - Http::ResponseHeaderMap&, Buffer::Instance& response, - AdminStream&) { - const Http::Utility::QueryParams params = Http::Utility::parseQueryString(path_and_query); - const bool used_only = params.find("usedonly") != params.end(); - absl::optional regex; - if (!filterParam(params, response, regex)) { - return Http::Code::BadRequest; - } - PrometheusStatsFormatter::statsAsPrometheus(server_.stats().counters(), server_.stats().gauges(), - server_.stats().histograms(), response, used_only, - regex); - return Http::Code::OK; -} - -std::string PrometheusStatsFormatter::sanitizeName(const std::string& name) { - // The name must match the regex [a-zA-Z_][a-zA-Z0-9_]* as required by - // prometheus. Refer to https://prometheus.io/docs/concepts/data_model/. - std::string stats_name = std::regex_replace(name, PromRegex, "_"); - if (stats_name[0] >= '0' && stats_name[0] <= '9') { - return absl::StrCat("_", stats_name); - } else { - return stats_name; - } -} - -std::string PrometheusStatsFormatter::formattedTags(const std::vector& tags) { - std::vector buf; - buf.reserve(tags.size()); - for (const Stats::Tag& tag : tags) { - buf.push_back(fmt::format("{}=\"{}\"", sanitizeName(tag.name_), tag.value_)); - } - return absl::StrJoin(buf, ","); -} - -std::string PrometheusStatsFormatter::metricName(const std::string& extracted_name) { - // Add namespacing prefix to avoid conflicts, as per best practice: - // https://prometheus.io/docs/practices/naming/#metric-names - // Also, naming conventions on https://prometheus.io/docs/concepts/data_model/ - return sanitizeName(fmt::format("envoy_{0}", extracted_name)); -} - -uint64_t PrometheusStatsFormatter::statsAsPrometheus( - const std::vector& counters, - const std::vector& gauges, - const std::vector& histograms, Buffer::Instance& response, - const bool used_only, const absl::optional& regex) { - std::unordered_set metric_type_tracker; - for (const auto& counter : counters) { - if (!shouldShowMetric(*counter, used_only, regex)) { - continue; - } - - const std::string tags = formattedTags(counter->tags()); - const std::string metric_name = metricName(counter->tagExtractedName()); - if (metric_type_tracker.find(metric_name) == metric_type_tracker.end()) { - metric_type_tracker.insert(metric_name); - response.add(fmt::format("# TYPE {0} counter\n", metric_name)); - } - response.add(fmt::format("{0}{{{1}}} {2}\n", metric_name, tags, counter->value())); - } - - for (const auto& gauge : gauges) { - if (!shouldShowMetric(*gauge, used_only, regex)) { - continue; - } - - const std::string tags = formattedTags(gauge->tags()); - const std::string metric_name = metricName(gauge->tagExtractedName()); - if (metric_type_tracker.find(metric_name) == metric_type_tracker.end()) { - metric_type_tracker.insert(metric_name); - response.add(fmt::format("# TYPE {0} gauge\n", metric_name)); - } - response.add(fmt::format("{0}{{{1}}} {2}\n", metric_name, tags, gauge->value())); - } - - for (const auto& histogram : histograms) { - if (!shouldShowMetric(*histogram, used_only, regex)) { - continue; - } - - const std::string tags = formattedTags(histogram->tags()); - const std::string hist_tags = histogram->tags().empty() ? EMPTY_STRING : (tags + ","); - - const std::string metric_name = metricName(histogram->tagExtractedName()); - if (metric_type_tracker.find(metric_name) == metric_type_tracker.end()) { - metric_type_tracker.insert(metric_name); - response.add(fmt::format("# TYPE {0} histogram\n", metric_name)); - } - - const Stats::HistogramStatistics& stats = histogram->cumulativeStatistics(); - const std::vector& supported_buckets = stats.supportedBuckets(); - const std::vector& computed_buckets = stats.computedBuckets(); - for (size_t i = 0; i < supported_buckets.size(); ++i) { - double bucket = supported_buckets[i]; - uint64_t value = computed_buckets[i]; - // We want to print the bucket in a fixed point (non-scientific) format. The fmt library - // doesn't have a specific modifier to format as a fixed-point value only so we use the - // 'g' operator which prints the number in general fixed point format or scientific format - // with precision 50 to round the number up to 32 significant digits in fixed point format - // which should cover pretty much all cases - response.add(fmt::format("{0}_bucket{{{1}le=\"{2:.32g}\"}} {3}\n", metric_name, hist_tags, - bucket, value)); - } - - response.add(fmt::format("{0}_bucket{{{1}le=\"+Inf\"}} {2}\n", metric_name, hist_tags, - stats.sampleCount())); - response.add(fmt::format("{0}_sum{{{1}}} {2:.32g}\n", metric_name, tags, stats.sampleSum())); - response.add(fmt::format("{0}_count{{{1}}} {2}\n", metric_name, tags, stats.sampleCount())); - } - - return metric_type_tracker.size(); -} - -std::string -AdminImpl::statsAsJson(const std::map& all_stats, - const std::vector& all_histograms, - const bool used_only, const absl::optional regex, - const bool pretty_print) { - - ProtobufWkt::Struct document; - std::vector stats_array; - for (const auto& stat : all_stats) { - ProtobufWkt::Struct stat_obj; - auto* stat_obj_fields = stat_obj.mutable_fields(); - (*stat_obj_fields)["name"] = ValueUtil::stringValue(stat.first); - (*stat_obj_fields)["value"] = ValueUtil::numberValue(stat.second); - stats_array.push_back(ValueUtil::structValue(stat_obj)); - } - - ProtobufWkt::Struct histograms_obj; - auto* histograms_obj_fields = histograms_obj.mutable_fields(); - - ProtobufWkt::Struct histograms_obj_container; - auto* histograms_obj_container_fields = histograms_obj_container.mutable_fields(); - std::vector computed_quantile_array; - - bool found_used_histogram = false; - for (const Stats::ParentHistogramSharedPtr& histogram : all_histograms) { - if (shouldShowMetric(*histogram, used_only, regex)) { - if (!found_used_histogram) { - // It is not possible for the supported quantiles to differ across histograms, so it is ok - // to send them once. - Stats::HistogramStatisticsImpl empty_statistics; - std::vector supported_quantile_array; - for (double quantile : empty_statistics.supportedQuantiles()) { - supported_quantile_array.push_back(ValueUtil::numberValue(quantile * 100)); - } - (*histograms_obj_fields)["supported_quantiles"] = - ValueUtil::listValue(supported_quantile_array); - found_used_histogram = true; - } - - ProtobufWkt::Struct computed_quantile; - auto* computed_quantile_fields = computed_quantile.mutable_fields(); - (*computed_quantile_fields)["name"] = ValueUtil::stringValue(histogram->name()); - - std::vector computed_quantile_value_array; - for (size_t i = 0; i < histogram->intervalStatistics().supportedQuantiles().size(); ++i) { - ProtobufWkt::Struct computed_quantile_value; - auto* computed_quantile_value_fields = computed_quantile_value.mutable_fields(); - const auto& interval = histogram->intervalStatistics().computedQuantiles()[i]; - const auto& cumulative = histogram->cumulativeStatistics().computedQuantiles()[i]; - (*computed_quantile_value_fields)["interval"] = - std::isnan(interval) ? ValueUtil::nullValue() : ValueUtil::numberValue(interval); - (*computed_quantile_value_fields)["cumulative"] = - std::isnan(cumulative) ? ValueUtil::nullValue() : ValueUtil::numberValue(cumulative); - - computed_quantile_value_array.push_back(ValueUtil::structValue(computed_quantile_value)); - } - (*computed_quantile_fields)["values"] = ValueUtil::listValue(computed_quantile_value_array); - computed_quantile_array.push_back(ValueUtil::structValue(computed_quantile)); - } - } - - if (found_used_histogram) { - (*histograms_obj_fields)["computed_quantiles"] = ValueUtil::listValue(computed_quantile_array); - (*histograms_obj_container_fields)["histograms"] = ValueUtil::structValue(histograms_obj); - stats_array.push_back(ValueUtil::structValue(histograms_obj_container)); - } - - auto* document_fields = document.mutable_fields(); - (*document_fields)["stats"] = ValueUtil::listValue(stats_array); - - return MessageUtil::getJsonStringFromMessage(document, pretty_print, true); -} - Http::Code AdminImpl::handlerQuitQuitQuit(absl::string_view, Http::ResponseHeaderMap&, Buffer::Instance& response, AdminStream&) { server_.shutdown(); @@ -1144,7 +816,7 @@ Http::Code AdminImpl::handlerListenerInfo(absl::string_view url, Http::ResponseHeaderMap& response_headers, Buffer::Instance& response, AdminStream&) { const Http::Utility::QueryParams query_params = Http::Utility::parseQueryString(url); - const auto format_value = formatParam(query_params); + const auto format_value = Utility::formatParam(query_params); if (format_value.has_value() && format_value.value() == "json") { writeListenersAsJson(response); @@ -1349,25 +1021,25 @@ AdminImpl::AdminImpl(const std::string& profile_path, Server::Instance& server) false, false}, {"/quitquitquit", "exit the server", MAKE_ADMIN_HANDLER(handlerQuitQuitQuit), false, true}, - {"/reset_counters", "reset all counters to zero", - MAKE_ADMIN_HANDLER(handlerResetCounters), false, true}, + {"/reset_counters", "reset all counters to zero", StatsHandler::handlerResetCounters, + false, true}, {"/drain_listeners", "drain listeners", MAKE_ADMIN_HANDLER(handlerDrainListeners), false, true}, {"/server_info", "print server version/status information", MAKE_ADMIN_HANDLER(handlerServerInfo), false, false}, {"/ready", "print server state, return 200 if LIVE, otherwise return 503", MAKE_ADMIN_HANDLER(handlerReady), false, false}, - {"/stats", "print server stats", MAKE_ADMIN_HANDLER(handlerStats), false, false}, + {"/stats", "print server stats", StatsHandler::handlerStats, false, false}, {"/stats/prometheus", "print server stats in prometheus format", - MAKE_ADMIN_HANDLER(handlerPrometheusStats), false, false}, + StatsHandler::handlerPrometheusStats, false, false}, {"/stats/recentlookups", "Show recent stat-name lookups", - MAKE_ADMIN_HANDLER(handlerStatsRecentLookups), false, false}, + StatsHandler::handlerStatsRecentLookups, false, false}, {"/stats/recentlookups/clear", "clear list of stat-name lookups and counter", - MAKE_ADMIN_HANDLER(handlerStatsRecentLookupsClear), false, true}, + StatsHandler::handlerStatsRecentLookupsClear, false, true}, {"/stats/recentlookups/disable", "disable recording of reset stat-name lookup names", - MAKE_ADMIN_HANDLER(handlerStatsRecentLookupsDisable), false, true}, + StatsHandler::handlerStatsRecentLookupsDisable, false, true}, {"/stats/recentlookups/enable", "enable recording of reset stat-name lookup names", - MAKE_ADMIN_HANDLER(handlerStatsRecentLookupsEnable), false, true}, + StatsHandler::handlerStatsRecentLookupsEnable, false, true}, {"/listeners", "print listener info", MAKE_ADMIN_HANDLER(handlerListenerInfo), false, false}, {"/runtime", "print runtime values", MAKE_ADMIN_HANDLER(handlerRuntime), false, false}, @@ -1429,7 +1101,12 @@ Http::Code AdminImpl::runCallback(absl::string_view path_and_query, break; } } - code = handler.handler_(path_and_query, response_headers, response, admin_stream); + if (handler.requires_server_) { + code = handler.handler_with_server_(path_and_query, response_headers, response, + admin_stream, server_); + } else { + code = handler.handler_(path_and_query, response_headers, response, admin_stream); + } Memory::Utils::tryShrinkHeap(); break; } @@ -1576,7 +1253,7 @@ void AdminImpl::closeSocket() { void AdminImpl::addListenerToHandler(Network::ConnectionHandler* handler) { if (listener_) { - handler->addListener(*listener_); + handler->addListener(absl::nullopt, *listener_); } } diff --git a/source/server/http/admin.h b/source/server/http/admin.h index 82332658de56..19dada14018f 100644 --- a/source/server/http/admin.h +++ b/source/server/http/admin.h @@ -21,7 +21,6 @@ #include "envoy/server/admin.h" #include "envoy/server/instance.h" #include "envoy/server/listener_manager.h" -#include "envoy/stats/scope.h" #include "envoy/upstream/outlier_detection.h" #include "envoy/upstream/resource_manager.h" @@ -179,16 +178,33 @@ class AdminImpl : public Admin, }; } + using HandlerWithServerCb = std::function; + private: /** * Individual admin handler including prefix, help text, and callback. */ struct UrlHandler { + UrlHandler(std::string prefix, std::string help_text, HandlerCb handler, bool removable, + bool mutates_server_state) + : prefix_(prefix), help_text_(help_text), handler_(handler), removable_(removable), + mutates_server_state_(mutates_server_state), requires_server_(false) {} + + UrlHandler(std::string prefix, std::string help_text, HandlerWithServerCb handler_with_server, + bool removable, bool mutates_server_state) + : prefix_(prefix), help_text_(help_text), handler_with_server_(handler_with_server), + removable_(removable), mutates_server_state_(mutates_server_state), + requires_server_(true) {} + const std::string prefix_; const std::string help_text_; const HandlerCb handler_; + const HandlerWithServerCb handler_with_server_; const bool removable_; const bool mutates_server_state_; + const bool requires_server_; }; /** @@ -234,8 +250,6 @@ class AdminImpl : public Admin, TimeSource& time_source_; }; - friend class AdminStatsTest; - /** * Attempt to change the log level of a logger or all loggers * @param params supplies the incoming endpoint query params. @@ -274,18 +288,6 @@ class AdminImpl : public Admin, addResourceToDump(envoy::admin::v3::ConfigDump& dump, const absl::optional& mask, const std::string& resource) const; - template - static bool shouldShowMetric(const StatType& metric, const bool used_only, - const absl::optional& regex) { - return ((!used_only || metric.used()) && - (!regex.has_value() || std::regex_search(metric.name(), regex.value()))); - } - static std::string statsAsJson(const std::map& all_stats, - const std::vector& all_histograms, - bool used_only, - const absl::optional regex = absl::nullopt, - bool pretty_print = false); - std::vector sortedHandlers() const; envoy::admin::v3::ServerInfo::State serverState(); /** @@ -340,33 +342,12 @@ class AdminImpl : public Admin, Http::Code handlerDrainListeners(absl::string_view path_and_query, Http::ResponseHeaderMap& response_headers, Buffer::Instance& response, AdminStream&); - Http::Code handlerResetCounters(absl::string_view path_and_query, - Http::ResponseHeaderMap& response_headers, - Buffer::Instance& response, AdminStream&); - Http::Code handlerStatsRecentLookups(absl::string_view path_and_query, - Http::ResponseHeaderMap& response_headers, - Buffer::Instance& response, AdminStream&); - Http::Code handlerStatsRecentLookupsClear(absl::string_view path_and_query, - Http::ResponseHeaderMap& response_headers, - Buffer::Instance& response, AdminStream&); - Http::Code handlerStatsRecentLookupsDisable(absl::string_view path_and_query, - Http::ResponseHeaderMap& response_headers, - Buffer::Instance& response, AdminStream&); - Http::Code handlerStatsRecentLookupsEnable(absl::string_view path_and_query, - Http::ResponseHeaderMap& response_headers, - Buffer::Instance& response, AdminStream&); Http::Code handlerServerInfo(absl::string_view path_and_query, Http::ResponseHeaderMap& response_headers, Buffer::Instance& response, AdminStream&); Http::Code handlerReady(absl::string_view path_and_query, Http::ResponseHeaderMap& response_headers, Buffer::Instance& response, AdminStream&); - Http::Code handlerStats(absl::string_view path_and_query, - Http::ResponseHeaderMap& response_headers, Buffer::Instance& response, - AdminStream&); - Http::Code handlerPrometheusStats(absl::string_view path_and_query, - Http::ResponseHeaderMap& response_headers, - Buffer::Instance& response, AdminStream&); Http::Code handlerRuntime(absl::string_view path_and_query, Http::ResponseHeaderMap& response_headers, Buffer::Instance& response, AdminStream&); @@ -494,50 +475,5 @@ class AdminImpl : public Admin, const AdminInternalAddressConfig internal_address_config_; }; -/** - * Formatter for metric/labels exported to Prometheus. - * - * See: https://prometheus.io/docs/concepts/data_model - */ -class PrometheusStatsFormatter { -public: - /** - * Extracts counters and gauges and relevant tags, appending them to - * the response buffer after sanitizing the metric / label names. - * @return uint64_t total number of metric types inserted in response. - */ - static uint64_t statsAsPrometheus(const std::vector& counters, - const std::vector& gauges, - const std::vector& histograms, - Buffer::Instance& response, const bool used_only, - const absl::optional& regex); - /** - * Format the given tags, returning a string as a comma-separated list - * of ="" pairs. - */ - static std::string formattedTags(const std::vector& tags); - /** - * Format the given metric name, prefixed with "envoy_". - */ - static std::string metricName(const std::string& extracted_name); - -private: - /** - * Take a string and sanitize it according to Prometheus conventions. - */ - static std::string sanitizeName(const std::string& name); - - /* - * Determine whether a metric has never been emitted and choose to - * not show it if we only wanted used metrics. - */ - template - static bool shouldShowMetric(const StatType& metric, const bool used_only, - const absl::optional& regex) { - return ((!used_only || metric.used()) && - (!regex.has_value() || std::regex_search(metric.name(), regex.value()))); - } -}; - } // namespace Server } // namespace Envoy diff --git a/source/server/http/stats_handler.cc b/source/server/http/stats_handler.cc new file mode 100644 index 000000000000..0d37267cf66f --- /dev/null +++ b/source/server/http/stats_handler.cc @@ -0,0 +1,342 @@ +#include "server/http/stats_handler.h" + +#include "common/common/empty_string.h" +#include "common/html/utility.h" +#include "common/http/headers.h" +#include "common/http/utility.h" + +#include "server/http/utils.h" + +namespace Envoy { +namespace Server { + +const uint64_t RecentLookupsCapacity = 100; + +namespace { +const std::regex& promRegex() { CONSTRUCT_ON_FIRST_USE(std::regex, "[^a-zA-Z0-9_]"); } +} // namespace + +Http::Code StatsHandler::handlerResetCounters(absl::string_view, Http::ResponseHeaderMap&, + Buffer::Instance& response, AdminStream&, + Server::Instance& server) { + for (const Stats::CounterSharedPtr& counter : server.stats().counters()) { + counter->reset(); + } + server.stats().symbolTable().clearRecentLookups(); + response.add("OK\n"); + return Http::Code::OK; +} + +Http::Code StatsHandler::handlerStatsRecentLookups(absl::string_view, Http::ResponseHeaderMap&, + Buffer::Instance& response, AdminStream&, + Server::Instance& server) { + Stats::SymbolTable& symbol_table = server.stats().symbolTable(); + std::string table; + const uint64_t total = + symbol_table.getRecentLookups([&table](absl::string_view name, uint64_t count) { + table += fmt::format("{:8d} {}\n", count, name); + }); + if (table.empty() && symbol_table.recentLookupCapacity() == 0) { + table = "Lookup tracking is not enabled. Use /stats/recentlookups/enable to enable.\n"; + } else { + response.add(" Count Lookup\n"); + } + response.add(absl::StrCat(table, "\ntotal: ", total, "\n")); + return Http::Code::OK; +} + +Http::Code StatsHandler::handlerStatsRecentLookupsClear(absl::string_view, Http::ResponseHeaderMap&, + Buffer::Instance& response, AdminStream&, + Server::Instance& server) { + server.stats().symbolTable().clearRecentLookups(); + response.add("OK\n"); + return Http::Code::OK; +} + +Http::Code StatsHandler::handlerStatsRecentLookupsDisable(absl::string_view, + Http::ResponseHeaderMap&, + Buffer::Instance& response, AdminStream&, + Server::Instance& server) { + server.stats().symbolTable().setRecentLookupCapacity(0); + response.add("OK\n"); + return Http::Code::OK; +} + +Http::Code StatsHandler::handlerStatsRecentLookupsEnable(absl::string_view, + Http::ResponseHeaderMap&, + Buffer::Instance& response, AdminStream&, + Server::Instance& server) { + server.stats().symbolTable().setRecentLookupCapacity(RecentLookupsCapacity); + response.add("OK\n"); + return Http::Code::OK; +} + +Http::Code StatsHandler::handlerStats(absl::string_view url, + Http::ResponseHeaderMap& response_headers, + Buffer::Instance& response, AdminStream& admin_stream, + Server::Instance& server) { + Http::Code rc = Http::Code::OK; + const Http::Utility::QueryParams params = Http::Utility::parseQueryString(url); + + const bool used_only = params.find("usedonly") != params.end(); + absl::optional regex; + if (!Utility::filterParam(params, response, regex)) { + return Http::Code::BadRequest; + } + + std::map all_stats; + for (const Stats::CounterSharedPtr& counter : server.stats().counters()) { + if (shouldShowMetric(*counter, used_only, regex)) { + all_stats.emplace(counter->name(), counter->value()); + } + } + + for (const Stats::GaugeSharedPtr& gauge : server.stats().gauges()) { + if (shouldShowMetric(*gauge, used_only, regex)) { + ASSERT(gauge->importMode() != Stats::Gauge::ImportMode::Uninitialized); + all_stats.emplace(gauge->name(), gauge->value()); + } + } + + std::map text_readouts; + for (const auto& text_readout : server.stats().textReadouts()) { + if (shouldShowMetric(*text_readout, used_only, regex)) { + text_readouts.emplace(text_readout->name(), text_readout->value()); + } + } + + if (const auto format_value = Utility::formatParam(params)) { + if (format_value.value() == "json") { + response_headers.setReferenceContentType(Http::Headers::get().ContentTypeValues.Json); + response.add( + statsAsJson(all_stats, text_readouts, server.stats().histograms(), used_only, regex)); + } else if (format_value.value() == "prometheus") { + return handlerPrometheusStats(url, response_headers, response, admin_stream, server); + } else { + response.add("usage: /stats?format=json or /stats?format=prometheus \n"); + response.add("\n"); + rc = Http::Code::NotFound; + } + } else { // Display plain stats if format query param is not there. + for (const auto& text_readout : text_readouts) { + response.add(fmt::format("{}: \"{}\"\n", text_readout.first, + Html::Utility::sanitize(text_readout.second))); + } + for (const auto& stat : all_stats) { + response.add(fmt::format("{}: {}\n", stat.first, stat.second)); + } + // TODO(ramaraochavali): See the comment in ThreadLocalStoreImpl::histograms() for why we use a + // multimap here. This makes sure that duplicate histograms get output. When shared storage is + // implemented this can be switched back to a normal map. + std::multimap all_histograms; + for (const Stats::ParentHistogramSharedPtr& histogram : server.stats().histograms()) { + if (shouldShowMetric(*histogram, used_only, regex)) { + all_histograms.emplace(histogram->name(), histogram->quantileSummary()); + } + } + for (const auto& histogram : all_histograms) { + response.add(fmt::format("{}: {}\n", histogram.first, histogram.second)); + } + } + return rc; +} + +Http::Code StatsHandler::handlerPrometheusStats(absl::string_view path_and_query, + Http::ResponseHeaderMap&, + Buffer::Instance& response, AdminStream&, + Server::Instance& server) { + const Http::Utility::QueryParams params = Http::Utility::parseQueryString(path_and_query); + const bool used_only = params.find("usedonly") != params.end(); + absl::optional regex; + if (!Utility::filterParam(params, response, regex)) { + return Http::Code::BadRequest; + } + PrometheusStatsFormatter::statsAsPrometheus(server.stats().counters(), server.stats().gauges(), + server.stats().histograms(), response, used_only, + regex); + return Http::Code::OK; +} + +std::string PrometheusStatsFormatter::sanitizeName(const std::string& name) { + // The name must match the regex [a-zA-Z_][a-zA-Z0-9_]* as required by + // prometheus. Refer to https://prometheus.io/docs/concepts/data_model/. + std::string stats_name = std::regex_replace(name, promRegex(), "_"); + if (stats_name[0] >= '0' && stats_name[0] <= '9') { + return absl::StrCat("_", stats_name); + } else { + return stats_name; + } +} + +std::string PrometheusStatsFormatter::formattedTags(const std::vector& tags) { + std::vector buf; + buf.reserve(tags.size()); + for (const Stats::Tag& tag : tags) { + buf.push_back(fmt::format("{}=\"{}\"", sanitizeName(tag.name_), tag.value_)); + } + return absl::StrJoin(buf, ","); +} + +std::string PrometheusStatsFormatter::metricName(const std::string& extracted_name) { + // Add namespacing prefix to avoid conflicts, as per best practice: + // https://prometheus.io/docs/practices/naming/#metric-names + // Also, naming conventions on https://prometheus.io/docs/concepts/data_model/ + return sanitizeName(fmt::format("envoy_{0}", extracted_name)); +} + +// TODO(efimki): Add support of text readouts stats. +uint64_t PrometheusStatsFormatter::statsAsPrometheus( + const std::vector& counters, + const std::vector& gauges, + const std::vector& histograms, Buffer::Instance& response, + const bool used_only, const absl::optional& regex) { + std::unordered_set metric_type_tracker; + for (const auto& counter : counters) { + if (!shouldShowMetric(*counter, used_only, regex)) { + continue; + } + + const std::string tags = formattedTags(counter->tags()); + const std::string metric_name = metricName(counter->tagExtractedName()); + if (metric_type_tracker.find(metric_name) == metric_type_tracker.end()) { + metric_type_tracker.insert(metric_name); + response.add(fmt::format("# TYPE {0} counter\n", metric_name)); + } + response.add(fmt::format("{0}{{{1}}} {2}\n", metric_name, tags, counter->value())); + } + + for (const auto& gauge : gauges) { + if (!shouldShowMetric(*gauge, used_only, regex)) { + continue; + } + + const std::string tags = formattedTags(gauge->tags()); + const std::string metric_name = metricName(gauge->tagExtractedName()); + if (metric_type_tracker.find(metric_name) == metric_type_tracker.end()) { + metric_type_tracker.insert(metric_name); + response.add(fmt::format("# TYPE {0} gauge\n", metric_name)); + } + response.add(fmt::format("{0}{{{1}}} {2}\n", metric_name, tags, gauge->value())); + } + + for (const auto& histogram : histograms) { + if (!shouldShowMetric(*histogram, used_only, regex)) { + continue; + } + + const std::string tags = formattedTags(histogram->tags()); + const std::string hist_tags = histogram->tags().empty() ? EMPTY_STRING : (tags + ","); + + const std::string metric_name = metricName(histogram->tagExtractedName()); + if (metric_type_tracker.find(metric_name) == metric_type_tracker.end()) { + metric_type_tracker.insert(metric_name); + response.add(fmt::format("# TYPE {0} histogram\n", metric_name)); + } + + const Stats::HistogramStatistics& stats = histogram->cumulativeStatistics(); + const std::vector& supported_buckets = stats.supportedBuckets(); + const std::vector& computed_buckets = stats.computedBuckets(); + for (size_t i = 0; i < supported_buckets.size(); ++i) { + double bucket = supported_buckets[i]; + uint64_t value = computed_buckets[i]; + // We want to print the bucket in a fixed point (non-scientific) format. The fmt library + // doesn't have a specific modifier to format as a fixed-point value only so we use the + // 'g' operator which prints the number in general fixed point format or scientific format + // with precision 50 to round the number up to 32 significant digits in fixed point format + // which should cover pretty much all cases + response.add(fmt::format("{0}_bucket{{{1}le=\"{2:.32g}\"}} {3}\n", metric_name, hist_tags, + bucket, value)); + } + + response.add(fmt::format("{0}_bucket{{{1}le=\"+Inf\"}} {2}\n", metric_name, hist_tags, + stats.sampleCount())); + response.add(fmt::format("{0}_sum{{{1}}} {2:.32g}\n", metric_name, tags, stats.sampleSum())); + response.add(fmt::format("{0}_count{{{1}}} {2}\n", metric_name, tags, stats.sampleCount())); + } + + return metric_type_tracker.size(); +} + +std::string +StatsHandler::statsAsJson(const std::map& all_stats, + const std::map& text_readouts, + const std::vector& all_histograms, + const bool used_only, const absl::optional regex, + const bool pretty_print) { + + ProtobufWkt::Struct document; + std::vector stats_array; + for (const auto& text_readout : text_readouts) { + ProtobufWkt::Struct stat_obj; + auto* stat_obj_fields = stat_obj.mutable_fields(); + (*stat_obj_fields)["name"] = ValueUtil::stringValue(text_readout.first); + (*stat_obj_fields)["value"] = ValueUtil::stringValue(text_readout.second); + stats_array.push_back(ValueUtil::structValue(stat_obj)); + } + for (const auto& stat : all_stats) { + ProtobufWkt::Struct stat_obj; + auto* stat_obj_fields = stat_obj.mutable_fields(); + (*stat_obj_fields)["name"] = ValueUtil::stringValue(stat.first); + (*stat_obj_fields)["value"] = ValueUtil::numberValue(stat.second); + stats_array.push_back(ValueUtil::structValue(stat_obj)); + } + + ProtobufWkt::Struct histograms_obj; + auto* histograms_obj_fields = histograms_obj.mutable_fields(); + + ProtobufWkt::Struct histograms_obj_container; + auto* histograms_obj_container_fields = histograms_obj_container.mutable_fields(); + std::vector computed_quantile_array; + + bool found_used_histogram = false; + for (const Stats::ParentHistogramSharedPtr& histogram : all_histograms) { + if (shouldShowMetric(*histogram, used_only, regex)) { + if (!found_used_histogram) { + // It is not possible for the supported quantiles to differ across histograms, so it is ok + // to send them once. + Stats::HistogramStatisticsImpl empty_statistics; + std::vector supported_quantile_array; + for (double quantile : empty_statistics.supportedQuantiles()) { + supported_quantile_array.push_back(ValueUtil::numberValue(quantile * 100)); + } + (*histograms_obj_fields)["supported_quantiles"] = + ValueUtil::listValue(supported_quantile_array); + found_used_histogram = true; + } + + ProtobufWkt::Struct computed_quantile; + auto* computed_quantile_fields = computed_quantile.mutable_fields(); + (*computed_quantile_fields)["name"] = ValueUtil::stringValue(histogram->name()); + + std::vector computed_quantile_value_array; + for (size_t i = 0; i < histogram->intervalStatistics().supportedQuantiles().size(); ++i) { + ProtobufWkt::Struct computed_quantile_value; + auto* computed_quantile_value_fields = computed_quantile_value.mutable_fields(); + const auto& interval = histogram->intervalStatistics().computedQuantiles()[i]; + const auto& cumulative = histogram->cumulativeStatistics().computedQuantiles()[i]; + (*computed_quantile_value_fields)["interval"] = + std::isnan(interval) ? ValueUtil::nullValue() : ValueUtil::numberValue(interval); + (*computed_quantile_value_fields)["cumulative"] = + std::isnan(cumulative) ? ValueUtil::nullValue() : ValueUtil::numberValue(cumulative); + + computed_quantile_value_array.push_back(ValueUtil::structValue(computed_quantile_value)); + } + (*computed_quantile_fields)["values"] = ValueUtil::listValue(computed_quantile_value_array); + computed_quantile_array.push_back(ValueUtil::structValue(computed_quantile)); + } + } + + if (found_used_histogram) { + (*histograms_obj_fields)["computed_quantiles"] = ValueUtil::listValue(computed_quantile_array); + (*histograms_obj_container_fields)["histograms"] = ValueUtil::structValue(histograms_obj); + stats_array.push_back(ValueUtil::structValue(histograms_obj_container)); + } + + auto* document_fields = document.mutable_fields(); + (*document_fields)["stats"] = ValueUtil::listValue(stats_array); + + return MessageUtil::getJsonStringFromMessage(document, pretty_print, true); +} + +} // namespace Server +} // namespace Envoy diff --git a/source/server/http/stats_handler.h b/source/server/http/stats_handler.h new file mode 100644 index 000000000000..ad4272c1d193 --- /dev/null +++ b/source/server/http/stats_handler.h @@ -0,0 +1,115 @@ +#pragma once + +#include +#include + +#include "envoy/buffer/buffer.h" +#include "envoy/http/codes.h" +#include "envoy/http/header_map.h" +#include "envoy/server/admin.h" +#include "envoy/server/instance.h" + +#include "common/stats/histogram_impl.h" + +#include "absl/strings/string_view.h" + +namespace Envoy { +namespace Server { + +class StatsHandler { + +public: + static Http::Code handlerResetCounters(absl::string_view path_and_query, + Http::ResponseHeaderMap& response_headers, + Buffer::Instance& response, AdminStream&, + Server::Instance& server); + static Http::Code handlerStatsRecentLookups(absl::string_view path_and_query, + Http::ResponseHeaderMap& response_headers, + Buffer::Instance& response, AdminStream&, + Server::Instance& server); + static Http::Code handlerStatsRecentLookupsClear(absl::string_view path_and_query, + Http::ResponseHeaderMap& response_headers, + Buffer::Instance& response, AdminStream&, + Server::Instance& server); + static Http::Code handlerStatsRecentLookupsDisable(absl::string_view path_and_query, + Http::ResponseHeaderMap& response_headers, + Buffer::Instance& response, AdminStream&, + Server::Instance& server); + static Http::Code handlerStatsRecentLookupsEnable(absl::string_view path_and_query, + Http::ResponseHeaderMap& response_headers, + Buffer::Instance& response, AdminStream&, + Server::Instance& server); + static Http::Code handlerStats(absl::string_view path_and_query, + Http::ResponseHeaderMap& response_headers, + Buffer::Instance& response, AdminStream&, + Server::Instance& server); + static Http::Code handlerPrometheusStats(absl::string_view path_and_query, + Http::ResponseHeaderMap& response_headers, + Buffer::Instance& response, AdminStream&, + Server::Instance& server); + +private: + template + static bool shouldShowMetric(const StatType& metric, const bool used_only, + const absl::optional& regex) { + return ((!used_only || metric.used()) && + (!regex.has_value() || std::regex_search(metric.name(), regex.value()))); + } + + friend class AdminStatsTest; + + static std::string statsAsJson(const std::map& all_stats, + const std::map& text_readouts, + const std::vector& all_histograms, + bool used_only, + const absl::optional regex = absl::nullopt, + bool pretty_print = false); +}; + +/** + * Formatter for metric/labels exported to Prometheus. + * + * See: https://prometheus.io/docs/concepts/data_model + */ +class PrometheusStatsFormatter { +public: + /** + * Extracts counters and gauges and relevant tags, appending them to + * the response buffer after sanitizing the metric / label names. + * @return uint64_t total number of metric types inserted in response. + */ + static uint64_t statsAsPrometheus(const std::vector& counters, + const std::vector& gauges, + const std::vector& histograms, + Buffer::Instance& response, const bool used_only, + const absl::optional& regex); + /** + * Format the given tags, returning a string as a comma-separated list + * of ="" pairs. + */ + static std::string formattedTags(const std::vector& tags); + /** + * Format the given metric name, prefixed with "envoy_". + */ + static std::string metricName(const std::string& extracted_name); + +private: + /** + * Take a string and sanitize it according to Prometheus conventions. + */ + static std::string sanitizeName(const std::string& name); + + /* + * Determine whether a metric has never been emitted and choose to + * not show it if we only wanted used metrics. + */ + template + static bool shouldShowMetric(const StatType& metric, const bool used_only, + const absl::optional& regex) { + return ((!used_only || metric.used()) && + (!regex.has_value() || std::regex_search(metric.name(), regex.value()))); + } +}; + +} // namespace Server +} // namespace Envoy diff --git a/source/server/http/utils.cc b/source/server/http/utils.cc index 4a63ea8cc56b..cd564784a566 100644 --- a/source/server/http/utils.cc +++ b/source/server/http/utils.cc @@ -37,6 +37,36 @@ void populateFallbackResponseHeaders(Http::Code code, Http::ResponseHeaderMap& h header_map.addReference(headers.XContentTypeOptions, headers.XContentTypeOptionValues.Nosniff); } +// Helper method to get filter parameter, or report an error for an invalid regex. +bool filterParam(Http::Utility::QueryParams params, Buffer::Instance& response, + absl::optional& regex) { + auto p = params.find("filter"); + if (p != params.end()) { + const std::string& pattern = p->second; + try { + regex = std::regex(pattern); + } catch (std::regex_error& error) { + // Include the offending pattern in the log, but not the error message. + response.add(fmt::format("Invalid regex: \"{}\"\n", error.what())); + ENVOY_LOG_MISC(error, "admin: Invalid regex: \"{}\": {}", error.what(), pattern); + return false; + } + } + return true; +} + +// Helper method to get the format parameter. +absl::optional formatParam(const Http::Utility::QueryParams& params) { + return queryParam(params, "format"); +} + +// Helper method to get a query parameter. +absl::optional queryParam(const Http::Utility::QueryParams& params, + const std::string& key) { + return (params.find(key) != params.end()) ? absl::optional{params.at(key)} + : absl::nullopt; +} + } // namespace Utility } // namespace Server } // namespace Envoy diff --git a/source/server/http/utils.h b/source/server/http/utils.h index 14a1d59dde24..fc091467450a 100644 --- a/source/server/http/utils.h +++ b/source/server/http/utils.h @@ -1,10 +1,13 @@ #pragma once +#include + #include "envoy/admin/v3/server_info.pb.h" #include "envoy/init/manager.h" #include "common/http/codes.h" #include "common/http/header_map_impl.h" +#include "common/http/utility.h" namespace Envoy { namespace Server { @@ -15,6 +18,14 @@ envoy::admin::v3::ServerInfo::State serverState(Init::Manager::State state, void populateFallbackResponseHeaders(Http::Code code, Http::ResponseHeaderMap& header_map); +bool filterParam(Http::Utility::QueryParams params, Buffer::Instance& response, + absl::optional& regex); + +absl::optional formatParam(const Http::Utility::QueryParams& params); + +absl::optional queryParam(const Http::Utility::QueryParams& params, + const std::string& key); + } // namespace Utility } // namespace Server } // namespace Envoy diff --git a/source/server/listener_manager_impl.cc b/source/server/listener_manager_impl.cc index e14faefffd3e..d6c8de15d828 100644 --- a/source/server/listener_manager_impl.cc +++ b/source/server/listener_manager_impl.cc @@ -245,6 +245,11 @@ DrainManagerPtr ProdListenerComponentFactory::createDrainManager( return DrainManagerPtr{new DrainManagerImpl(server_, drain_type)}; } +DrainingFilterChainsManager::DrainingFilterChainsManager(ListenerImplPtr&& draining_listener, + uint64_t workers_pending_removal) + : draining_listener_(std::move(draining_listener)), + workers_pending_removal_(workers_pending_removal) {} + ListenerManagerImpl::ListenerManagerImpl(Instance& server, ListenerComponentFactory& listener_factory, WorkerFactory& worker_factory, @@ -619,41 +624,56 @@ std::vector> ListenerManagerImpl return ret; } -void ListenerManagerImpl::addListenerToWorker(Worker& worker, ListenerImpl& listener, +void ListenerManagerImpl::addListenerToWorker(Worker& worker, + absl::optional overridden_listener, + ListenerImpl& listener, ListenerCompletionCallback completion_callback) { - worker.addListener(listener, [this, &listener, completion_callback](bool success) -> void { - // The add listener completion runs on the worker thread. Post back to the main thread to - // avoid locking. - server_.dispatcher().post([this, success, &listener, completion_callback]() -> void { - // It is theoretically possible for a listener to get added on 1 worker but not the others. - // The below check with onListenerCreateFailure() is there to ensure we execute the - // removal/logging/stats at most once on failure. Note also that drain/removal can race - // with addition. It's guaranteed that workers process remove after add so this should be - // fine. - if (!success && !listener.onListenerCreateFailure()) { - // TODO(mattklein123): In addition to a critical log and a stat, we should consider adding - // a startup option here to cause the server to exit. I think we - // probably want this at Lyft but I will do it in a follow up. - ENVOY_LOG(critical, "listener '{}' failed to listen on address '{}' on worker", - listener.name(), listener.listenSocketFactory().localAddress()->asString()); - stats_.listener_create_failure_.inc(); - removeListener(listener.name()); - } - if (success) { + if (overridden_listener.has_value()) { + ENVOY_LOG(debug, "replacing existing listener {}", overridden_listener.value()); + worker.addListener(overridden_listener, listener, [this, completion_callback](bool) -> void { + server_.dispatcher().post([this, completion_callback]() -> void { stats_.listener_create_success_.inc(); - } - if (completion_callback) { - completion_callback(); - } + if (completion_callback) { + completion_callback(); + } + }); }); - }); + return; + } + worker.addListener( + overridden_listener, listener, [this, &listener, completion_callback](bool success) -> void { + // The add listener completion runs on the worker thread. Post back to the main thread to + // avoid locking. + server_.dispatcher().post([this, success, &listener, completion_callback]() -> void { + // It is theoretically possible for a listener to get added on 1 worker but not the + // others. The below check with onListenerCreateFailure() is there to ensure we execute + // the removal/logging/stats at most once on failure. Note also that drain/removal can + // race with addition. It's guaranteed that workers process remove after add so this + // should be fine. + if (!success && !listener.onListenerCreateFailure()) { + // TODO(mattklein123): In addition to a critical log and a stat, we should consider + // adding a startup option here to cause the server to exit. I think we probably want + // this at Lyft but I will do it in a follow up. + ENVOY_LOG(critical, "listener '{}' failed to listen on address '{}' on worker", + listener.name(), listener.listenSocketFactory().localAddress()->asString()); + stats_.listener_create_failure_.inc(); + removeListener(listener.name()); + } + if (success) { + stats_.listener_create_success_.inc(); + } + if (completion_callback) { + completion_callback(); + } + }); + }); } void ListenerManagerImpl::onListenerWarmed(ListenerImpl& listener) { // The warmed listener should be added first so that the worker will accept new connections // when it stops listening on the old listener. for (const auto& worker : workers_) { - addListenerToWorker(*worker, listener, /* callback */ nullptr); + addListenerToWorker(*worker, absl::nullopt, listener, nullptr); } auto existing_active_listener = getListenerByName(active_listeners_, listener.name()); @@ -674,6 +694,57 @@ void ListenerManagerImpl::onListenerWarmed(ListenerImpl& listener) { updateWarmingActiveGauges(); } +void ListenerManagerImpl::drainFilterChains(ListenerImplPtr&& listener) { + // First add the listener to the draining list. + std::list::iterator draining_group = + draining_filter_chains_manager_.emplace(draining_filter_chains_manager_.begin(), + std::move(listener), workers_.size()); + int filter_chain_size = draining_group->getDrainingFilterChains().size(); + + // Using set() avoids a multiple modifiers problem during the multiple processes phase of hot + // restart. Same below inside the lambda. + // TODO(lambdai): Currently the number of DrainFilterChains objects are tracked: + // len(filter_chains). What we really need is accumulate(filter_chains, filter_chains: + // len(filter_chains)) + stats_.total_filter_chains_draining_.set(draining_filter_chains_manager_.size()); + + draining_group->getDrainingListener().debugLog( + absl::StrCat("draining ", filter_chain_size, " filter chains in listener ", + draining_group->getDrainingListener().name())); + + // Start the drain sequence which completes when the listener's drain manager has completed + // draining at whatever the server configured drain times are. + draining_group->startDrainSequence( + server_.options().drainTime(), server_.dispatcher(), [this, draining_group]() -> void { + draining_group->getDrainingListener().debugLog( + absl::StrCat("removing draining filter chains from listener ", + draining_group->getDrainingListener().name())); + for (const auto& worker : workers_) { + // Once the drain time has completed via the drain manager's timer, we tell the workers + // to remove the filter chains. + worker->removeFilterChains( + draining_group->getDrainingListenerTag(), draining_group->getDrainingFilterChains(), + [this, draining_group]() -> void { + // The remove listener completion is called on the worker thread. We post back to + // the main thread to avoid locking. This makes sure that we don't destroy the + // listener while filters might still be using its context (stats, etc.). + server_.dispatcher().post([this, draining_group]() -> void { + if (draining_group->decWorkersPendingRemoval() == 0) { + draining_group->getDrainingListener().debugLog( + absl::StrCat("draining filter chains from listener ", + draining_group->getDrainingListener().name(), " complete")); + draining_filter_chains_manager_.erase(draining_group); + stats_.total_filter_chains_draining_.set( + draining_filter_chains_manager_.size()); + } + }); + }); + } + }); + + updateWarmingActiveGauges(); +} + uint64_t ListenerManagerImpl::numConnections() const { uint64_t num_connections = 0; for (const auto& worker : workers_) { @@ -736,7 +807,7 @@ void ListenerManagerImpl::startWorkers(GuardDog& guard_dog) { ENVOY_LOG(debug, "starting worker {}", i); ASSERT(warming_listeners_.empty()); for (const auto& listener : active_listeners_) { - addListenerToWorker(*worker, *listener, [this, listeners_pending_init]() { + addListenerToWorker(*worker, absl::nullopt, *listener, [this, listeners_pending_init]() { if (--(*listeners_pending_init) == 0) { stats_.workers_started_.set(1); } diff --git a/source/server/listener_manager_impl.h b/source/server/listener_manager_impl.h index 3a897e82946b..cfa2603322c7 100644 --- a/source/server/listener_manager_impl.h +++ b/source/server/listener_manager_impl.h @@ -115,6 +115,7 @@ using ListenerImplPtr = std::unique_ptr; COUNTER(listener_modified) \ COUNTER(listener_removed) \ COUNTER(listener_stopped) \ + GAUGE(total_filter_chains_draining, NeverImport) \ GAUGE(total_listeners_active, NeverImport) \ GAUGE(total_listeners_draining, NeverImport) \ GAUGE(total_listeners_warming, NeverImport) \ @@ -127,6 +128,39 @@ struct ListenerManagerStats { ALL_LISTENER_MANAGER_STATS(GENERATE_COUNTER_STRUCT, GENERATE_GAUGE_STRUCT) }; +/** + * Provides the draining filter chains and the functionality to schedule listener destroy. + */ +class DrainingFilterChainsManager { +public: + DrainingFilterChainsManager(ListenerImplPtr&& draining_listener, + uint64_t workers_pending_removal); + uint64_t getDrainingListenerTag() const { return draining_listener_->listenerTag(); } + const std::list& getDrainingFilterChains() const { + return draining_filter_chains_; + } + ListenerImpl& getDrainingListener() { return *draining_listener_; } + uint64_t decWorkersPendingRemoval() { return --workers_pending_removal_; } + + // Schedule listener destroy. + void startDrainSequence(std::chrono::seconds drain_time, Event::Dispatcher& dispatcher, + std::function completion) { + drain_sequence_completion_ = completion; + ASSERT(!drain_timer_); + + drain_timer_ = dispatcher.createTimer([this]() -> void { drain_sequence_completion_(); }); + drain_timer_->enableTimer(drain_time); + } + +private: + ListenerImplPtr draining_listener_; + std::list draining_filter_chains_; + + uint64_t workers_pending_removal_; + Event::TimerPtr drain_timer_; + std::function drain_sequence_completion_; +}; + /** * Implementation of ListenerManager. */ @@ -155,6 +189,27 @@ class ListenerManagerImpl : public ListenerManager, Logger::Loggable overridden_listener, + ListenerImpl& listener, + std::function completion_callback) { + addListenerToWorker(worker, overridden_listener, listener, completion_callback); + } + // Erase the the listener draining filter chain from active listeners and then start the drain + // sequence. + void drainFilterChainsForTest(ListenerImpl* listener_raw_ptr) { + auto iter = std::find_if(active_listeners_.begin(), active_listeners_.end(), + [listener_raw_ptr](const ListenerImplPtr& ptr) { + return ptr != nullptr && ptr.get() == listener_raw_ptr; + }); + ASSERT(iter != active_listeners_.end()); + + ListenerImplPtr listener_impl_ptr = std::move(*iter); + active_listeners_.erase(iter); + drainFilterChains(std::move(listener_impl_ptr)); + } + Instance& server_; ListenerComponentFactory& factory_; @@ -176,8 +231,9 @@ class ListenerManagerImpl : public ListenerManager, Logger::Loggable overridden_listener, + ListenerImpl& listener, ListenerCompletionCallback completion_callback); + ProtobufTypes::MessagePtr dumpListenerConfigs(); static ListenerManagerStats generateStats(Stats::Scope& scope); static bool hasListenerWithAddress(const ListenerList& list, @@ -207,6 +263,13 @@ class ListenerManagerImpl : public ListenerManager, Logger::Loggable draining_listeners_; + std::list draining_filter_chains_manager_; + std::list workers_; bool workers_started_{}; absl::optional stop_listeners_type_; diff --git a/source/server/worker_impl.cc b/source/server/worker_impl.cc index d04e547b481e..17e02486e5f8 100644 --- a/source/server/worker_impl.cc +++ b/source/server/worker_impl.cc @@ -33,14 +33,15 @@ WorkerImpl::WorkerImpl(ThreadLocal::Instance& tls, ListenerHooks& hooks, [this](OverloadActionState state) { stopAcceptingConnectionsCb(state); }); } -void WorkerImpl::addListener(Network::ListenerConfig& listener, AddListenerCompletion completion) { +void WorkerImpl::addListener(absl::optional overridden_listener, + Network::ListenerConfig& listener, AddListenerCompletion completion) { // All listener additions happen via post. However, we must deal with the case where the listener // can not be created on the worker. There is a race condition where 2 processes can successfully // bind to an address, but then fail to listen() with EADDRINUSE. During initial startup, we want // to surface this. - dispatcher_->post([this, &listener, completion]() -> void { + dispatcher_->post([this, overridden_listener, &listener, completion]() -> void { try { - handler_->addListener(listener); + handler_->addListener(overridden_listener, listener); hooks_.onWorkerListenerAdded(); completion(true); } catch (const Network::CreateListenerException& e) { @@ -68,6 +69,16 @@ void WorkerImpl::removeListener(Network::ListenerConfig& listener, }); } +void WorkerImpl::removeFilterChains(uint64_t listener_tag, + const std::list& filter_chains, + std::function completion) { + ASSERT(thread_); + dispatcher_->post( + [this, listener_tag, &filter_chains, completion = std::move(completion)]() -> void { + handler_->removeFilterChains(listener_tag, filter_chains, completion); + }); +} + void WorkerImpl::start(GuardDog& guard_dog) { ASSERT(!thread_); thread_ = diff --git a/source/server/worker_impl.h b/source/server/worker_impl.h index 28797269d8b6..4161c1abcc0a 100644 --- a/source/server/worker_impl.h +++ b/source/server/worker_impl.h @@ -42,9 +42,14 @@ class WorkerImpl : public Worker, Logger::Loggable { Api::Api& api); // Server::Worker - void addListener(Network::ListenerConfig& listener, AddListenerCompletion completion) override; + void addListener(absl::optional overridden_listener, Network::ListenerConfig& listener, + AddListenerCompletion completion) override; uint64_t numConnections() const override; + void removeListener(Network::ListenerConfig& listener, std::function completion) override; + void removeFilterChains(uint64_t listener_tag, + const std::list& filter_chains, + std::function completion) override; void start(GuardDog& guard_dog) override; void initializeStats(Stats::Scope& scope) override; void stop() override; diff --git a/test/common/access_log/access_log_formatter_test.cc b/test/common/access_log/access_log_formatter_test.cc index c67268c737e8..0de62f0887da 100644 --- a/test/common/access_log/access_log_formatter_test.cc +++ b/test/common/access_log/access_log_formatter_test.cc @@ -75,6 +75,23 @@ class TestSerializedStructFilterState : public StreamInfo::FilterState::Object { ProtobufWkt::Duration duration_; }; +// Class used to test serializeAsString and serializeAsProto of FilterState +class TestSerializedStringFilterState : public StreamInfo::FilterState::Object { +public: + TestSerializedStringFilterState(std::string str) : raw_string_(str) {} + absl::optional serializeAsString() const override { + return raw_string_ + " By PLAIN"; + } + ProtobufTypes::MessagePtr serializeAsProto() const override { + auto message = std::make_unique(); + message->set_value(raw_string_ + " By TYPED"); + return message; + } + +private: + std::string raw_string_; +}; + TEST(AccessLogFormatUtilsTest, protocolToString) { EXPECT_EQ("HTTP/1.0", AccessLogFormatUtils::protocolToString(Http::Protocol::Http10)); EXPECT_EQ("HTTP/1.1", AccessLogFormatUtils::protocolToString(Http::Protocol::Http11)); @@ -1241,10 +1258,13 @@ TEST(AccessLogFormatterTest, FilterStateFormatter) { "key-serialization-error", std::make_unique(std::chrono::seconds(-281474976710656)), StreamInfo::FilterState::StateType::ReadOnly); + stream_info.filter_state_->setData( + "test_key", std::make_unique("test_value"), + StreamInfo::FilterState::StateType::ReadOnly); EXPECT_CALL(Const(stream_info), filterState()).Times(testing::AtLeast(1)); { - FilterStateFormatter formatter("key", absl::optional()); + FilterStateFormatter formatter("key", absl::optional(), false); EXPECT_EQ("\"test_value\"", formatter.format(request_headers, response_headers, response_trailers, stream_info)); @@ -1253,7 +1273,7 @@ TEST(AccessLogFormatterTest, FilterStateFormatter) { ProtoEq(ValueUtil::stringValue("test_value"))); } { - FilterStateFormatter formatter("key-struct", absl::optional()); + FilterStateFormatter formatter("key-struct", absl::optional(), false); EXPECT_EQ("{\"inner_key\":\"inner_value\"}", formatter.format(request_headers, response_headers, response_trailers, stream_info)); @@ -1269,7 +1289,7 @@ TEST(AccessLogFormatterTest, FilterStateFormatter) { // not found case { - FilterStateFormatter formatter("key-not-found", absl::optional()); + FilterStateFormatter formatter("key-not-found", absl::optional(), false); EXPECT_EQ("-", formatter.format(request_headers, response_headers, response_trailers, stream_info)); @@ -1280,7 +1300,7 @@ TEST(AccessLogFormatterTest, FilterStateFormatter) { // no serialization case { - FilterStateFormatter formatter("key-no-serialization", absl::optional()); + FilterStateFormatter formatter("key-no-serialization", absl::optional(), false); EXPECT_EQ("-", formatter.format(request_headers, response_headers, response_trailers, stream_info)); @@ -1291,7 +1311,7 @@ TEST(AccessLogFormatterTest, FilterStateFormatter) { // serialization error case { - FilterStateFormatter formatter("key-serialization-error", absl::optional()); + FilterStateFormatter formatter("key-serialization-error", absl::optional(), false); EXPECT_EQ("-", formatter.format(request_headers, response_headers, response_trailers, stream_info)); @@ -1302,7 +1322,7 @@ TEST(AccessLogFormatterTest, FilterStateFormatter) { // size limit { - FilterStateFormatter formatter("key", absl::optional(5)); + FilterStateFormatter formatter("key", absl::optional(5), false); EXPECT_EQ("\"test", formatter.format(request_headers, response_headers, response_trailers, stream_info)); @@ -1312,6 +1332,33 @@ TEST(AccessLogFormatterTest, FilterStateFormatter) { formatter.formatValue(request_headers, response_headers, response_trailers, stream_info), ProtoEq(ValueUtil::stringValue("test_value"))); } + + // serializeAsString case + { + FilterStateFormatter formatter("test_key", absl::optional(), true); + + EXPECT_EQ("test_value By PLAIN", + formatter.format(request_headers, response_headers, response_trailers, stream_info)); + } + + // size limit for serializeAsString + { + FilterStateFormatter formatter("test_key", absl::optional(10), true); + + EXPECT_EQ("test_value", + formatter.format(request_headers, response_headers, response_trailers, stream_info)); + } + + // no serialization case for serializeAsString + { + FilterStateFormatter formatter("key-no-serialization", absl::optional(), true); + + EXPECT_EQ("-", + formatter.format(request_headers, response_headers, response_trailers, stream_info)); + EXPECT_THAT( + formatter.formatValue(request_headers, response_headers, response_trailers, stream_info), + ProtoEq(ValueUtil::nullValue())); + } } TEST(AccessLogFormatterTest, StartTimeFormatter) { @@ -1569,7 +1616,7 @@ TEST(AccessLogFormatterTest, JsonFormatterTypedDynamicMetadataTest) { fields.at("test_obj").struct_value().fields().at("inner_key").string_value()); } -TEST(AccessLogFormatterTets, JsonFormatterFilterStateTest) { +TEST(AccessLogFormatterTest, JsonFormatterFilterStateTest) { Http::TestRequestHeaderMapImpl request_headers; Http::TestResponseHeaderMapImpl response_headers; Http::TestResponseTrailerMapImpl response_trailers; @@ -1595,7 +1642,7 @@ TEST(AccessLogFormatterTets, JsonFormatterFilterStateTest) { expected_json_map); } -TEST(AccessLogFormatterTets, JsonFormatterTypedFilterStateTest) { +TEST(AccessLogFormatterTest, JsonFormatterTypedFilterStateTest) { Http::TestRequestHeaderMapImpl request_headers; Http::TestResponseHeaderMapImpl response_headers; Http::TestResponseTrailerMapImpl response_trailers; @@ -1624,6 +1671,82 @@ TEST(AccessLogFormatterTets, JsonFormatterTypedFilterStateTest) { fields.at("test_obj").struct_value().fields().at("inner_key").string_value()); } +// Test new specifier (PLAIN/TYPED) of FilterState. Ensure that after adding additional specifier, +// the FilterState can call the serializeAsProto or serializeAsString methods correctly. +TEST(AccessLogFormatterTest, FilterStateSpeciferTest) { + Http::TestRequestHeaderMapImpl request_headers; + Http::TestResponseHeaderMapImpl response_headers; + Http::TestResponseTrailerMapImpl response_trailers; + StreamInfo::MockStreamInfo stream_info; + stream_info.filter_state_->setData( + "test_key", std::make_unique("test_value"), + StreamInfo::FilterState::StateType::ReadOnly); + EXPECT_CALL(Const(stream_info), filterState()).Times(testing::AtLeast(1)); + + std::unordered_map expected_json_map = { + {"test_key_plain", "test_value By PLAIN"}, + {"test_key_typed", "\"test_value By TYPED\""}, + }; + + std::unordered_map key_mapping = { + {"test_key_plain", "%FILTER_STATE(test_key:PLAIN)%"}, + {"test_key_typed", "%FILTER_STATE(test_key:TYPED)%"}}; + + JsonFormatterImpl formatter(key_mapping, false); + + verifyJsonOutput( + formatter.format(request_headers, response_headers, response_trailers, stream_info), + expected_json_map); +} + +// Test new specifier (PLAIN/TYPED) of FilterState and convert the output log string to proto +// and then verify the result. +TEST(AccessLogFormatterTest, TypedFilterStateSpeciferTest) { + Http::TestRequestHeaderMapImpl request_headers; + Http::TestResponseHeaderMapImpl response_headers; + Http::TestResponseTrailerMapImpl response_trailers; + StreamInfo::MockStreamInfo stream_info; + stream_info.filter_state_->setData( + "test_key", std::make_unique("test_value"), + StreamInfo::FilterState::StateType::ReadOnly); + EXPECT_CALL(Const(stream_info), filterState()).Times(testing::AtLeast(1)); + + std::unordered_map key_mapping = { + {"test_key_plain", "%FILTER_STATE(test_key:PLAIN)%"}, + {"test_key_typed", "%FILTER_STATE(test_key:TYPED)%"}}; + + JsonFormatterImpl formatter(key_mapping, true); + + std::string json = + formatter.format(request_headers, response_headers, response_trailers, stream_info); + + ProtobufWkt::Struct output; + MessageUtil::loadFromJson(json, output); + + const auto& fields = output.fields(); + EXPECT_EQ("test_value By PLAIN", fields.at("test_key_plain").string_value()); + EXPECT_EQ("test_value By TYPED", fields.at("test_key_typed").string_value()); +} + +// Error specifier will cause an exception to be thrown. +TEST(AccessLogFormatterTest, FilterStateErrorSpeciferTest) { + Http::TestRequestHeaderMapImpl request_headers; + Http::TestResponseHeaderMapImpl response_headers; + Http::TestResponseTrailerMapImpl response_trailers; + StreamInfo::MockStreamInfo stream_info; + stream_info.filter_state_->setData( + "test_key", std::make_unique("test_value"), + StreamInfo::FilterState::StateType::ReadOnly); + + // 'ABCDE' is error specifier. + std::unordered_map key_mapping = { + {"test_key_plain", "%FILTER_STATE(test_key:ABCDE)%"}, + {"test_key_typed", "%FILTER_STATE(test_key:TYPED)%"}}; + + EXPECT_THROW_WITH_MESSAGE(JsonFormatterImpl formatter(key_mapping, false), EnvoyException, + "Invalid filter state serialize type, only support PLAIN/TYPED."); +} + TEST(AccessLogFormatterTest, JsonFormatterStartTimeTest) { StreamInfo::MockStreamInfo stream_info; Http::TestRequestHeaderMapImpl request_header; diff --git a/test/common/compressor/zlib_compressor_impl_test.cc b/test/common/compressor/zlib_compressor_impl_test.cc index 8c5b181b7d0e..3e2db26f4d43 100644 --- a/test/common/compressor/zlib_compressor_impl_test.cc +++ b/test/common/compressor/zlib_compressor_impl_test.cc @@ -61,9 +61,9 @@ class ZlibCompressorImplTest : public testing::Test { void drainBuffer(Buffer::OwnedImpl& buffer) { buffer.drain(buffer.length()); } - static const int64_t gzip_window_bits{31}; - static const int64_t memory_level{8}; - static const uint64_t default_input_size{796}; + static constexpr int64_t gzip_window_bits{31}; + static constexpr int64_t memory_level{8}; + static constexpr uint64_t default_input_size{796}; }; class ZlibCompressorImplTester : public ZlibCompressorImpl { diff --git a/test/common/decompressor/zlib_decompressor_impl_test.cc b/test/common/decompressor/zlib_decompressor_impl_test.cc index d2468aa0d966..93ff4e07729d 100644 --- a/test/common/decompressor/zlib_decompressor_impl_test.cc +++ b/test/common/decompressor/zlib_decompressor_impl_test.cc @@ -53,9 +53,9 @@ class ZlibDecompressorImplTest : public testing::Test { ASSERT_EQ(0, decompressor.decompression_error_); } - static const int64_t gzip_window_bits{31}; - static const int64_t memory_level{8}; - static const uint64_t default_input_size{796}; + static constexpr int64_t gzip_window_bits{31}; + static constexpr int64_t memory_level{8}; + static constexpr uint64_t default_input_size{796}; }; class ZlibDecompressorImplFailureTest : public ZlibDecompressorImplTest { diff --git a/test/common/event/BUILD b/test/common/event/BUILD index 76c55d3c2706..f2af3acf3dc5 100644 --- a/test/common/event/BUILD +++ b/test/common/event/BUILD @@ -13,6 +13,7 @@ envoy_cc_test( srcs = ["dispatcher_impl_test.cc"], deps = [ "//source/common/api:api_lib", + "//source/common/event:deferred_task", "//source/common/event:dispatcher_includes", "//source/common/event:dispatcher_lib", "//source/common/stats:isolated_store_lib", diff --git a/test/common/event/dispatcher_impl_test.cc b/test/common/event/dispatcher_impl_test.cc index 2f3efdfce603..f6db84f844bf 100644 --- a/test/common/event/dispatcher_impl_test.cc +++ b/test/common/event/dispatcher_impl_test.cc @@ -4,6 +4,7 @@ #include "common/api/api_impl.h" #include "common/common/lock_guard.h" +#include "common/event/deferred_task.h" #include "common/event/dispatcher_impl.h" #include "common/event/timer_impl.h" #include "common/stats/isolated_store_impl.h" @@ -64,6 +65,27 @@ TEST(DeferredDeleteTest, DeferredDelete) { dispatcher->clearDeferredDeleteList(); } +TEST(DeferredTaskTest, DeferredTask) { + InSequence s; + Api::ApiPtr api = Api::createApiForTest(); + DispatcherPtr dispatcher(api->allocateDispatcher("test_thread")); + ReadyWatcher watcher1; + + DeferredTaskUtil::deferredRun(*dispatcher, [&watcher1]() -> void { watcher1.ready(); }); + // The first one will get deleted inline. + EXPECT_CALL(watcher1, ready()); + dispatcher->clearDeferredDeleteList(); + + // Deferred task is scheduled FIFO. + ReadyWatcher watcher2; + ReadyWatcher watcher3; + DeferredTaskUtil::deferredRun(*dispatcher, [&watcher2]() -> void { watcher2.ready(); }); + DeferredTaskUtil::deferredRun(*dispatcher, [&watcher3]() -> void { watcher3.ready(); }); + EXPECT_CALL(watcher2, ready()); + EXPECT_CALL(watcher3, ready()); + dispatcher->clearDeferredDeleteList(); +} + class DispatcherImplTest : public testing::Test { protected: DispatcherImplTest() diff --git a/test/common/http/codec_impl_fuzz_test.cc b/test/common/http/codec_impl_fuzz_test.cc index 831e038d9b81..33856e11a6ac 100644 --- a/test/common/http/codec_impl_fuzz_test.cc +++ b/test/common/http/codec_impl_fuzz_test.cc @@ -413,7 +413,8 @@ void codecFuzz(const test::common::http::CodecImplFuzzTestCase& input, HttpVersi if (http2) { client = std::make_unique( client_connection, client_callbacks, stats_store, client_http2_options, - max_request_headers_kb, max_response_headers_count); + max_request_headers_kb, max_response_headers_count, + Http2::ProdNghttp2SessionFactory::get()); } else { client = std::make_unique(client_connection, stats_store, client_callbacks, client_http1settings, diff --git a/test/common/http/conn_manager_impl_test.cc b/test/common/http/conn_manager_impl_test.cc index a820323b4286..99fa69b05acb 100644 --- a/test/common/http/conn_manager_impl_test.cc +++ b/test/common/http/conn_manager_impl_test.cc @@ -5388,11 +5388,11 @@ TEST_F(HttpConnectionManagerImplTest, ConnectionFilterState) { decoder_filters_[0]->callbacks_->streamInfo().filterState()->setData( "per_downstream_request", std::make_unique(2), StreamInfo::FilterState::StateType::ReadOnly, - StreamInfo::FilterState::LifeSpan::DownstreamRequest); + StreamInfo::FilterState::LifeSpan::Request); decoder_filters_[0]->callbacks_->streamInfo().filterState()->setData( "per_downstream_connection", std::make_unique(3), StreamInfo::FilterState::StateType::ReadOnly, - StreamInfo::FilterState::LifeSpan::DownstreamConnection); + StreamInfo::FilterState::LifeSpan::Connection); return FilterHeadersStatus::StopIteration; })); EXPECT_CALL(*decoder_filters_[1], decodeHeaders(_, true)) diff --git a/test/common/http/conn_manager_utility_test.cc b/test/common/http/conn_manager_utility_test.cc index 276d8a127256..71e7b5a40933 100644 --- a/test/common/http/conn_manager_utility_test.cc +++ b/test/common/http/conn_manager_utility_test.cc @@ -764,17 +764,36 @@ TEST_F(ConnectionManagerUtilityTest, MutateResponseHeaders) { // Make sure we don't remove connection headers on all Upgrade responses. TEST_F(ConnectionManagerUtilityTest, DoNotRemoveConnectionUpgradeForWebSocketResponses) { + TestRequestHeaderMapImpl request_headers{{"connection", "UpGrAdE"}, {"upgrade", "foo"}}; + TestResponseHeaderMapImpl response_headers{{":status", "101"}, + {"connection", "upgrade"}, + {"transfer-encoding", "foo"}, + {"upgrade", "bar"}}; + EXPECT_TRUE(Utility::isUpgrade(request_headers)); + EXPECT_TRUE(Utility::isUpgrade(response_headers)); + ConnectionManagerUtility::mutateResponseHeaders(response_headers, &request_headers, + config_.requestIDExtension(), ""); + + EXPECT_EQ(3UL, response_headers.size()) << response_headers; + EXPECT_EQ("upgrade", response_headers.get_("connection")); + EXPECT_EQ("bar", response_headers.get_("upgrade")); + EXPECT_EQ("101", response_headers.get_(":status")); +} + +// Make sure we don't add a content-length header on Upgrade responses. +TEST_F(ConnectionManagerUtilityTest, DoNotAddConnectionLengthForWebSocket101Responses) { TestRequestHeaderMapImpl request_headers{{"connection", "UpGrAdE"}, {"upgrade", "foo"}}; TestResponseHeaderMapImpl response_headers{ - {"connection", "upgrade"}, {"transfer-encoding", "foo"}, {"upgrade", "bar"}}; + {":status", "101"}, {"connection", "upgrade"}, {"upgrade", "bar"}}; EXPECT_TRUE(Utility::isUpgrade(request_headers)); EXPECT_TRUE(Utility::isUpgrade(response_headers)); ConnectionManagerUtility::mutateResponseHeaders(response_headers, &request_headers, config_.requestIDExtension(), ""); - EXPECT_EQ(2UL, response_headers.size()) << response_headers; + EXPECT_EQ(3UL, response_headers.size()) << response_headers; EXPECT_EQ("upgrade", response_headers.get_("connection")); EXPECT_EQ("bar", response_headers.get_("upgrade")); + EXPECT_EQ("101", response_headers.get_(":status")); } TEST_F(ConnectionManagerUtilityTest, ClearUpgradeHeadersForNonUpgradeRequests) { diff --git a/test/common/http/http2/BUILD b/test/common/http/http2/BUILD index 29e0f2c73ea8..870d27aced8c 100644 --- a/test/common/http/http2/BUILD +++ b/test/common/http/http2/BUILD @@ -32,6 +32,7 @@ envoy_cc_test( "//test/mocks/protobuf:protobuf_mocks", "//test/mocks/thread_local:thread_local_mocks", "//test/mocks/upstream:upstream_mocks", + "//test/test_common:registry_lib", "//test/test_common:test_runtime_lib", "//test/test_common:utility_lib", ], diff --git a/test/common/http/http2/codec_impl_test.cc b/test/common/http/http2/codec_impl_test.cc index 69c3e47db3cc..e44c29198a2e 100644 --- a/test/common/http/http2/codec_impl_test.cc +++ b/test/common/http/http2/codec_impl_test.cc @@ -18,6 +18,7 @@ #include "test/mocks/protobuf/mocks.h" #include "test/mocks/thread_local/mocks.h" #include "test/test_common/printers.h" +#include "test/test_common/registry.h" #include "test/test_common/test_runtime.h" #include "test/test_common/utility.h" @@ -67,16 +68,17 @@ class Http2CodecImplTestFixture { InitialConnectionWindowSize }; + Http2CodecImplTestFixture() = default; Http2CodecImplTestFixture(Http2SettingsTuple client_settings, Http2SettingsTuple server_settings) : client_settings_(client_settings), server_settings_(server_settings) {} virtual ~Http2CodecImplTestFixture() = default; virtual void initialize() { - Http2OptionsFromTuple(client_http2_options_, client_settings_); - Http2OptionsFromTuple(server_http2_options_, server_settings_); + http2OptionsFromTuple(client_http2_options_, client_settings_); + http2OptionsFromTuple(server_http2_options_, server_settings_); client_ = std::make_unique( client_connection_, client_callbacks_, stats_store_, client_http2_options_, - max_request_headers_kb_, max_response_headers_count_); + max_request_headers_kb_, max_response_headers_count_, ProdNghttp2SessionFactory::get()); server_ = std::make_unique( server_connection_, server_callbacks_, stats_store_, server_http2_options_, max_request_headers_kb_, max_request_headers_count_, headers_with_underscores_action_); @@ -106,16 +108,20 @@ class Http2CodecImplTestFixture { })); } - void Http2OptionsFromTuple(envoy::config::core::v3::Http2ProtocolOptions& options, - const Http2SettingsTuple& tp) { + void http2OptionsFromTuple(envoy::config::core::v3::Http2ProtocolOptions& options, + const absl::optional& tp) { options.mutable_hpack_table_size()->set_value( - ::testing::get(tp)); + (tp.has_value()) ? ::testing::get(*tp) + : CommonUtility::OptionsLimits::DEFAULT_HPACK_TABLE_SIZE); options.mutable_max_concurrent_streams()->set_value( - ::testing::get(tp)); + (tp.has_value()) ? ::testing::get(*tp) + : CommonUtility::OptionsLimits::DEFAULT_MAX_CONCURRENT_STREAMS); options.mutable_initial_stream_window_size()->set_value( - ::testing::get(tp)); + (tp.has_value()) ? ::testing::get(*tp) + : CommonUtility::OptionsLimits::DEFAULT_INITIAL_STREAM_WINDOW_SIZE); options.mutable_initial_connection_window_size()->set_value( - ::testing::get(tp)); + (tp.has_value()) ? ::testing::get(*tp) + : CommonUtility::OptionsLimits::DEFAULT_INITIAL_CONNECTION_WINDOW_SIZE); options.set_allow_metadata(allow_metadata_); options.set_stream_error_on_invalid_http_messaging(stream_error_on_invalid_http_messaging_); options.mutable_max_outbound_frames()->set_value(max_outbound_frames_); @@ -155,8 +161,8 @@ class Http2CodecImplTestFixture { EXPECT_EQ(details, response_encoder_->getStream().responseDetails()); } - const Http2SettingsTuple client_settings_; - const Http2SettingsTuple server_settings_; + absl::optional client_settings_; + absl::optional server_settings_; bool allow_metadata_ = false; bool stream_error_on_invalid_http_messaging_ = false; Stats::TestUtil::TestStore stats_store_; @@ -991,11 +997,11 @@ class Http2CodecImplStreamLimitTest : public Http2CodecImplTest {}; // TODO(PiotrSikora): add tests that exercise both scenarios: before and after receiving // the HTTP/2 SETTINGS frame. TEST_P(Http2CodecImplStreamLimitTest, MaxClientStreams) { - Http2OptionsFromTuple(client_http2_options_, ::testing::get<0>(GetParam())); - Http2OptionsFromTuple(server_http2_options_, ::testing::get<1>(GetParam())); + http2OptionsFromTuple(client_http2_options_, ::testing::get<0>(GetParam())); + http2OptionsFromTuple(server_http2_options_, ::testing::get<1>(GetParam())); client_ = std::make_unique( client_connection_, client_callbacks_, stats_store_, client_http2_options_, - max_request_headers_kb_, max_response_headers_count_); + max_request_headers_kb_, max_response_headers_count_, ProdNghttp2SessionFactory::get()); server_ = std::make_unique( server_connection_, server_callbacks_, stats_store_, server_http2_options_, max_request_headers_kb_, max_request_headers_count_, headers_with_underscores_action_); @@ -1160,7 +1166,8 @@ class Http2CustomSettingsTestBase : public Http2CodecImplTestFixture { // Returns the settings tuple which specifies a subset of the settings parameters to be sent to // the endpoint being validated. const Http2SettingsTuple& getSettingsTuple() { - return validate_client_ ? server_settings_ : client_settings_; + ASSERT(client_settings_.has_value() && server_settings_.has_value()); + return validate_client_ ? *server_settings_ : *client_settings_; } protected: @@ -1779,6 +1786,131 @@ TEST_P(Http2CodecImplTest, EmptyDataFloodOverride) { EXPECT_NO_THROW(server_wrapper_.dispatch(data, *server_)); } +class TestNghttp2SessionFactory; + +// Test client for H/2 METADATA frame edge cases. +class MetadataTestClientConnectionImpl : public TestClientConnectionImpl { +public: + MetadataTestClientConnectionImpl( + Network::Connection& connection, Http::ConnectionCallbacks& callbacks, Stats::Scope& scope, + const envoy::config::core::v3::Http2ProtocolOptions& http2_options, + uint32_t max_request_headers_kb, uint32_t max_request_headers_count, + Nghttp2SessionFactory& http2_session_factory) + : TestClientConnectionImpl(connection, callbacks, scope, http2_options, + max_request_headers_kb, max_request_headers_count, + http2_session_factory) {} + + // Overrides TestClientConnectionImpl::submitMetadata(). + bool submitMetadata(const MetadataMapVector& metadata_map_vector, int32_t stream_id) override { + // Creates metadata payload. + encoder_.createPayload(metadata_map_vector); + for (uint8_t flags : encoder_.payloadFrameFlagBytes()) { + int result = nghttp2_submit_extension(session(), ::Envoy::Http::METADATA_FRAME_TYPE, flags, + stream_id, nullptr); + if (result != 0) { + return false; + } + } + // Triggers nghttp2 to populate the payloads of the METADATA frames. + int result = nghttp2_session_send(session()); + return result == 0; + } + +protected: + friend class TestNghttp2SessionFactory; + + MetadataEncoder encoder_; +}; + +class TestNghttp2SessionFactory : public Nghttp2SessionFactory { +public: + ~TestNghttp2SessionFactory() override { + nghttp2_session_callbacks_del(callbacks_); + nghttp2_option_del(options_); + } + + nghttp2_session* create(const nghttp2_session_callbacks*, ConnectionImpl* connection, + const nghttp2_option*) override { + // Only need to provide callbacks required to send METADATA frames. + nghttp2_session_callbacks_new(&callbacks_); + nghttp2_session_callbacks_set_pack_extension_callback( + callbacks_, + [](nghttp2_session*, uint8_t* data, size_t length, const nghttp2_frame*, + void* user_data) -> ssize_t { + // Double cast required due to multiple inheritance. + return static_cast( + static_cast(user_data)) + ->encoder_.packNextFramePayload(data, length); + }); + nghttp2_session_callbacks_set_send_callback( + callbacks_, + [](nghttp2_session*, const uint8_t* data, size_t length, int, void* user_data) -> ssize_t { + // Cast down to MetadataTestClientConnectionImpl to leverage friendship. + return static_cast( + static_cast(user_data)) + ->onSend(data, length); + }); + nghttp2_option_new(&options_); + nghttp2_option_set_user_recv_extension_type(options_, METADATA_FRAME_TYPE); + nghttp2_session* session; + nghttp2_session_client_new2(&session, callbacks_, connection, options_); + return session; + } + + void init(nghttp2_session*, ConnectionImpl*, + const envoy::config::core::v3::Http2ProtocolOptions&) override {} + +private: + nghttp2_session_callbacks* callbacks_; + nghttp2_option* options_; +}; + +class Http2CodecMetadataTest : public Http2CodecImplTestFixture, public ::testing::Test { +public: + Http2CodecMetadataTest() = default; + +protected: + void initialize() override { + allow_metadata_ = true; + http2OptionsFromTuple(client_http2_options_, client_settings_); + http2OptionsFromTuple(server_http2_options_, server_settings_); + client_ = std::make_unique( + client_connection_, client_callbacks_, stats_store_, client_http2_options_, + max_request_headers_kb_, max_response_headers_count_, http2_session_factory_); + server_ = std::make_unique( + server_connection_, server_callbacks_, stats_store_, server_http2_options_, + max_request_headers_kb_, max_request_headers_count_, headers_with_underscores_action_); + ON_CALL(client_connection_, write(_, _)) + .WillByDefault(Invoke([&](Buffer::Instance& data, bool) -> void { + server_wrapper_.dispatch(data, *server_); + })); + ON_CALL(server_connection_, write(_, _)) + .WillByDefault(Invoke([&](Buffer::Instance& data, bool) -> void { + client_wrapper_.dispatch(data, *client_); + })); + } + +private: + TestNghttp2SessionFactory http2_session_factory_; +}; + +// Validates noop handling of METADATA frames without a known stream ID. +// This is required per RFC 7540, section 5.1.1, which states that stream ID = 0 can be used for +// "connection control" messages, and per the H2 METADATA spec (source/docs/h2_metadata.md), which +// states that these frames can be received prior to the headers. +TEST_F(Http2CodecMetadataTest, UnknownStreamId) { + initialize(); + MetadataMap metadata_map = {{"key", "value"}}; + MetadataMapVector metadata_vector; + metadata_vector.emplace_back(std::make_unique(metadata_map)); + // SETTINGS are required as part of the preface. + ASSERT_EQ(nghttp2_submit_settings(client_->session(), NGHTTP2_FLAG_NONE, nullptr, 0), 0); + // Validate both the ID = 0 special case and a non-zero ID not already bound to a stream (any ID > + // 0 for this test). + EXPECT_TRUE(client_->submitMetadata(metadata_vector, 0)); + EXPECT_TRUE(client_->submitMetadata(metadata_vector, 1000)); +} + } // namespace Http2 } // namespace Http } // namespace Envoy diff --git a/test/common/http/http2/codec_impl_test_util.h b/test/common/http/http2/codec_impl_test_util.h index c6d859056d8a..462e94297bd2 100644 --- a/test/common/http/http2/codec_impl_test_util.h +++ b/test/common/http/http2/codec_impl_test_util.h @@ -59,10 +59,21 @@ class TestClientConnectionImpl : public ClientConnectionImpl, public TestCodecSe TestClientConnectionImpl(Network::Connection& connection, Http::ConnectionCallbacks& callbacks, Stats::Scope& scope, const envoy::config::core::v3::Http2ProtocolOptions& http2_options, - uint32_t max_request_headers_kb, uint32_t max_request_headers_count) + uint32_t max_request_headers_kb, uint32_t max_request_headers_count, + Nghttp2SessionFactory& http2_session_factory) : ClientConnectionImpl(connection, callbacks, scope, http2_options, max_request_headers_kb, - max_request_headers_count) {} + max_request_headers_count, http2_session_factory) {} + nghttp2_session* session() { return session_; } + + // Submits an H/2 METADATA frame to the peer. + // Returns true on success, false otherwise. + virtual bool submitMetadata(const MetadataMapVector& mm_vector, int32_t stream_id) { + UNREFERENCED_PARAMETER(mm_vector); + UNREFERENCED_PARAMETER(stream_id); + return false; + } + using ClientConnectionImpl::getStream; using ConnectionImpl::sendPendingFrames; diff --git a/test/common/http/http2/frame_replay_test.cc b/test/common/http/http2/frame_replay_test.cc index c55a00f21d63..c6afd9a8be5e 100644 --- a/test/common/http/http2/frame_replay_test.cc +++ b/test/common/http/http2/frame_replay_test.cc @@ -91,7 +91,8 @@ TEST_F(ResponseFrameCommentTest, SimpleExampleHuffman) { ClientCodecFrameInjector codec; TestClientConnectionImpl connection( codec.client_connection_, codec.client_callbacks_, codec.stats_store_, codec.options_, - Http::DEFAULT_MAX_REQUEST_HEADERS_KB, Http::DEFAULT_MAX_HEADERS_COUNT); + Http::DEFAULT_MAX_REQUEST_HEADERS_KB, Http::DEFAULT_MAX_HEADERS_COUNT, + ProdNghttp2SessionFactory::get()); setupStream(codec, connection); codec.write(WellKnownFrames::defaultSettingsFrame(), connection); @@ -170,7 +171,8 @@ TEST_F(ResponseFrameCommentTest, SimpleExamplePlain) { ClientCodecFrameInjector codec; TestClientConnectionImpl connection( codec.client_connection_, codec.client_callbacks_, codec.stats_store_, codec.options_, - Http::DEFAULT_MAX_REQUEST_HEADERS_KB, Http::DEFAULT_MAX_HEADERS_COUNT); + Http::DEFAULT_MAX_REQUEST_HEADERS_KB, Http::DEFAULT_MAX_HEADERS_COUNT, + ProdNghttp2SessionFactory::get()); setupStream(codec, connection); codec.write(WellKnownFrames::defaultSettingsFrame(), connection); @@ -233,7 +235,8 @@ TEST_F(ResponseFrameCommentTest, SingleByteNulCrLfInHeaderFrame) { ClientCodecFrameInjector codec; TestClientConnectionImpl connection( codec.client_connection_, codec.client_callbacks_, codec.stats_store_, codec.options_, - Http::DEFAULT_MAX_REQUEST_HEADERS_KB, Http::DEFAULT_MAX_HEADERS_COUNT); + Http::DEFAULT_MAX_REQUEST_HEADERS_KB, Http::DEFAULT_MAX_HEADERS_COUNT, + ProdNghttp2SessionFactory::get()); setupStream(codec, connection); codec.write(WellKnownFrames::defaultSettingsFrame(), connection); @@ -307,7 +310,8 @@ TEST_F(ResponseFrameCommentTest, SingleByteNulCrLfInHeaderField) { ClientCodecFrameInjector codec; TestClientConnectionImpl connection( codec.client_connection_, codec.client_callbacks_, codec.stats_store_, codec.options_, - Http::DEFAULT_MAX_REQUEST_HEADERS_KB, Http::DEFAULT_MAX_HEADERS_COUNT); + Http::DEFAULT_MAX_REQUEST_HEADERS_KB, Http::DEFAULT_MAX_HEADERS_COUNT, + ProdNghttp2SessionFactory::get()); setupStream(codec, connection); codec.write(WellKnownFrames::defaultSettingsFrame(), connection); diff --git a/test/common/http/http2/response_header_fuzz_test.cc b/test/common/http/http2/response_header_fuzz_test.cc index b670b49f8795..756af3860c3f 100644 --- a/test/common/http/http2/response_header_fuzz_test.cc +++ b/test/common/http/http2/response_header_fuzz_test.cc @@ -17,7 +17,8 @@ void Replay(const Frame& frame, ClientCodecFrameInjector& codec) { // Create the client connection containing the nghttp2 session. TestClientConnectionImpl connection( codec.client_connection_, codec.client_callbacks_, codec.stats_store_, codec.options_, - Http::DEFAULT_MAX_REQUEST_HEADERS_KB, Http::DEFAULT_MAX_HEADERS_COUNT); + Http::DEFAULT_MAX_REQUEST_HEADERS_KB, Http::DEFAULT_MAX_HEADERS_COUNT, + ProdNghttp2SessionFactory::get()); // Create a new stream. codec.request_encoder_ = &connection.newStream(codec.response_decoder_); codec.request_encoder_->getStream().addCallbacks(codec.client_stream_callbacks_); diff --git a/test/common/http/utility_corpus/valid b/test/common/http/utility_corpus/valid new file mode 100644 index 000000000000..f47f99e15a99 --- /dev/null +++ b/test/common/http/utility_corpus/valid @@ -0,0 +1,2 @@ +find_query_string: "\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\7\177\177\17 +U²@/177\177N¿77\177" \ No newline at end of file diff --git a/test/common/http/utility_fuzz.proto b/test/common/http/utility_fuzz.proto index 51e63a2ac39a..940be6f4e0f3 100644 --- a/test/common/http/utility_fuzz.proto +++ b/test/common/http/utility_fuzz.proto @@ -2,6 +2,8 @@ syntax = "proto3"; package test.common.http; +import "validate/validate.proto"; + // Structured input for utility_fuzz_test. message CookiesKey { @@ -38,7 +40,8 @@ message UtilityTestCase { string percent_encoding_string = 6; string percent_decoding_string = 7; Parameter parse_parameters = 8; - string find_query_string = 9; + string find_query_string = 9 + [(validate.rules).string = {well_known_regex: HTTP_HEADER_VALUE, strict: false}]; CookieValue make_set_cookie_value = 10; } } diff --git a/test/common/http/utility_fuzz_test.cc b/test/common/http/utility_fuzz_test.cc index 381b42dfacba..54d5ce8bfa1b 100644 --- a/test/common/http/utility_fuzz_test.cc +++ b/test/common/http/utility_fuzz_test.cc @@ -1,6 +1,6 @@ #include "common/http/utility.h" -#include "test/common/http/utility_fuzz.pb.h" +#include "test/common/http/utility_fuzz.pb.validate.h" #include "test/fuzz/fuzz_runner.h" #include "test/fuzz/utility.h" #include "test/test_common/utility.h" diff --git a/test/common/router/route_corpus/valid_headers_to_remove b/test/common/router/route_corpus/valid_headers_to_remove new file mode 100644 index 000000000000..f8b4415aae88 --- /dev/null +++ b/test/common/router/route_corpus/valid_headers_to_remove @@ -0,0 +1,64 @@ +config { + virtual_hosts { + name: "e" + domains: "*" + require_tls: ALL + cors { + allow_methods: "?" + allow_headers: "\000\000\000\000\000\000\000\000" + max_age: "e" + allow_credentials { + value: true + } + allow_origin_string_match { + safe_regex { + google_re2 { + } + regex: "\000\000\000-\000\000" + } + } + allow_origin_string_match { + safe_regex { + google_re2 { + max_program_size { + value: 543382106 + } + } + regex: "\000\000\000\000\000\000\000\000" + } + } + allow_origin_string_match { + safe_regex { + google_re2 { + max_program_size { + value: 543382106 + } + } + regex: "\177\177\177\177\177\177\177\177\177\177\363\273\271\260\362\215\217\270\362\275\246\217\361\250\252\235\361\233\273\226\363\232\203\254\363\223\243\222\364\200\265\232\362\216\240\256\363\253\252\201\361\205\214\273\363\211\262\206\364\217\217\236\360\250\216\235\360\237\265\217\361\234\223\251\361\271\210\201\361\241\200\254\361\247\235\276\364\200\247\204\361\215\222\221\364\210\204\246\360\262\231\222\362\230\220\274\364\205\217\245\363\237\271\236\364\217\245\255\362\251\200\224\362\206\221\261\361\244\251\215\361\266\223\223\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177|35142047902481845977\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177?\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177?\177\177\177\177\177" + } + } + allow_origin_string_match { + safe_regex { + google_re2 { + max_program_size { + value: 543382106 + } + } + regex: "^" + } + } + allow_origin_string_match { + safe_regex { + google_re2 { + max_program_size { + value: 543382106 + } + } + regex: "\000\000\000\000\000\000\000\000" + } + } + } + request_headers_to_remove: "\361`>ùHöxj/^Ò +ž|èL1\251\251\361\251\251\251\361\251\251\251\361\251\251\251\361\251\251\251\361\251\251\251\361\251\251\251\361\251\251\251\361\251\251\251)" + } +} \ No newline at end of file diff --git a/test/common/router/router_test.cc b/test/common/router/router_test.cc index 36e5eab6ddc7..37da958c2799 100644 --- a/test/common/router/router_test.cc +++ b/test/common/router/router_test.cc @@ -1,5 +1,6 @@ #include #include +#include #include #include "envoy/config/core/v3/base.pb.h" @@ -303,8 +304,7 @@ class RouterTestBase : public testing::Test { callbacks_.streamInfo().filterState()->setData( "num_internal_redirects", std::make_shared(num_previous_redirects), - StreamInfo::FilterState::StateType::Mutable, - StreamInfo::FilterState::LifeSpan::DownstreamRequest); + StreamInfo::FilterState::StateType::Mutable, StreamInfo::FilterState::LifeSpan::Request); } void setIncludeAttemptCountInRequest(bool include) { @@ -2752,7 +2752,9 @@ TEST_F(RouterTest, BadHeadersDroppedIfPreviousRetryScheduled) { response_decoder2->decodeHeaders(std::move(response_headers2), true); } -TEST_F(RouterTest, RetryRequestNotComplete) { +// Test retrying a request, when the first attempt fails before the client +// has sent any of the body. +TEST_F(RouterTest, RetryRequestBeforeBody) { NiceMock encoder1; Http::ResponseDecoder* response_decoder = nullptr; EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) @@ -2763,23 +2765,359 @@ TEST_F(RouterTest, RetryRequestNotComplete) { callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); - EXPECT_CALL(callbacks_.stream_info_, - setResponseFlag(StreamInfo::ResponseFlag::UpstreamRemoteReset)); - EXPECT_CALL(callbacks_.stream_info_, onUpstreamHostSelected(_)) - .WillOnce(Invoke([&](const Upstream::HostDescriptionConstSharedPtr host) -> void { - EXPECT_EQ(host_address_, host->address()); + expectResponseTimerCreate(); + + Http::TestRequestHeaderMapImpl headers{ + {"x-envoy-retry-on", "5xx"}, {"x-envoy-internal", "true"}, {"myheader", "present"}}; + HttpTestUtility::addDefaultHeaders(headers); + router_.decodeHeaders(headers, false); + + router_.retry_state_->expectResetRetry(); + encoder1.stream_.resetStream(Http::StreamResetReason::RemoteReset); + + NiceMock encoder2; + EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) + .WillOnce(Invoke( + [&](Http::ResponseDecoder& decoder, + Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { + response_decoder = &decoder; + callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_, upstream_stream_info_); + return nullptr; + })); + EXPECT_CALL(encoder2, encodeHeaders(HeaderHasValueRef("myheader", "present"), false)); + router_.retry_state_->callback_(); + EXPECT_EQ(2U, + callbacks_.route_->route_entry_.virtual_cluster_.stats().upstream_rq_total_.value()); + EXPECT_TRUE(verifyHostUpstreamStats(0, 1)); + + // Complete request. Ensure original headers are present. + const std::string body("body"); + EXPECT_CALL(encoder2, encodeData(BufferStringEqual(body), true)); + Buffer::OwnedImpl buf(body); + router_.decodeData(buf, true); + + // Send successful response, verify success. + Http::ResponseHeaderMapPtr response_headers( + new Http::TestResponseHeaderMapImpl({{":status", "200"}})); + EXPECT_CALL(callbacks_, encodeHeaders_(_, _)) + .WillOnce(Invoke([&](Http::ResponseHeaderMap& headers, bool) -> void { + EXPECT_EQ(headers.Status()->value(), "200"); })); + response_decoder->decodeHeaders(std::move(response_headers), true); + EXPECT_TRUE(verifyHostUpstreamStats(1, 1)); +} - Http::TestRequestHeaderMapImpl headers{{"x-envoy-retry-on", "5xx"}, {"x-envoy-internal", "true"}}; +// Test retrying a request, when the first attempt fails while the client +// is sending the body. +TEST_F(RouterTest, RetryRequestDuringBody) { + Buffer::OwnedImpl decoding_buffer; + EXPECT_CALL(callbacks_, decodingBuffer()).WillRepeatedly(Return(&decoding_buffer)); + EXPECT_CALL(callbacks_, addDecodedData(_, true)) + .WillRepeatedly(Invoke([&](Buffer::Instance& data, bool) { decoding_buffer.move(data); })); + + NiceMock encoder1; + Http::ResponseDecoder* response_decoder = nullptr; + EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) + .WillOnce(Invoke( + [&](Http::ResponseDecoder& decoder, + Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { + response_decoder = &decoder; + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); + return nullptr; + })); + expectResponseTimerCreate(); + + Http::TestRequestHeaderMapImpl headers{ + {"x-envoy-retry-on", "5xx"}, {"x-envoy-internal", "true"}, {"myheader", "present"}}; HttpTestUtility::addDefaultHeaders(headers); router_.decodeHeaders(headers, false); + const std::string body1("body1"); + Buffer::OwnedImpl buf1(body1); + EXPECT_CALL(*router_.retry_state_, enabled()).WillOnce(Return(true)); + router_.decodeData(buf1, false); + + router_.retry_state_->expectResetRetry(); + encoder1.stream_.resetStream(Http::StreamResetReason::RemoteReset); + + NiceMock encoder2; + EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) + .WillOnce(Invoke( + [&](Http::ResponseDecoder& decoder, + Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { + response_decoder = &decoder; + callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_, upstream_stream_info_); + return nullptr; + })); + + EXPECT_CALL(encoder2, encodeHeaders(HeaderHasValueRef("myheader", "present"), false)); + EXPECT_CALL(encoder2, encodeData(BufferStringEqual(body1), false)); + router_.retry_state_->callback_(); + EXPECT_EQ(2U, + callbacks_.route_->route_entry_.virtual_cluster_.stats().upstream_rq_total_.value()); + EXPECT_TRUE(verifyHostUpstreamStats(0, 1)); + + // Complete request. Ensure original headers are present. + const std::string body2("body2"); + EXPECT_CALL(encoder2, encodeData(BufferStringEqual(body2), true)); + Buffer::OwnedImpl buf2(body2); + EXPECT_CALL(*router_.retry_state_, enabled()).WillOnce(Return(true)); + router_.decodeData(buf2, true); + + // Send successful response, verify success. + Http::ResponseHeaderMapPtr response_headers( + new Http::TestResponseHeaderMapImpl({{":status", "200"}})); + EXPECT_CALL(callbacks_, encodeHeaders_(_, _)) + .WillOnce(Invoke([&](Http::ResponseHeaderMap& headers, bool) -> void { + EXPECT_EQ(headers.Status()->value(), "200"); + })); + response_decoder->decodeHeaders(std::move(response_headers), true); + EXPECT_TRUE(verifyHostUpstreamStats(1, 1)); +} + +// Test retrying a request, when the first attempt fails while the client +// is sending the body, with more data arriving in between upstream attempts +// (which would normally happen during the backoff timer interval), but not end_stream. +TEST_F(RouterTest, RetryRequestDuringBodyDataBetweenAttemptsNotEndStream) { + Buffer::OwnedImpl decoding_buffer; + EXPECT_CALL(callbacks_, decodingBuffer()).WillRepeatedly(Return(&decoding_buffer)); + EXPECT_CALL(callbacks_, addDecodedData(_, true)) + .WillRepeatedly(Invoke([&](Buffer::Instance& data, bool) { decoding_buffer.move(data); })); + + NiceMock encoder1; + Http::ResponseDecoder* response_decoder = nullptr; + EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) + .WillOnce(Invoke( + [&](Http::ResponseDecoder& decoder, + Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { + response_decoder = &decoder; + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); + return nullptr; + })); + expectResponseTimerCreate(); + + Http::TestRequestHeaderMapImpl headers{ + {"x-envoy-retry-on", "5xx"}, {"x-envoy-internal", "true"}, {"myheader", "present"}}; + HttpTestUtility::addDefaultHeaders(headers); + router_.decodeHeaders(headers, false); + const std::string body1("body1"); + Buffer::OwnedImpl buf1(body1); + EXPECT_CALL(*router_.retry_state_, enabled()).Times(3).WillRepeatedly(Return(true)); + router_.decodeData(buf1, false); + + router_.retry_state_->expectResetRetry(); + encoder1.stream_.resetStream(Http::StreamResetReason::RemoteReset); + + const std::string body2("body2"); + Buffer::OwnedImpl buf2(body2); + router_.decodeData(buf2, false); + + NiceMock encoder2; + EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) + .WillOnce(Invoke( + [&](Http::ResponseDecoder& decoder, + Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { + response_decoder = &decoder; + callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_, upstream_stream_info_); + return nullptr; + })); + + EXPECT_CALL(encoder2, encodeHeaders(HeaderHasValueRef("myheader", "present"), false)); + EXPECT_CALL(encoder2, encodeData(BufferStringEqual(body1 + body2), false)); + router_.retry_state_->callback_(); + EXPECT_EQ(2U, + callbacks_.route_->route_entry_.virtual_cluster_.stats().upstream_rq_total_.value()); + EXPECT_TRUE(verifyHostUpstreamStats(0, 1)); + + // Complete request. Ensure original headers are present. + const std::string body3("body3"); + EXPECT_CALL(encoder2, encodeData(BufferStringEqual(body3), true)); + Buffer::OwnedImpl buf3(body3); + router_.decodeData(buf3, true); + + // Send successful response, verify success. + Http::ResponseHeaderMapPtr response_headers( + new Http::TestResponseHeaderMapImpl({{":status", "200"}})); + EXPECT_CALL(callbacks_, encodeHeaders_(_, _)) + .WillOnce(Invoke([&](Http::ResponseHeaderMap& headers, bool) -> void { + EXPECT_EQ(headers.Status()->value(), "200"); + })); + response_decoder->decodeHeaders(std::move(response_headers), true); + EXPECT_TRUE(verifyHostUpstreamStats(1, 1)); +} + +// Test retrying a request, when the first attempt fails while the client +// is sending the body, with the rest of the request arriving in between upstream +// request attempts. +TEST_F(RouterTest, RetryRequestDuringBodyCompleteBetweenAttempts) { + Buffer::OwnedImpl decoding_buffer; + EXPECT_CALL(callbacks_, decodingBuffer()).WillRepeatedly(Return(&decoding_buffer)); + EXPECT_CALL(callbacks_, addDecodedData(_, true)) + .WillRepeatedly(Invoke([&](Buffer::Instance& data, bool) { decoding_buffer.move(data); })); + + NiceMock encoder1; + Http::ResponseDecoder* response_decoder = nullptr; + EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) + .WillOnce(Invoke( + [&](Http::ResponseDecoder& decoder, + Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { + response_decoder = &decoder; + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); + return nullptr; + })); + + Http::TestRequestHeaderMapImpl headers{ + {"x-envoy-retry-on", "5xx"}, {"x-envoy-internal", "true"}, {"myheader", "present"}}; + HttpTestUtility::addDefaultHeaders(headers); + router_.decodeHeaders(headers, false); + const std::string body1("body1"); + Buffer::OwnedImpl buf1(body1); + EXPECT_CALL(*router_.retry_state_, enabled()).Times(2).WillRepeatedly(Return(true)); + router_.decodeData(buf1, false); + + router_.retry_state_->expectResetRetry(); + encoder1.stream_.resetStream(Http::StreamResetReason::RemoteReset); + + // Complete request while there is no upstream request. + const std::string body2("body2"); + Buffer::OwnedImpl buf2(body2); + router_.decodeData(buf2, true); + + NiceMock encoder2; + EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) + .WillOnce(Invoke( + [&](Http::ResponseDecoder& decoder, + Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { + response_decoder = &decoder; + callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_, upstream_stream_info_); + return nullptr; + })); + + EXPECT_CALL(encoder2, encodeHeaders(HeaderHasValueRef("myheader", "present"), false)); + EXPECT_CALL(encoder2, encodeData(BufferStringEqual(body1 + body2), true)); + router_.retry_state_->callback_(); + EXPECT_EQ(2U, + callbacks_.route_->route_entry_.virtual_cluster_.stats().upstream_rq_total_.value()); + EXPECT_TRUE(verifyHostUpstreamStats(0, 1)); + + // Send successful response, verify success. + Http::ResponseHeaderMapPtr response_headers( + new Http::TestResponseHeaderMapImpl({{":status", "200"}})); + EXPECT_CALL(callbacks_, encodeHeaders_(_, _)) + .WillOnce(Invoke([&](Http::ResponseHeaderMap& headers, bool) -> void { + EXPECT_EQ(headers.Status()->value(), "200"); + })); + response_decoder->decodeHeaders(std::move(response_headers), true); + EXPECT_TRUE(verifyHostUpstreamStats(1, 1)); +} + +// Test retrying a request, when the first attempt fails while the client +// is sending the body, with the trailers arriving in between upstream +// request attempts. +TEST_F(RouterTest, RetryRequestDuringBodyTrailerBetweenAttempts) { + Buffer::OwnedImpl decoding_buffer; + EXPECT_CALL(callbacks_, decodingBuffer()).WillRepeatedly(Return(&decoding_buffer)); + EXPECT_CALL(callbacks_, addDecodedData(_, true)) + .WillRepeatedly(Invoke([&](Buffer::Instance& data, bool) { decoding_buffer.move(data); })); + + NiceMock encoder1; + Http::ResponseDecoder* response_decoder = nullptr; + EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) + .WillOnce(Invoke( + [&](Http::ResponseDecoder& decoder, + Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { + response_decoder = &decoder; + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); + return nullptr; + })); + + Http::TestRequestHeaderMapImpl headers{ + {"x-envoy-retry-on", "5xx"}, {"x-envoy-internal", "true"}, {"myheader", "present"}}; + HttpTestUtility::addDefaultHeaders(headers); + router_.decodeHeaders(headers, false); + const std::string body1("body1"); + Buffer::OwnedImpl buf1(body1); + EXPECT_CALL(*router_.retry_state_, enabled()).WillOnce(Return(true)); + router_.decodeData(buf1, false); + + router_.retry_state_->expectResetRetry(); + encoder1.stream_.resetStream(Http::StreamResetReason::RemoteReset); + + // Complete request while there is no upstream request. + Http::TestRequestTrailerMapImpl trailers{{"some", "trailer"}}; + router_.decodeTrailers(trailers); + + NiceMock encoder2; + EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) + .WillOnce(Invoke( + [&](Http::ResponseDecoder& decoder, + Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { + response_decoder = &decoder; + callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_, upstream_stream_info_); + return nullptr; + })); + + EXPECT_CALL(encoder2, encodeHeaders(HeaderHasValueRef("myheader", "present"), false)); + EXPECT_CALL(encoder2, encodeData(BufferStringEqual(body1), false)); + EXPECT_CALL(encoder2, encodeTrailers(HeaderMapEqualRef(&trailers))); + router_.retry_state_->callback_(); + EXPECT_EQ(2U, + callbacks_.route_->route_entry_.virtual_cluster_.stats().upstream_rq_total_.value()); + EXPECT_TRUE(verifyHostUpstreamStats(0, 1)); + + // Send successful response, verify success. + Http::ResponseHeaderMapPtr response_headers( + new Http::TestResponseHeaderMapImpl({{":status", "200"}})); + EXPECT_CALL(callbacks_, encodeHeaders_(_, _)) + .WillOnce(Invoke([&](Http::ResponseHeaderMap& headers, bool) -> void { + EXPECT_EQ(headers.Status()->value(), "200"); + })); + response_decoder->decodeHeaders(std::move(response_headers), true); + EXPECT_TRUE(verifyHostUpstreamStats(1, 1)); +} + +// Test retrying a request, when the first attempt fails while the client +// is sending the body, with the rest of the request arriving in between upstream +// request attempts, but exceeding the buffer limit causing a downstream request abort. +TEST_F(RouterTest, RetryRequestDuringBodyBufferLimitExceeded) { + Buffer::OwnedImpl decoding_buffer; + EXPECT_CALL(callbacks_, decodingBuffer()).WillRepeatedly(Return(&decoding_buffer)); + EXPECT_CALL(callbacks_, addDecodedData(_, true)) + .WillRepeatedly(Invoke([&](Buffer::Instance& data, bool) { decoding_buffer.move(data); })); + EXPECT_CALL(callbacks_.route_->route_entry_, retryShadowBufferLimit()).WillOnce(Return(10)); + + NiceMock encoder1; + Http::ResponseDecoder* response_decoder = nullptr; + EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) + .WillOnce(Invoke( + [&](Http::ResponseDecoder& decoder, + Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { + response_decoder = &decoder; + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); + return nullptr; + })); + + Http::TestRequestHeaderMapImpl headers{ + {"x-envoy-retry-on", "5xx"}, {"x-envoy-internal", "true"}, {"myheader", "present"}}; + HttpTestUtility::addDefaultHeaders(headers); + router_.decodeHeaders(headers, false); + const std::string body1("body1"); + Buffer::OwnedImpl buf1(body1); + EXPECT_CALL(*router_.retry_state_, enabled()).Times(2).WillRepeatedly(Return(true)); + router_.decodeData(buf1, false); router_.retry_state_->expectResetRetry(); - EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, - putResult(Upstream::Outlier::Result::LocalOriginConnectFailed, _)); encoder1.stream_.resetStream(Http::StreamResetReason::RemoteReset); + + // Complete request while there is no upstream request. + const std::string body2(50, 'a'); + Buffer::OwnedImpl buf2(body2); + router_.decodeData(buf2, false); + + EXPECT_EQ(callbacks_.details_, "request_payload_exceeded_retry_buffer_limit"); + EXPECT_EQ(1U, cm_.thread_local_cluster_.cluster_.info_->stats_store_ + .counter("retry_or_shadow_abandoned") + .value()); EXPECT_TRUE(verifyHostUpstreamStats(0, 1)); - EXPECT_EQ(1UL, stats_store_.counter("test.rq_retry_skipped_request_not_complete").value()); } // Two requests are sent (slow request + hedged retry) and then global timeout @@ -4331,6 +4669,58 @@ TEST_F(RouterTest, DirectResponseWithoutLocation) { EXPECT_EQ(1UL, config_.stats_.rq_direct_response_.value()); } +// Allows verifying the state of the upstream StreamInfo +class TestAccessLog : public AccessLog::Instance { +public: + explicit TestAccessLog(std::function func) : func_(func) {} + + void log(const Http::RequestHeaderMap*, const Http::ResponseHeaderMap*, + const Http::ResponseTrailerMap*, const StreamInfo::StreamInfo& info) override { + func_(info); + } + +private: + std::function func_; +}; + +// Verifies that we propagate the upstream connection filter state to the upstream request filter +// state. +TEST_F(RouterTest, PropagatesUpstreamFilterState) { + NiceMock encoder; + Http::ResponseDecoder* response_decoder = nullptr; + + // This pattern helps ensure that we're actually invoking the callback. + bool filter_state_verified = false; + router_.config().upstream_logs_.push_back( + std::make_shared([&](const auto& stream_info) { + filter_state_verified = stream_info.upstreamFilterState()->hasDataWithName("upstream data"); + })); + + upstream_stream_info_.filterState()->setData( + "upstream data", std::make_unique(123), + StreamInfo::FilterState::StateType::ReadOnly, StreamInfo::FilterState::LifeSpan::Connection); + expectResponseTimerCreate(); + EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) + .WillOnce(Invoke( + [&](Http::ResponseDecoder& decoder, + Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { + response_decoder = &decoder; + callbacks.onPoolReady(encoder, cm_.conn_pool_.host_, upstream_stream_info_); + return nullptr; + })); + + Http::TestRequestHeaderMapImpl headers{}; + HttpTestUtility::addDefaultHeaders(headers); + router_.decodeHeaders(headers, true); + + Http::ResponseHeaderMapPtr response_headers( + new Http::TestResponseHeaderMapImpl{{":status", "200"}}); + response_decoder->decodeHeaders(std::move(response_headers), true); + EXPECT_TRUE(verifyHostUpstreamStats(1, 0)); + + EXPECT_TRUE(filter_state_verified); +} + TEST_F(RouterTest, UpstreamSSLConnection) { NiceMock encoder; Http::ResponseDecoder* response_decoder = nullptr; diff --git a/test/common/runtime/runtime_flag_override_test.cc b/test/common/runtime/runtime_flag_override_test.cc index a083e723bad8..37a1ab644473 100644 --- a/test/common/runtime/runtime_flag_override_test.cc +++ b/test/common/runtime/runtime_flag_override_test.cc @@ -1,4 +1,4 @@ -#include "common/runtime/runtime_impl.h" +#include "common/runtime/runtime_features.h" #include "gmock/gmock.h" diff --git a/test/common/runtime/runtime_impl_test.cc b/test/common/runtime/runtime_impl_test.cc index 85c14f3bbe21..1d9d7076a372 100644 --- a/test/common/runtime/runtime_impl_test.cc +++ b/test/common/runtime/runtime_impl_test.cc @@ -8,6 +8,7 @@ #include "envoy/type/v3/percent.pb.h" #include "common/config/runtime_utility.h" +#include "common/runtime/runtime_features.h" #include "common/runtime/runtime_impl.h" #include "test/common/stats/stat_test_utility.h" diff --git a/test/common/stats/isolated_store_impl_test.cc b/test/common/stats/isolated_store_impl_test.cc index ffa8e94f6915..ee618669cb2c 100644 --- a/test/common/stats/isolated_store_impl_test.cc +++ b/test/common/stats/isolated_store_impl_test.cc @@ -151,8 +151,17 @@ TEST_F(StatsIsolatedStoreImplTest, AllWithSymbolTable) { EXPECT_EQ("g1", g1.tagExtractedName()); EXPECT_EQ("scope1.g2", g2.tagExtractedName()); EXPECT_EQ(0, g1.tags().size()); - EXPECT_EQ(0, g1.tags().size()); + EXPECT_EQ(0, g2.tags().size()); + TextReadout& b1 = store_->textReadoutFromStatName(makeStatName("b1")); + TextReadout& b2 = scope1->textReadoutFromStatName(makeStatName("b2")); + EXPECT_NE(&b1, &b2); + EXPECT_EQ("b1", b1.name()); + EXPECT_EQ("scope1.b2", b2.name()); + EXPECT_EQ("b1", b1.tagExtractedName()); + EXPECT_EQ("scope1.b2", b2.tagExtractedName()); + EXPECT_EQ(0, b1.tags().size()); + EXPECT_EQ(0, b2.tags().size()); Histogram& h1 = store_->histogramFromStatName(makeStatName("h1"), Stats::Histogram::Unit::Unspecified); Histogram& h2 = @@ -176,6 +185,7 @@ TEST_F(StatsIsolatedStoreImplTest, AllWithSymbolTable) { EXPECT_EQ(4UL, store_->counters().size()); EXPECT_EQ(2UL, store_->gauges().size()); + EXPECT_EQ(2UL, store_->textReadouts().size()); } TEST_F(StatsIsolatedStoreImplTest, ConstSymtabAccessor) { @@ -197,19 +207,21 @@ TEST_F(StatsIsolatedStoreImplTest, LongStatName) { /** * Test stats macros. @see stats_macros.h */ -#define ALL_TEST_STATS(COUNTER, GAUGE, HISTOGRAM) \ +#define ALL_TEST_STATS(COUNTER, GAUGE, HISTOGRAM, TEXT_READOUT) \ COUNTER(test_counter) \ GAUGE(test_gauge, Accumulate) \ - HISTOGRAM(test_histogram, Microseconds) + HISTOGRAM(test_histogram, Microseconds) \ + TEXT_READOUT(test_text_readout) struct TestStats { - ALL_TEST_STATS(GENERATE_COUNTER_STRUCT, GENERATE_GAUGE_STRUCT, GENERATE_HISTOGRAM_STRUCT) + ALL_TEST_STATS(GENERATE_COUNTER_STRUCT, GENERATE_GAUGE_STRUCT, GENERATE_HISTOGRAM_STRUCT, + GENERATE_TEXT_READOUT_STRUCT) }; TEST_F(StatsIsolatedStoreImplTest, StatsMacros) { - TestStats test_stats{ALL_TEST_STATS(POOL_COUNTER_PREFIX(*store_, "test."), - POOL_GAUGE_PREFIX(*store_, "test."), - POOL_HISTOGRAM_PREFIX(*store_, "test."))}; + TestStats test_stats{ALL_TEST_STATS( + POOL_COUNTER_PREFIX(*store_, "test."), POOL_GAUGE_PREFIX(*store_, "test."), + POOL_HISTOGRAM_PREFIX(*store_, "test."), POOL_TEXT_READOUT_PREFIX(*store_, "test."))}; Counter& counter = test_stats.test_counter_; EXPECT_EQ("test.test_counter", counter.name()); @@ -217,6 +229,9 @@ TEST_F(StatsIsolatedStoreImplTest, StatsMacros) { Gauge& gauge = test_stats.test_gauge_; EXPECT_EQ("test.test_gauge", gauge.name()); + TextReadout& textReadout = test_stats.test_text_readout_; + EXPECT_EQ("test.test_text_readout", textReadout.name()); + Histogram& histogram = test_stats.test_histogram_; EXPECT_EQ("test.test_histogram", histogram.name()); EXPECT_EQ(Histogram::Unit::Microseconds, histogram.unit()); diff --git a/test/common/stats/thread_local_store_test.cc b/test/common/stats/thread_local_store_test.cc index ef71478b6559..3dc88b03c9b3 100644 --- a/test/common/stats/thread_local_store_test.cc +++ b/test/common/stats/thread_local_store_test.cc @@ -214,6 +214,9 @@ TEST_F(StatsThreadLocalStoreTest, NoTls) { ASSERT_TRUE(found_histogram.has_value()); EXPECT_EQ(&h1, &found_histogram->get()); + TextReadout& t1 = store_->textReadoutFromString("t1"); + EXPECT_EQ(&t1, &store_->textReadoutFromString("t1")); + EXPECT_CALL(sink_, onHistogramComplete(Ref(h1), 200)); h1.recordValue(200); EXPECT_CALL(sink_, onHistogramComplete(Ref(h1), 100)); @@ -225,6 +228,9 @@ TEST_F(StatsThreadLocalStoreTest, NoTls) { EXPECT_EQ(1UL, store_->gauges().size()); EXPECT_EQ(&g1, store_->gauges().front().get()); // front() ok when size()==1 EXPECT_EQ(2L, store_->gauges().front().use_count()); + EXPECT_EQ(1UL, store_->textReadouts().size()); + EXPECT_EQ(&t1, store_->textReadouts().front().get()); // front() ok when size()==1 + EXPECT_EQ(2L, store_->textReadouts().front().use_count()); store_->shutdownThreading(); } @@ -262,12 +268,19 @@ TEST_F(StatsThreadLocalStoreTest, Tls) { ASSERT_TRUE(found_histogram.has_value()); EXPECT_EQ(&h1, &found_histogram->get()); + TextReadout& t1 = store_->textReadoutFromString("t1"); + EXPECT_EQ(&t1, &store_->textReadoutFromString("t1")); + EXPECT_EQ(1UL, store_->counters().size()); + EXPECT_EQ(&c1, TestUtility::findCounter(*store_, "c1").get()); EXPECT_EQ(2L, TestUtility::findCounter(*store_, "c1").use_count()); EXPECT_EQ(1UL, store_->gauges().size()); EXPECT_EQ(&g1, store_->gauges().front().get()); // front() ok when size()==1 EXPECT_EQ(2L, store_->gauges().front().use_count()); + EXPECT_EQ(1UL, store_->textReadouts().size()); + EXPECT_EQ(&t1, store_->textReadouts().front().get()); // front() ok when size()==1 + EXPECT_EQ(2UL, store_->textReadouts().front().use_count()); store_->shutdownThreading(); tls_.shutdownThread(); @@ -278,6 +291,9 @@ TEST_F(StatsThreadLocalStoreTest, Tls) { EXPECT_EQ(1UL, store_->gauges().size()); EXPECT_EQ(&g1, store_->gauges().front().get()); // front() ok when size()==1 EXPECT_EQ(2L, store_->gauges().front().use_count()); + EXPECT_EQ(1UL, store_->textReadouts().size()); + EXPECT_EQ(&t1, store_->textReadouts().front().get()); // front() ok when size()==1 + EXPECT_EQ(2L, store_->textReadouts().front().use_count()); } TEST_F(StatsThreadLocalStoreTest, BasicScope) { @@ -328,6 +344,11 @@ TEST_F(StatsThreadLocalStoreTest, BasicScope) { ASSERT_TRUE(found_histogram2.has_value()); EXPECT_EQ(&h2, &found_histogram2->get()); + TextReadout& t1 = store_->textReadoutFromString("t1"); + TextReadout& t2 = scope1->textReadoutFromString("t2"); + EXPECT_EQ("t1", t1.name()); + EXPECT_EQ("scope1.t2", t2.name()); + StatNameManagedStorage tag_key("a", *symbol_table_); StatNameManagedStorage tag_value("b", *symbol_table_); StatNameTagVector tags{{StatName(tag_key.statName()), StatName(tag_value.statName())}}; @@ -435,6 +456,9 @@ TEST_F(StatsThreadLocalStoreTest, NestedScopes) { Gauge& g1 = scope2->gaugeFromString("some_gauge", Gauge::ImportMode::Accumulate); EXPECT_EQ("scope1.foo.some_gauge", g1.name()); + TextReadout& t1 = scope2->textReadoutFromString("some_string"); + EXPECT_EQ("scope1.foo.some_string", t1.name()); + store_->shutdownThreading(); tls_.shutdownThread(); } @@ -474,6 +498,19 @@ TEST_F(StatsThreadLocalStoreTest, OverlappingScopes) { EXPECT_EQ(1UL, g2.value()); EXPECT_EQ(1UL, store_->gauges().size()); + // TextReadouts should work just like gauges. + TextReadout& t1 = scope1->textReadoutFromString("b"); + TextReadout& t2 = scope2->textReadoutFromString("b"); + EXPECT_EQ(&t1, &t2); + + t1.set("hello"); + EXPECT_EQ("hello", t1.value()); + EXPECT_EQ("hello", t2.value()); + t2.set("goodbye"); + EXPECT_EQ("goodbye", t1.value()); + EXPECT_EQ("goodbye", t2.value()); + EXPECT_EQ(1UL, store_->textReadouts().size()); + // Deleting scope 1 will call free but will be reference counted. It still leaves scope 2 valid. scope1.reset(); c2.inc(); @@ -482,6 +519,54 @@ TEST_F(StatsThreadLocalStoreTest, OverlappingScopes) { g2.set(10); EXPECT_EQ(10UL, g2.value()); EXPECT_EQ(1UL, store_->gauges().size()); + t2.set("abc"); + EXPECT_EQ("abc", t2.value()); + EXPECT_EQ(1UL, store_->textReadouts().size()); + + store_->shutdownThreading(); + tls_.shutdownThread(); +} + +TEST_F(StatsThreadLocalStoreTest, TextReadoutAllLengths) { + store_->initializeThreading(main_thread_dispatcher_, tls_); + + TextReadout& t = store_->textReadoutFromString("t"); + EXPECT_EQ("", t.value()); + std::string str; + // ASCII + for (int i = 0; i < 15; i++) { + str += ('a' + i); + t.set(std::string(str)); + EXPECT_EQ(str, t.value()); + } + + // Non-ASCII + str = ""; + for (int i = 0; i < 15; i++) { + str += ('\xEE' + i); + t.set(std::string(str)); + EXPECT_EQ(str, t.value()); + } + + // Null bytes ok; the TextReadout implementation doesn't use null termination in its storage + t.set(std::string("\x00", 1)); + EXPECT_EQ(std::string("\x00", 1), t.value()); + t.set(std::string("\x00\x00\x00", 3)); + EXPECT_EQ(std::string("\x00\x00\x00", 3), t.value()); + EXPECT_NE(std::string("\x00", 1), t.value()); + EXPECT_NE(std::string("", 0), t.value()); + + // No Truncation to 15 + t.set("aaaabbbbccccdddX"); + EXPECT_EQ("aaaabbbbccccdddX", t.value()); + t.set("aaaabbbbccccdddXX"); + EXPECT_EQ("aaaabbbbccccdddXX", t.value()); + t.set("aaaabbbbccccdddXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"); + // EXPECT_EQ("aaaabbbbccccddd", t.value()); + + // Can set back to empty + t.set(""); + EXPECT_EQ("", t.value()); store_->shutdownThreading(); tls_.shutdownThread(); @@ -611,6 +696,21 @@ TEST_F(StatsMatcherTLSTest, TestNoOpStatImpls) { Gauge& noop_gauge_2 = store_->gaugeFromString("noop_gauge_2", Gauge::ImportMode::Accumulate); EXPECT_EQ(&noop_gauge, &noop_gauge_2); + // TextReadout + TextReadout& noop_string = store_->textReadoutFromString("noop_string"); + EXPECT_EQ(noop_string.name(), ""); + EXPECT_EQ("", noop_string.value()); + noop_string.set("hello"); + EXPECT_EQ("", noop_string.value()); + noop_string.set("hello"); + EXPECT_EQ("", noop_string.value()); + noop_string.set("goodbye"); + EXPECT_EQ("", noop_string.value()); + noop_string.set("hello"); + EXPECT_EQ("", noop_string.value()); + TextReadout& noop_string_2 = store_->textReadoutFromString("noop_string_2"); + EXPECT_EQ(&noop_string, &noop_string_2); + // Histogram Histogram& noop_histogram = store_->histogramFromString("noop_histogram", Stats::Histogram::Unit::Unspecified); @@ -647,6 +747,8 @@ TEST_F(StatsMatcherTLSTest, TestExclusionRegex) { store_->histogramFromString("lowercase_histogram", Stats::Histogram::Unit::Unspecified); EXPECT_EQ(lowercase_histogram.name(), "lowercase_histogram"); + TextReadout& lowercase_string = store_->textReadoutFromString("lowercase_string"); + EXPECT_EQ(lowercase_string.name(), "lowercase_string"); // And the creation of counters/gauges/histograms which have uppercase letters should fail. Counter& uppercase_counter = store_->counterFromString("UPPERCASE_counter"); EXPECT_EQ(uppercase_counter.name(), ""); @@ -663,6 +765,11 @@ TEST_F(StatsMatcherTLSTest, TestExclusionRegex) { uppercase_gauge.inc(); EXPECT_EQ(uppercase_gauge.value(), 0); + TextReadout& uppercase_string = store_->textReadoutFromString("uppercase_STRING"); + EXPECT_EQ(uppercase_string.name(), ""); + uppercase_string.set("A STRING VALUE"); + EXPECT_EQ("", uppercase_string.value()); + // Histograms are harder to query and test, so we resort to testing that name() returns the empty // string. Histogram& uppercase_histogram = @@ -714,6 +821,18 @@ TEST_F(StatsMatcherTLSTest, TestExclusionRegex) { store_->histogramFromString("also_INVALID_histogram", Stats::Histogram::Unit::Unspecified); EXPECT_EQ(invalid_histogram_2.name(), ""); + TextReadout& valid_string = store_->textReadoutFromString("valid_string"); + valid_string.set("i'm valid"); + EXPECT_EQ("i'm valid", valid_string.value()); + + TextReadout& invalid_string_1 = store_->textReadoutFromString("invalid_string"); + invalid_string_1.set("nope"); + EXPECT_EQ("", invalid_string_1.value()); + + TextReadout& invalid_string_2 = store_->textReadoutFromString("also_INVLD_string"); + invalid_string_2.set("still no"); + EXPECT_EQ("", invalid_string_2.value()); + // Expected to free lowercase_counter, lowercase_gauge, valid_counter, valid_gauge store_->shutdownThreading(); } @@ -865,6 +984,7 @@ TEST_F(StatsThreadLocalStoreTest, RemoveRejectedStats) { Counter& counter = store_->counterFromString("c1"); Gauge& gauge = store_->gaugeFromString("g1", Gauge::ImportMode::Accumulate); Histogram& histogram = store_->histogramFromString("h1", Stats::Histogram::Unit::Unspecified); + TextReadout& textReadout = store_->textReadoutFromString("t1"); ASSERT_EQ(1, store_->counters().size()); // "c1". EXPECT_TRUE(&counter == store_->counters()[0].get() || &counter == store_->counters()[1].get()); // counters() order is non-deterministic. @@ -872,6 +992,8 @@ TEST_F(StatsThreadLocalStoreTest, RemoveRejectedStats) { EXPECT_EQ("g1", store_->gauges()[0]->name()); ASSERT_EQ(1, store_->histograms().size()); EXPECT_EQ("h1", store_->histograms()[0]->name()); + ASSERT_EQ(1, store_->textReadouts().size()); + EXPECT_EQ("t1", store_->textReadouts()[0]->name()); // Will effectively block all stats, and remove all the non-matching stats. envoy::config::metrics::v3::StatsConfig stats_config; @@ -883,12 +1005,14 @@ TEST_F(StatsThreadLocalStoreTest, RemoveRejectedStats) { EXPECT_EQ(0, store_->counters().size()); EXPECT_EQ(0, store_->gauges().size()); EXPECT_EQ(0, store_->histograms().size()); + EXPECT_EQ(0, store_->textReadouts().size()); // However, referencing the previously allocated stats will not crash. counter.inc(); gauge.inc(); EXPECT_CALL(sink_, onHistogramComplete(Ref(histogram), 42)); histogram.recordValue(42); + textReadout.set("fortytwo"); store_->shutdownThreading(); tls_.shutdownThread(); } @@ -963,7 +1087,7 @@ TEST_F(StatsThreadLocalStoreTestNoFixture, MemoryWithTlsFakeSymbolTable) { TestUtil::MemoryTest memory_test; TestUtil::forEachSampleStat( 100, [this](absl::string_view name) { store_->counterFromString(std::string(name)); }); - EXPECT_MEMORY_EQ(memory_test.consumedBytes(), 1498128); // Jan 23, 2020 + EXPECT_MEMORY_EQ(memory_test.consumedBytes(), 1498160); // Apr 8, 2020 EXPECT_MEMORY_LE(memory_test.consumedBytes(), 1.6 * million_); } @@ -983,7 +1107,7 @@ TEST_F(StatsThreadLocalStoreTestNoFixture, MemoryWithTlsRealSymbolTable) { TestUtil::MemoryTest memory_test; TestUtil::forEachSampleStat( 100, [this](absl::string_view name) { store_->counterFromString(std::string(name)); }); - EXPECT_MEMORY_EQ(memory_test.consumedBytes(), 829200); // Jan 23, 2020 + EXPECT_MEMORY_EQ(memory_test.consumedBytes(), 829232); // Apr 08, 2020 EXPECT_MEMORY_LE(memory_test.consumedBytes(), 0.9 * million_); } @@ -993,17 +1117,25 @@ TEST_F(StatsThreadLocalStoreTest, ShuttingDown) { store_->counterFromString("c1"); store_->gaugeFromString("g1", Gauge::ImportMode::Accumulate); + store_->textReadoutFromString("t1"); store_->shutdownThreading(); store_->counterFromString("c2"); store_->gaugeFromString("g2", Gauge::ImportMode::Accumulate); + store_->textReadoutFromString("t2"); // We do not keep ref-counts for counters and gauges in the TLS cache, so // all these stats should have a ref-count of 2: one for the SharedPtr // returned from find*(), and one for the central cache. EXPECT_EQ(2L, TestUtility::findCounter(*store_, "c1").use_count()); EXPECT_EQ(2L, TestUtility::findGauge(*store_, "g1").use_count()); + + // c1, g1, t1 should have a thread local ref, but c2, g2, t2 should not. + EXPECT_EQ(2L, TestUtility::findCounter(*store_, "c1").use_count()); + EXPECT_EQ(2L, TestUtility::findGauge(*store_, "g1").use_count()); + EXPECT_EQ(2L, TestUtility::findTextReadout(*store_, "t1").use_count()); EXPECT_EQ(2L, TestUtility::findCounter(*store_, "c2").use_count()); EXPECT_EQ(2L, TestUtility::findGauge(*store_, "g2").use_count()); + EXPECT_EQ(2L, TestUtility::findTextReadout(*store_, "t2").use_count()); store_->shutdownThreading(); tls_.shutdownThread(); diff --git a/test/common/stream_info/filter_state_impl_test.cc b/test/common/stream_info/filter_state_impl_test.cc index 4c2659463ada..24590fa7c371 100644 --- a/test/common/stream_info/filter_state_impl_test.cc +++ b/test/common/stream_info/filter_state_impl_test.cc @@ -249,15 +249,13 @@ TEST_F(FilterStateImplTest, LifeSpanInitFromParent) { filter_state().setData("test_2", std::make_unique(2), FilterState::StateType::Mutable, FilterState::LifeSpan::FilterChain); filter_state().setData("test_3", std::make_unique(3), - FilterState::StateType::ReadOnly, - FilterState::LifeSpan::DownstreamRequest); + FilterState::StateType::ReadOnly, FilterState::LifeSpan::Request); filter_state().setData("test_4", std::make_unique(4), FilterState::StateType::Mutable, - FilterState::LifeSpan::DownstreamRequest); + FilterState::LifeSpan::Request); filter_state().setData("test_5", std::make_unique(5), - FilterState::StateType::ReadOnly, - FilterState::LifeSpan::DownstreamConnection); + FilterState::StateType::ReadOnly, FilterState::LifeSpan::Connection); filter_state().setData("test_6", std::make_unique(6), FilterState::StateType::Mutable, - FilterState::LifeSpan::DownstreamConnection); + FilterState::LifeSpan::Connection); FilterStateImpl new_filter_state(filter_state().parent(), FilterState::LifeSpan::FilterChain); EXPECT_FALSE(new_filter_state.hasDataWithName("test_1")); @@ -282,15 +280,13 @@ TEST_F(FilterStateImplTest, LifeSpanInitFromGrandparent) { filter_state().setData("test_2", std::make_unique(2), FilterState::StateType::Mutable, FilterState::LifeSpan::FilterChain); filter_state().setData("test_3", std::make_unique(3), - FilterState::StateType::ReadOnly, - FilterState::LifeSpan::DownstreamRequest); + FilterState::StateType::ReadOnly, FilterState::LifeSpan::Request); filter_state().setData("test_4", std::make_unique(4), FilterState::StateType::Mutable, - FilterState::LifeSpan::DownstreamRequest); + FilterState::LifeSpan::Request); filter_state().setData("test_5", std::make_unique(5), - FilterState::StateType::ReadOnly, - FilterState::LifeSpan::DownstreamConnection); + FilterState::StateType::ReadOnly, FilterState::LifeSpan::Connection); filter_state().setData("test_6", std::make_unique(6), FilterState::StateType::Mutable, - FilterState::LifeSpan::DownstreamConnection); + FilterState::LifeSpan::Connection); FilterStateImpl new_filter_state(filter_state().parent()->parent(), FilterState::LifeSpan::FilterChain); @@ -312,18 +308,15 @@ TEST_F(FilterStateImplTest, LifeSpanInitFromNonParent) { filter_state().setData("test_2", std::make_unique(2), FilterState::StateType::Mutable, FilterState::LifeSpan::FilterChain); filter_state().setData("test_3", std::make_unique(3), - FilterState::StateType::ReadOnly, - FilterState::LifeSpan::DownstreamRequest); + FilterState::StateType::ReadOnly, FilterState::LifeSpan::Request); filter_state().setData("test_4", std::make_unique(4), FilterState::StateType::Mutable, - FilterState::LifeSpan::DownstreamRequest); + FilterState::LifeSpan::Request); filter_state().setData("test_5", std::make_unique(5), - FilterState::StateType::ReadOnly, - FilterState::LifeSpan::DownstreamConnection); + FilterState::StateType::ReadOnly, FilterState::LifeSpan::Connection); filter_state().setData("test_6", std::make_unique(6), FilterState::StateType::Mutable, - FilterState::LifeSpan::DownstreamConnection); + FilterState::LifeSpan::Connection); - FilterStateImpl new_filter_state(filter_state().parent(), - FilterState::LifeSpan::DownstreamRequest); + FilterStateImpl new_filter_state(filter_state().parent(), FilterState::LifeSpan::Request); EXPECT_FALSE(new_filter_state.hasDataWithName("test_1")); EXPECT_FALSE(new_filter_state.hasDataWithName("test_2")); EXPECT_FALSE(new_filter_state.hasDataWithName("test_3")); @@ -336,29 +329,25 @@ TEST_F(FilterStateImplTest, HasDataAtOrAboveLifeSpan) { filter_state().setData("test_1", std::make_unique(1), FilterState::StateType::ReadOnly, FilterState::LifeSpan::FilterChain); EXPECT_TRUE(filter_state().hasDataAtOrAboveLifeSpan(FilterState::LifeSpan::FilterChain)); - EXPECT_FALSE(filter_state().hasDataAtOrAboveLifeSpan(FilterState::LifeSpan::DownstreamRequest)); - EXPECT_FALSE( - filter_state().hasDataAtOrAboveLifeSpan(FilterState::LifeSpan::DownstreamConnection)); + EXPECT_FALSE(filter_state().hasDataAtOrAboveLifeSpan(FilterState::LifeSpan::Request)); + EXPECT_FALSE(filter_state().hasDataAtOrAboveLifeSpan(FilterState::LifeSpan::Connection)); filter_state().setData("test_2", std::make_unique(2), - FilterState::StateType::ReadOnly, - FilterState::LifeSpan::DownstreamRequest); + FilterState::StateType::ReadOnly, FilterState::LifeSpan::Request); EXPECT_TRUE(filter_state().hasDataAtOrAboveLifeSpan(FilterState::LifeSpan::FilterChain)); - EXPECT_TRUE(filter_state().hasDataAtOrAboveLifeSpan(FilterState::LifeSpan::DownstreamRequest)); - EXPECT_FALSE( - filter_state().hasDataAtOrAboveLifeSpan(FilterState::LifeSpan::DownstreamConnection)); + EXPECT_TRUE(filter_state().hasDataAtOrAboveLifeSpan(FilterState::LifeSpan::Request)); + EXPECT_FALSE(filter_state().hasDataAtOrAboveLifeSpan(FilterState::LifeSpan::Connection)); filter_state().setData("test_3", std::make_unique(3), - FilterState::StateType::ReadOnly, - FilterState::LifeSpan::DownstreamConnection); + FilterState::StateType::ReadOnly, FilterState::LifeSpan::Connection); EXPECT_TRUE(filter_state().hasDataAtOrAboveLifeSpan(FilterState::LifeSpan::FilterChain)); - EXPECT_TRUE(filter_state().hasDataAtOrAboveLifeSpan(FilterState::LifeSpan::DownstreamRequest)); - EXPECT_TRUE(filter_state().hasDataAtOrAboveLifeSpan(FilterState::LifeSpan::DownstreamConnection)); + EXPECT_TRUE(filter_state().hasDataAtOrAboveLifeSpan(FilterState::LifeSpan::Request)); + EXPECT_TRUE(filter_state().hasDataAtOrAboveLifeSpan(FilterState::LifeSpan::Connection)); } TEST_F(FilterStateImplTest, SetSameDataWithDifferentLifeSpan) { filter_state().setData("test_1", std::make_unique(1), FilterState::StateType::Mutable, - FilterState::LifeSpan::DownstreamConnection); + FilterState::LifeSpan::Connection); // Test reset on smaller LifeSpan EXPECT_THROW_WITH_MESSAGE( filter_state().setData("test_1", std::make_unique(2), @@ -367,18 +356,17 @@ TEST_F(FilterStateImplTest, SetSameDataWithDifferentLifeSpan) { "FilterState::setData called twice with conflicting life_span on the same data_name."); EXPECT_THROW_WITH_MESSAGE( filter_state().setData("test_1", std::make_unique(2), - FilterState::StateType::Mutable, - FilterState::LifeSpan::DownstreamRequest), + FilterState::StateType::Mutable, FilterState::LifeSpan::Request), EnvoyException, "FilterState::setData called twice with conflicting life_span on the same data_name."); // Still mutable on the correct LifeSpan. filter_state().setData("test_1", std::make_unique(2), FilterState::StateType::Mutable, - FilterState::LifeSpan::DownstreamConnection); + FilterState::LifeSpan::Connection); EXPECT_EQ(2, filter_state().getDataMutable("test_1").access()); filter_state().setData("test_2", std::make_unique(1), FilterState::StateType::Mutable, - FilterState::LifeSpan::DownstreamRequest); + FilterState::LifeSpan::Request); // Test reset on smaller and greater LifeSpan EXPECT_THROW_WITH_MESSAGE( filter_state().setData("test_2", std::make_unique(2), @@ -387,14 +375,13 @@ TEST_F(FilterStateImplTest, SetSameDataWithDifferentLifeSpan) { "FilterState::setData called twice with conflicting life_span on the same data_name."); EXPECT_THROW_WITH_MESSAGE( filter_state().setData("test_2", std::make_unique(2), - FilterState::StateType::Mutable, - FilterState::LifeSpan::DownstreamConnection), + FilterState::StateType::Mutable, FilterState::LifeSpan::Connection), EnvoyException, "FilterState::setData called twice with conflicting life_span on the same data_name."); // Still mutable on the correct LifeSpan. filter_state().setData("test_2", std::make_unique(2), FilterState::StateType::Mutable, - FilterState::LifeSpan::DownstreamRequest); + FilterState::LifeSpan::Request); EXPECT_EQ(2, filter_state().getDataMutable("test_2").access()); } diff --git a/test/common/tcp_proxy/tcp_proxy_test.cc b/test/common/tcp_proxy/tcp_proxy_test.cc index 34a92a7798cb..d3aed149adcd 100644 --- a/test/common/tcp_proxy/tcp_proxy_test.cc +++ b/test/common/tcp_proxy/tcp_proxy_test.cc @@ -751,8 +751,7 @@ TEST(ConfigTest, PerConnectionClusterWithTopLevelMetadataMatchConfig) { NiceMock connection; connection.stream_info_.filterState()->setData( "envoy.tcp_proxy.cluster", std::make_unique("filter_state_cluster"), - StreamInfo::FilterState::StateType::Mutable, - StreamInfo::FilterState::LifeSpan::DownstreamConnection); + StreamInfo::FilterState::StateType::Mutable, StreamInfo::FilterState::LifeSpan::Connection); const auto route = config_obj.getRouteFromEntries(connection); EXPECT_NE(nullptr, route); @@ -1719,8 +1718,7 @@ TEST_F(TcpProxyTest, ShareFilterState) { upstream_connections_.at(0)->streamInfo().filterState()->setData( "envoy.tcp_proxy.cluster", std::make_unique("filter_state_cluster"), - StreamInfo::FilterState::StateType::Mutable, - StreamInfo::FilterState::LifeSpan::DownstreamConnection); + StreamInfo::FilterState::StateType::Mutable, StreamInfo::FilterState::LifeSpan::Connection); raiseEventUpstreamConnected(0); EXPECT_EQ("filter_state_cluster", filter_callbacks_.connection_.streamInfo() @@ -1828,8 +1826,7 @@ TEST_F(TcpProxyRoutingTest, DEPRECATED_FEATURE_TEST(UseClusterFromPerConnectionC connection_.streamInfo().filterState()->setData( "envoy.tcp_proxy.cluster", std::make_unique("filter_state_cluster"), - StreamInfo::FilterState::StateType::Mutable, - StreamInfo::FilterState::LifeSpan::DownstreamConnection); + StreamInfo::FilterState::StateType::Mutable, StreamInfo::FilterState::LifeSpan::Connection); // Expect filter to try to open a connection to specified cluster. EXPECT_CALL(factory_context_.cluster_manager_, @@ -1846,8 +1843,7 @@ TEST_F(TcpProxyRoutingTest, DEPRECATED_FEATURE_TEST(UpstreamServerName)) { connection_.streamInfo().filterState()->setData( "envoy.network.upstream_server_name", std::make_unique("www.example.com"), - StreamInfo::FilterState::StateType::ReadOnly, - StreamInfo::FilterState::LifeSpan::DownstreamConnection); + StreamInfo::FilterState::StateType::ReadOnly, StreamInfo::FilterState::LifeSpan::Connection); // Expect filter to try to open a connection to a cluster with the transport socket options with // override-server-name @@ -1878,8 +1874,7 @@ TEST_F(TcpProxyRoutingTest, DEPRECATED_FEATURE_TEST(ApplicationProtocols)) { connection_.streamInfo().filterState()->setData( Network::ApplicationProtocols::key(), std::make_unique(std::vector{"foo", "bar"}), - StreamInfo::FilterState::StateType::ReadOnly, - StreamInfo::FilterState::LifeSpan::DownstreamConnection); + StreamInfo::FilterState::StateType::ReadOnly, StreamInfo::FilterState::LifeSpan::Connection); // Expect filter to try to open a connection to a cluster with the transport socket options with // override-application-protocol diff --git a/test/extensions/common/proxy_protocol/proxy_protocol_regression_test.cc b/test/extensions/common/proxy_protocol/proxy_protocol_regression_test.cc index 76a3d9e96d00..701df21eb901 100644 --- a/test/extensions/common/proxy_protocol/proxy_protocol_regression_test.cc +++ b/test/extensions/common/proxy_protocol/proxy_protocol_regression_test.cc @@ -51,7 +51,7 @@ class ProxyProtocolRegressionTest : public testing::TestWithParamlocalAddress())); EXPECT_CALL(socket_factory_, getListenSocket()).WillOnce(Return(socket_)); - connection_handler_->addListener(*this); + connection_handler_->addListener(absl::nullopt, *this); conn_ = dispatcher_->createClientConnection(socket_->localAddress(), Network::Address::InstanceConstSharedPtr(), Network::Test::createRawBufferSocket(), nullptr); diff --git a/test/extensions/filters/common/fault/fault_config_test.cc b/test/extensions/filters/common/fault/fault_config_test.cc index a402e1bf6156..340cd2c3dd70 100644 --- a/test/extensions/filters/common/fault/fault_config_test.cc +++ b/test/extensions/filters/common/fault/fault_config_test.cc @@ -18,26 +18,56 @@ TEST(FaultConfigTest, FaultAbortHeaderConfig) { proto_config.mutable_header_abort(); FaultAbortConfig config(proto_config); - // No header. - EXPECT_EQ(absl::nullopt, config.statusCode(nullptr)); - // Header with bad data. - Http::TestHeaderMapImpl bad_headers{{"x-envoy-fault-abort-request", "abc"}}; - EXPECT_EQ(absl::nullopt, config.statusCode(bad_headers.get(HeaderNames::get().AbortRequest))); + Http::TestRequestHeaderMapImpl bad_headers{{"x-envoy-fault-abort-request", "abc"}}; + EXPECT_EQ(absl::nullopt, config.statusCode(&bad_headers)); // Out of range header - value too low. - Http::TestHeaderMapImpl too_low_headers{{"x-envoy-fault-abort-request", "199"}}; - EXPECT_EQ(absl::nullopt, config.statusCode(too_low_headers.get(HeaderNames::get().AbortRequest))); + Http::TestRequestHeaderMapImpl too_low_headers{{"x-envoy-fault-abort-request", "199"}}; + EXPECT_EQ(absl::nullopt, config.statusCode(&too_low_headers)); // Out of range header - value too high. - Http::TestHeaderMapImpl too_high_headers{{"x-envoy-fault-abort-request", "600"}}; - EXPECT_EQ(absl::nullopt, - config.statusCode(too_high_headers.get(HeaderNames::get().AbortRequest))); + Http::TestRequestHeaderMapImpl too_high_headers{{"x-envoy-fault-abort-request", "600"}}; + EXPECT_EQ(absl::nullopt, config.statusCode(&too_high_headers)); // Valid header. - Http::TestHeaderMapImpl good_headers{{"x-envoy-fault-abort-request", "401"}}; - EXPECT_EQ(Http::Code::Unauthorized, - config.statusCode(good_headers.get(HeaderNames::get().AbortRequest)).value()); + Http::TestRequestHeaderMapImpl good_headers{{"x-envoy-fault-abort-request", "401"}}; + EXPECT_EQ(Http::Code::Unauthorized, config.statusCode(&good_headers).value()); +} + +TEST(FaultConfigTest, FaultAbortPercentageHeaderConfig) { + envoy::extensions::filters::http::fault::v3::FaultAbort proto_config; + proto_config.mutable_header_abort(); + proto_config.mutable_percentage()->set_numerator(80); + proto_config.mutable_percentage()->set_denominator(envoy::type::v3::FractionalPercent::HUNDRED); + FaultAbortConfig config(proto_config); + + // Header with bad data - fallback to proto config. + Http::TestRequestHeaderMapImpl bad_headers{{"x-envoy-fault-abort-request-percentage", "abc"}}; + const auto bad_headers_percentage = config.percentage(&bad_headers); + EXPECT_EQ(proto_config.percentage().numerator(), bad_headers_percentage.numerator()); + EXPECT_EQ(proto_config.percentage().denominator(), bad_headers_percentage.denominator()); + + // Out of range header, value too low - fallback to proto config. + Http::TestRequestHeaderMapImpl too_low_headers{{"x-envoy-fault-abort-request-percentage", "-1"}}; + const auto too_low_headers_percentage = config.percentage(&too_low_headers); + EXPECT_EQ(proto_config.percentage().numerator(), too_low_headers_percentage.numerator()); + EXPECT_EQ(proto_config.percentage().denominator(), too_low_headers_percentage.denominator()); + + // Valid header with value greater than the value of the numerator of default percentage - use + // proto config. + Http::TestRequestHeaderMapImpl good_headers{{"x-envoy-fault-abort-request-percentage", "90"}}; + const auto good_headers_percentage = config.percentage(&good_headers); + EXPECT_EQ(proto_config.percentage().numerator(), good_headers_percentage.numerator()); + EXPECT_EQ(proto_config.percentage().denominator(), good_headers_percentage.denominator()); + + // Valid header with value lesser than the value of the numerator of default percentage. + Http::TestRequestHeaderMapImpl greater_numerator_headers{ + {"x-envoy-fault-abort-request-percentage", "60"}}; + const auto greater_numerator_headers_percentage = config.percentage(&greater_numerator_headers); + EXPECT_EQ(60, greater_numerator_headers_percentage.numerator()); + EXPECT_EQ(proto_config.percentage().denominator(), + greater_numerator_headers_percentage.denominator()); } TEST(FaultConfigTest, FaultDelayHeaderConfig) { @@ -45,17 +75,49 @@ TEST(FaultConfigTest, FaultDelayHeaderConfig) { proto_config.mutable_header_delay(); FaultDelayConfig config(proto_config); - // No header. - EXPECT_EQ(absl::nullopt, config.duration(nullptr)); - // Header with bad data. - Http::TestHeaderMapImpl bad_headers{{"x-envoy-fault-delay-request", "abc"}}; - EXPECT_EQ(absl::nullopt, config.duration(bad_headers.get(HeaderNames::get().DelayRequest))); + Http::TestRequestHeaderMapImpl bad_headers{{"x-envoy-fault-delay-request", "abc"}}; + EXPECT_EQ(absl::nullopt, config.duration(&bad_headers)); // Valid header. - Http::TestHeaderMapImpl good_headers{{"x-envoy-fault-delay-request", "123"}}; - EXPECT_EQ(std::chrono::milliseconds(123), - config.duration(good_headers.get(HeaderNames::get().DelayRequest)).value()); + Http::TestRequestHeaderMapImpl good_headers{{"x-envoy-fault-delay-request", "123"}}; + EXPECT_EQ(std::chrono::milliseconds(123), config.duration(&good_headers).value()); +} + +TEST(FaultConfigTest, FaultDelayPercentageHeaderConfig) { + envoy::extensions::filters::common::fault::v3::FaultDelay proto_config; + proto_config.mutable_header_delay(); + proto_config.mutable_percentage()->set_numerator(80); + proto_config.mutable_percentage()->set_denominator( + envoy::type::v3::FractionalPercent::TEN_THOUSAND); + FaultDelayConfig config(proto_config); + + // Header with bad data - fallback to proto config. + Http::TestRequestHeaderMapImpl bad_headers{{"x-envoy-fault-delay-request-percentage", "abc"}}; + const auto bad_headers_percentage = config.percentage(&bad_headers); + EXPECT_EQ(proto_config.percentage().numerator(), bad_headers_percentage.numerator()); + EXPECT_EQ(proto_config.percentage().denominator(), bad_headers_percentage.denominator()); + + // Out of range header, value too low - fallback to proto config. + Http::TestRequestHeaderMapImpl too_low_headers{{"x-envoy-fault-delay-request-percentage", "-1"}}; + const auto too_low_headers_percentage = config.percentage(&too_low_headers); + EXPECT_EQ(proto_config.percentage().numerator(), too_low_headers_percentage.numerator()); + EXPECT_EQ(proto_config.percentage().denominator(), too_low_headers_percentage.denominator()); + + // Valid header with value greater than the value of the numerator of default percentage - use + // proto config. + Http::TestRequestHeaderMapImpl good_headers{{"x-envoy-fault-delay-request-percentage", "90"}}; + const auto good_headers_percentage = config.percentage(&good_headers); + EXPECT_EQ(proto_config.percentage().numerator(), good_headers_percentage.numerator()); + EXPECT_EQ(proto_config.percentage().denominator(), good_headers_percentage.denominator()); + + // Valid header with value lesser than the value of the numerator of default percentage. + Http::TestRequestHeaderMapImpl greater_numerator_headers{ + {"x-envoy-fault-delay-request-percentage", "60"}}; + const auto greater_numerator_headers_percentage = config.percentage(&greater_numerator_headers); + EXPECT_EQ(60, greater_numerator_headers_percentage.numerator()); + EXPECT_EQ(proto_config.percentage().denominator(), + greater_numerator_headers_percentage.denominator()); } TEST(FaultConfigTest, FaultRateLimitHeaderConfig) { @@ -63,22 +125,55 @@ TEST(FaultConfigTest, FaultRateLimitHeaderConfig) { proto_config.mutable_header_limit(); FaultRateLimitConfig config(proto_config); - // No header. - EXPECT_EQ(absl::nullopt, config.rateKbps(nullptr)); - // Header with bad data. - Http::TestHeaderMapImpl bad_headers{{"x-envoy-fault-throughput-response", "abc"}}; - EXPECT_EQ(absl::nullopt, config.rateKbps(bad_headers.get(HeaderNames::get().ThroughputResponse))); + Http::TestRequestHeaderMapImpl bad_headers{{"x-envoy-fault-throughput-response", "abc"}}; + EXPECT_EQ(absl::nullopt, config.rateKbps(&bad_headers)); // Header with zero. - Http::TestHeaderMapImpl zero_headers{{"x-envoy-fault-throughput-response", "0"}}; - EXPECT_EQ(absl::nullopt, - config.rateKbps(zero_headers.get(HeaderNames::get().ThroughputResponse))); + Http::TestRequestHeaderMapImpl zero_headers{{"x-envoy-fault-throughput-response", "0"}}; + EXPECT_EQ(absl::nullopt, config.rateKbps(&zero_headers)); // Valid header. - Http::TestHeaderMapImpl good_headers{{"x-envoy-fault-throughput-response", "123"}}; - EXPECT_EQ(123UL, - config.rateKbps(good_headers.get(HeaderNames::get().ThroughputResponse)).value()); + Http::TestRequestHeaderMapImpl good_headers{{"x-envoy-fault-throughput-response", "123"}}; + EXPECT_EQ(123UL, config.rateKbps(&good_headers).value()); +} + +TEST(FaultConfigTest, FaultRateLimitPercentageHeaderConfig) { + envoy::extensions::filters::common::fault::v3::FaultRateLimit proto_config; + proto_config.mutable_header_limit(); + proto_config.mutable_percentage()->set_numerator(80); + proto_config.mutable_percentage()->set_denominator(envoy::type::v3::FractionalPercent::MILLION); + FaultRateLimitConfig config(proto_config); + + // Header with bad data - fallback to proto config. + Http::TestRequestHeaderMapImpl bad_headers{ + {"x-envoy-fault-throughput-response-percentage", "abc"}}; + const auto bad_headers_percentage = config.percentage(&bad_headers); + EXPECT_EQ(proto_config.percentage().numerator(), bad_headers_percentage.numerator()); + EXPECT_EQ(proto_config.percentage().denominator(), bad_headers_percentage.denominator()); + + // Out of range header, value too low - fallback to proto config. + Http::TestRequestHeaderMapImpl too_low_headers{ + {"x-envoy-fault-throughput-response-percentage", "-1"}}; + const auto too_low_headers_percentage = config.percentage(&too_low_headers); + EXPECT_EQ(proto_config.percentage().numerator(), too_low_headers_percentage.numerator()); + EXPECT_EQ(proto_config.percentage().denominator(), too_low_headers_percentage.denominator()); + + // Valid header with value greater than the value of the numerator of default percentage - use + // proto config. + Http::TestRequestHeaderMapImpl good_headers{ + {"x-envoy-fault-throughput-response-percentage", "90"}}; + const auto good_headers_percentage = config.percentage(&good_headers); + EXPECT_EQ(proto_config.percentage().numerator(), good_headers_percentage.numerator()); + EXPECT_EQ(proto_config.percentage().denominator(), good_headers_percentage.denominator()); + + // Valid header with value lesser than the value of the numerator of default percentage. + Http::TestRequestHeaderMapImpl greater_numerator_headers{ + {"x-envoy-fault-throughput-response-percentage", "60"}}; + const auto greater_numerator_headers_percentage = config.percentage(&greater_numerator_headers); + EXPECT_EQ(60, greater_numerator_headers_percentage.numerator()); + EXPECT_EQ(proto_config.percentage().denominator(), + greater_numerator_headers_percentage.denominator()); } } // namespace diff --git a/test/extensions/filters/http/common/fuzz/filter_corpus/adminnullptr b/test/extensions/filters/http/common/fuzz/filter_corpus/adminnullptr new file mode 100644 index 000000000000..361841a4fc12 --- /dev/null +++ b/test/extensions/filters/http/common/fuzz/filter_corpus/adminnullptr @@ -0,0 +1,7 @@ +config { + name: "envoy.filters.http.tap" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.http.tap.v3.Tap" + value: "\n\036\022\034\n\022\022\020\n\n\022\010\n\002B\000\n\002B\000\n\002:\000\022\006\n\004\010\001\022\000" + } +} \ No newline at end of file diff --git a/test/extensions/filters/http/common/fuzz/filter_corpus/valid_jwt b/test/extensions/filters/http/common/fuzz/filter_corpus/valid_jwt new file mode 100644 index 000000000000..cf3e4b5fa0ea --- /dev/null +++ b/test/extensions/filters/http/common/fuzz/filter_corpus/valid_jwt @@ -0,0 +1,7 @@ +config { + name: "envoy.filters.http.jwt_authn" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.http.jwt_authn.v3.JwtAuthentication" + value: "\n#\n\000\022\037\n\001h\032\000B\025\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000J\001h\022\205\001\n\177\022}TTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTT\022\002\022\000" + } +} \ No newline at end of file diff --git a/test/extensions/filters/http/common/fuzz/uber_filter.h b/test/extensions/filters/http/common/fuzz/uber_filter.h index 2f97166f60a7..2df4483a4a61 100644 --- a/test/extensions/filters/http/common/fuzz/uber_filter.h +++ b/test/extensions/filters/http/common/fuzz/uber_filter.h @@ -55,6 +55,7 @@ class UberFilterFuzzer { } void prepareTap() { + ON_CALL(factory_context_, admin()).WillByDefault(testing::ReturnRef(factory_context_.admin_)); ON_CALL(factory_context_.admin_, addHandler(_, _, _, _, _)) .WillByDefault(testing::Return(true)); ON_CALL(factory_context_.admin_, removeHandler(_)).WillByDefault(testing::Return(true)); diff --git a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc index 07ccea3a5b6b..dc607359ca09 100644 --- a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc +++ b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc @@ -7,6 +7,7 @@ #include "extensions/transport_sockets/tls/ssl_socket.h" #include "test/integration/http_integration.h" +#include "test/integration/ssl_utility.h" namespace Envoy { namespace { @@ -17,22 +18,11 @@ class ProxyFilterIntegrationTest : public testing::TestWithParamadd_tls_certificates(); - tls_cert->mutable_certificate_chain()->set_filename(TestEnvironment::runfilesPath( - fmt::format("test/config/integration/certs/{}cert.pem", upstream_cert_name_))); - tls_cert->mutable_private_key()->set_filename(TestEnvironment::runfilesPath( - fmt::format("test/config/integration/certs/{}key.pem", upstream_cert_name_))); - - auto cfg = std::make_unique( - tls_context, factory_context_); - - static Stats::Scope* upstream_stats_store = new Stats::IsolatedStoreImpl(); - return std::make_unique( - std::move(cfg), context_manager_, *upstream_stats_store, std::vector{}); - } - bool upstream_tls_{}; std::string upstream_cert_name_{"upstreamlocalhost"}; CdsHelper cds_helper_; diff --git a/test/extensions/filters/http/fault/fault_filter_integration_test.cc b/test/extensions/filters/http/fault/fault_filter_integration_test.cc index affab35fbeb1..6059287a66c7 100644 --- a/test/extensions/filters/http/fault/fault_filter_integration_test.cc +++ b/test/extensions/filters/http/fault/fault_filter_integration_test.cc @@ -154,6 +154,54 @@ TEST_P(FaultIntegrationTestAllProtocols, HeaderFaultAbortConfig) { EXPECT_EQ(0UL, test_server_->counter("http.config_test.fault.response_rl_injected")->value()); } +// Request faults controlled via header configuration. +TEST_P(FaultIntegrationTestAllProtocols, HeaderFaultsConfig0PercentageHeaders) { + initializeFilter(header_fault_config_); + codec_client_ = makeHttpConnection(makeClientConnection(lookupPort("http"))); + + auto response = codec_client_->makeHeaderOnlyRequest( + Http::TestRequestHeaderMapImpl{{":method", "GET"}, + {":path", "/test/long/url"}, + {":scheme", "http"}, + {":authority", "host"}, + {"x-envoy-fault-abort-request", "429"}, + {"x-envoy-fault-abort-request-percentage", "0"}, + {"x-envoy-fault-delay-request", "100"}, + {"x-envoy-fault-delay-request-percentage", "0"}, + {"x-envoy-fault-throughput-response", "100"}, + {"x-envoy-fault-throughput-response-percentage", "0"}}); + waitForNextUpstreamRequest(); + upstream_request_->encodeHeaders(default_response_headers_, true); + response->waitForEndStream(); + + EXPECT_EQ(0UL, test_server_->counter("http.config_test.fault.aborts_injected")->value()); + EXPECT_EQ(0UL, test_server_->counter("http.config_test.fault.delays_injected")->value()); + EXPECT_EQ(0UL, test_server_->counter("http.config_test.fault.response_rl_injected")->value()); +} + +// Request faults controlled via header configuration. +TEST_P(FaultIntegrationTestAllProtocols, HeaderFaultsConfig100PercentageHeaders) { + initializeFilter(header_fault_config_); + codec_client_ = makeHttpConnection(makeClientConnection(lookupPort("http"))); + + auto response = codec_client_->makeHeaderOnlyRequest( + Http::TestRequestHeaderMapImpl{{":method", "GET"}, + {":path", "/test/long/url"}, + {":scheme", "http"}, + {":authority", "host"}, + {"x-envoy-fault-delay-request", "100"}, + {"x-envoy-fault-delay-request-percentage", "100"}, + {"x-envoy-fault-throughput-response", "100"}, + {"x-envoy-fault-throughput-response-percentage", "100"}}); + waitForNextUpstreamRequest(); + upstream_request_->encodeHeaders(default_response_headers_, true); + response->waitForEndStream(); + + EXPECT_EQ(0UL, test_server_->counter("http.config_test.fault.aborts_injected")->value()); + EXPECT_EQ(1UL, test_server_->counter("http.config_test.fault.delays_injected")->value()); + EXPECT_EQ(1UL, test_server_->counter("http.config_test.fault.response_rl_injected")->value()); +} + // Header configuration with no headers, so no fault injection. TEST_P(FaultIntegrationTestAllProtocols, HeaderFaultConfigNoHeaders) { initializeFilter(header_fault_config_); diff --git a/test/extensions/filters/http/grpc_json_transcoder/grpc_json_transcoder_integration_test.cc b/test/extensions/filters/http/grpc_json_transcoder/grpc_json_transcoder_integration_test.cc index f396b6704584..3760cbb6cb41 100644 --- a/test/extensions/filters/http/grpc_json_transcoder/grpc_json_transcoder_integration_test.cc +++ b/test/extensions/filters/http/grpc_json_transcoder/grpc_json_transcoder_integration_test.cc @@ -327,6 +327,20 @@ TEST_P(GrpcJsonTranscoderIntegrationTest, UnaryGetHttpBody) { R"(

Hello!

)"); } +TEST_P(GrpcJsonTranscoderIntegrationTest, StreamGetHttpBody) { + HttpIntegrationTest::initialize(); + + testTranscoding( + Http::TestRequestHeaderMapImpl{ + {":method", "GET"}, {":path", "/indexStream"}, {":authority", "host"}}, + "", {""}, + {R"(content_type: "text/html" data: "

Hello!

")", + R"(content_type: "text/plain" data: "Hello!")"}, + Status(), Http::TestResponseHeaderMapImpl{{":status", "200"}, {"content-type", "text/html"}}, + R"(

Hello!

)" + R"(Hello!)"); +} + TEST_P(GrpcJsonTranscoderIntegrationTest, UnaryEchoHttpBody) { HttpIntegrationTest::initialize(); testTranscoding( diff --git a/test/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter_test.cc b/test/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter_test.cc index 74513d9c0c7d..2170a79cf38e 100644 --- a/test/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter_test.cc +++ b/test/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter_test.cc @@ -847,6 +847,47 @@ TEST_F(GrpcJsonTranscoderFilterTest, TranscodingStreamPostWithHttpBody) { EXPECT_THAT(request, ProtoEq(expected_request)); } +TEST_F(GrpcJsonTranscoderFilterTest, TranscodingStreamWithHttpBodyAsOutput) { + Http::TestRequestHeaderMapImpl request_headers{{":method", "GET"}, {":path", "/indexStream"}}; + + EXPECT_CALL(decoder_callbacks_, clearRouteCache()); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers, false)); + EXPECT_EQ("application/grpc", request_headers.get_("content-type")); + EXPECT_EQ("/indexStream", request_headers.get_("x-envoy-original-path")); + EXPECT_EQ("/bookstore.Bookstore/GetIndexStream", request_headers.get_(":path")); + EXPECT_EQ("trailers", request_headers.get_("te")); + + Http::TestResponseHeaderMapImpl response_headers{{"content-type", "application/grpc"}, + {":status", "200"}}; + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, + filter_.encodeHeaders(response_headers, false)); + + // "Send" 1st gRPC message + google::api::HttpBody response; + response.set_content_type("text/html"); + response.set_data("

Message 1!

"); + auto response_data = Grpc::Common::serializeToGrpcFrame(response); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_.encodeData(*response_data, false)); + // Content type set to HttpBody.content_type / no content-length + EXPECT_EQ("text/html", response_headers.get_("content-type")); + EXPECT_EQ(nullptr, response_headers.ContentLength()); + EXPECT_EQ(response.data(), response_data->toString()); + + // "Send" 2nd message with different context type + response.set_content_type("text/plain"); + response.set_data("Message2"); + response_data = Grpc::Common::serializeToGrpcFrame(response); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_.encodeData(*response_data, false)); + // Content type unchanged + EXPECT_EQ("text/html", response_headers.get_("content-type")); + EXPECT_EQ(nullptr, response_headers.ContentLength()); + EXPECT_EQ(response.data(), response_data->toString()); + + Http::TestRequestTrailerMapImpl request_trailers; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_.decodeTrailers(request_trailers)); +} + class GrpcJsonTranscoderFilterGrpcStatusTest : public GrpcJsonTranscoderFilterTest { public: GrpcJsonTranscoderFilterGrpcStatusTest( diff --git a/test/extensions/filters/http/grpc_stats/config_test.cc b/test/extensions/filters/http/grpc_stats/config_test.cc index c2f753edf088..532bfaba752a 100644 --- a/test/extensions/filters/http/grpc_stats/config_test.cc +++ b/test/extensions/filters/http/grpc_stats/config_test.cc @@ -14,6 +14,7 @@ #include "gtest/gtest.h" using testing::_; +using testing::Property; using testing::Return; namespace Envoy { @@ -34,6 +35,10 @@ class GrpcStatsFilterConfigTest : public testing::Test { cb(filter_callback); ON_CALL(decoder_callbacks_, streamInfo()).WillByDefault(testing::ReturnRef(stream_info_)); + + ON_CALL(*decoder_callbacks_.cluster_info_, statsScope()) + .WillByDefault(testing::ReturnRef(stats_store_)); + filter_->setDecoderFilterCallbacks(decoder_callbacks_); } @@ -61,7 +66,8 @@ class GrpcStatsFilterConfigTest : public testing::Test { NiceMock context_; std::shared_ptr filter_; NiceMock decoder_callbacks_; - TestStreamInfo stream_info_; + NiceMock stream_info_; + NiceMock stats_store_; }; TEST_F(GrpcStatsFilterConfigTest, StatsHttp2HeaderOnlyResponse) { @@ -404,6 +410,32 @@ TEST_F(GrpcStatsFilterConfigTest, MessageCounts) { EXPECT_EQ(3U, filter_object.response_message_count()); } +TEST_F(GrpcStatsFilterConfigTest, UpstreamStats) { + config_.mutable_stats_for_all_methods()->set_value(true); + config_.set_emit_filter_state(true); + config_.set_enable_upstream_stats(true); + initialize(); + + Http::TestRequestHeaderMapImpl request_headers{ + {"content-type", "application/grpc+proto"}, + {":path", "/lyft.users.BadCompanions/GetBadCompanions"}}; + + ON_CALL(stream_info_, lastUpstreamRxByteReceived()) + .WillByDefault(testing::Return( + absl::optional(std::chrono::nanoseconds(30000000)))); + ON_CALL(stream_info_, lastUpstreamTxByteSent()) + .WillByDefault(testing::Return( + absl::optional(std::chrono::nanoseconds(20000000)))); + + EXPECT_CALL(stats_store_, + deliverHistogramToSinks( + Property(&Stats::Metric::name, + "grpc.lyft.users.BadCompanions.GetBadCompanions.upstream_rq_time"), + 10ul)); + + doRequestResponse(request_headers); +} + } // namespace } // namespace GrpcStats } // namespace HttpFilters diff --git a/test/extensions/filters/http/header_to_metadata/config_test.cc b/test/extensions/filters/http/header_to_metadata/config_test.cc index 86e0fb676101..861e4ee545a7 100644 --- a/test/extensions/filters/http/header_to_metadata/config_test.cc +++ b/test/extensions/filters/http/header_to_metadata/config_test.cc @@ -4,6 +4,7 @@ #include "envoy/extensions/filters/http/header_to_metadata/v3/header_to_metadata.pb.validate.h" #include "extensions/filters/http/header_to_metadata/config.h" +#include "extensions/filters/http/header_to_metadata/header_to_metadata_filter.h" #include "test/mocks/server/mocks.h" #include "test/test_common/utility.h" @@ -70,6 +71,34 @@ TEST(HeaderToMetadataFilterConfigTest, SimpleConfig) { cb(filter_callbacks); } +TEST(HeaderToMetadataFilterConfigTest, PerRouteConfig) { + const std::string yaml = R"EOF( +request_rules: + - header: x-version + on_header_present: + metadata_namespace: envoy.lb + key: version + type: STRING + on_header_missing: + metadata_namespace: envoy.lb + key: default + value: 'true' + type: STRING + )EOF"; + + HeaderToMetadataProtoConfig proto_config; + TestUtility::loadFromYamlAndValidate(yaml, proto_config); + + testing::NiceMock context; + HeaderToMetadataConfig factory; + + const auto route_config = factory.createRouteSpecificFilterConfig( + proto_config, context, ProtobufMessage::getNullValidationVisitor()); + const auto* config = dynamic_cast(route_config.get()); + EXPECT_TRUE(config->doRequest()); + EXPECT_FALSE(config->doResponse()); +} + } // namespace HeaderToMetadataFilter } // namespace HttpFilters } // namespace Extensions diff --git a/test/extensions/filters/http/header_to_metadata/header_to_metadata_filter_test.cc b/test/extensions/filters/http/header_to_metadata/header_to_metadata_filter_test.cc index 2dc2c3ec845e..ad3ee7af60ed 100644 --- a/test/extensions/filters/http/header_to_metadata/header_to_metadata_filter_test.cc +++ b/test/extensions/filters/http/header_to_metadata/header_to_metadata_filter_test.cc @@ -7,6 +7,7 @@ #include "common/protobuf/protobuf.h" #include "extensions/filters/http/header_to_metadata/header_to_metadata_filter.h" +#include "extensions/filters/http/well_known_names.h" #include "test/mocks/http/mocks.h" #include "test/mocks/stream_info/mocks.h" @@ -17,6 +18,7 @@ using testing::_; using testing::NiceMock; +using testing::Return; namespace Envoy { namespace Extensions { @@ -24,6 +26,35 @@ namespace HttpFilters { namespace HeaderToMetadataFilter { namespace { +MATCHER_P(MapEq, rhs, "") { + const ProtobufWkt::Struct& obj = arg; + EXPECT_TRUE(!rhs.empty()); + for (auto const& entry : rhs) { + EXPECT_EQ(obj.fields().at(entry.first).string_value(), entry.second); + } + return true; +} + +MATCHER_P(MapEqNum, rhs, "") { + const ProtobufWkt::Struct& obj = arg; + EXPECT_TRUE(!rhs.empty()); + for (auto const& entry : rhs) { + EXPECT_EQ(obj.fields().at(entry.first).number_value(), entry.second); + } + return true; +} + +MATCHER_P(MapEqValue, rhs, "") { + const ProtobufWkt::Struct& obj = arg; + EXPECT_TRUE(!rhs.empty()); + for (auto const& entry : rhs) { + EXPECT_TRUE(TestUtility::protoEqual(obj.fields().at(entry.first), entry.second)); + } + return true; +} + +} // namespace + class HeaderToMetadataTest : public testing::Test { public: const std::string request_config_yaml = R"EOF( @@ -49,6 +80,8 @@ class HeaderToMetadataTest : public testing::Test { filter_->setEncoderFilterCallbacks(encoder_callbacks_); } + const Config* getConfig() { return filter_->getConfig(); } + ConfigSharedPtr config_; std::shared_ptr filter_; NiceMock decoder_callbacks_; @@ -56,33 +89,6 @@ class HeaderToMetadataTest : public testing::Test { NiceMock req_info_; }; -MATCHER_P(MapEq, rhs, "") { - const ProtobufWkt::Struct& obj = arg; - EXPECT_TRUE(!rhs.empty()); - for (auto const& entry : rhs) { - EXPECT_EQ(obj.fields().at(entry.first).string_value(), entry.second); - } - return true; -} - -MATCHER_P(MapEqNum, rhs, "") { - const ProtobufWkt::Struct& obj = arg; - EXPECT_TRUE(!rhs.empty()); - for (auto const& entry : rhs) { - EXPECT_EQ(obj.fields().at(entry.first).number_value(), entry.second); - } - return true; -} - -MATCHER_P(MapEqValue, rhs, "") { - const ProtobufWkt::Struct& obj = arg; - EXPECT_TRUE(!rhs.empty()); - for (auto const& entry : rhs) { - EXPECT_TRUE(TestUtility::protoEqual(obj.fields().at(entry.first), entry.second)); - } - return true; -} - /** * Basic use-case. */ @@ -103,6 +109,50 @@ TEST_F(HeaderToMetadataTest, BasicRequestTest) { filter_->onDestroy(); } +TEST_F(HeaderToMetadataTest, PerRouteOverride) { + // Global config is empty. + initializeFilter("{}"); + Http::TestRequestHeaderMapImpl incoming_headers{{"X-VERSION", "0xdeadbeef"}}; + std::map expected = {{"version", "0xdeadbeef"}}; + + // Setup per route config. + envoy::extensions::filters::http::header_to_metadata::v3::Config config_proto; + TestUtility::loadFromYaml(request_config_yaml, config_proto); + Config per_route_config(config_proto, true); + EXPECT_CALL(decoder_callbacks_.route_->route_entry_.virtual_host_, + perFilterConfig(HttpFilterNames::get().HeaderToMetadata)) + .WillOnce(Return(&per_route_config)); + + EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); + EXPECT_CALL(req_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(incoming_headers, false)); + Http::MetadataMap metadata_map{{"metadata", "metadata"}}; + EXPECT_EQ(Http::FilterMetadataStatus::Continue, filter_->decodeMetadata(metadata_map)); + Buffer::OwnedImpl data("data"); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data, false)); + Http::TestRequestTrailerMapImpl incoming_trailers; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(incoming_trailers)); + filter_->onDestroy(); +} + +TEST_F(HeaderToMetadataTest, ConfigIsCached) { + // Global config is empty. + initializeFilter("{}"); + Http::TestRequestHeaderMapImpl incoming_headers{{"X-VERSION", "0xdeadbeef"}}; + std::map expected = {{"version", "0xdeadbeef"}}; + + // Setup per route config. + envoy::extensions::filters::http::header_to_metadata::v3::Config config_proto; + TestUtility::loadFromYaml(request_config_yaml, config_proto); + Config per_route_config(config_proto, true); + EXPECT_CALL(decoder_callbacks_.route_->route_entry_.virtual_host_, + perFilterConfig(HttpFilterNames::get().HeaderToMetadata)) + .WillOnce(Return(&per_route_config)); + + EXPECT_TRUE(getConfig()->doRequest()); + EXPECT_TRUE(getConfig()->doRequest()); +} + /** * X-version not set, the on missing value should be set. */ @@ -135,7 +185,7 @@ TEST_F(HeaderToMetadataTest, HeaderRemovedTest) { EXPECT_CALL(encoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); EXPECT_CALL(req_info_, - setDynamicMetadata("envoy.filters.http.header_to_metadata", MapEq(expected))); + setDynamicMetadata(HttpFilterNames::get().HeaderToMetadata, MapEq(expected))); Http::TestResponseHeaderMapImpl continue_response{{":status", "100"}}; EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encode100ContinueHeaders(continue_response)); @@ -167,7 +217,7 @@ TEST_F(HeaderToMetadataTest, NumberTypeTest) { EXPECT_CALL(encoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); EXPECT_CALL(req_info_, - setDynamicMetadata("envoy.filters.http.header_to_metadata", MapEqNum(expected))); + setDynamicMetadata(HttpFilterNames::get().HeaderToMetadata, MapEqNum(expected))); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(incoming_headers, false)); } @@ -192,7 +242,7 @@ TEST_F(HeaderToMetadataTest, StringTypeInBase64UrlTest) { EXPECT_CALL(encoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); EXPECT_CALL(req_info_, - setDynamicMetadata("envoy.filters.http.header_to_metadata", MapEq(expected))); + setDynamicMetadata(HttpFilterNames::get().HeaderToMetadata, MapEq(expected))); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(incoming_headers, false)); } @@ -229,7 +279,7 @@ TEST_F(HeaderToMetadataTest, ProtobufValueTypeInBase64UrlTest) { EXPECT_CALL(encoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); EXPECT_CALL(req_info_, - setDynamicMetadata("envoy.filters.http.header_to_metadata", MapEqValue(expected))); + setDynamicMetadata(HttpFilterNames::get().HeaderToMetadata, MapEqValue(expected))); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(incoming_headers, false)); } @@ -370,7 +420,7 @@ TEST_F(HeaderToMetadataTest, IgnoreHeaderValueUseConstant) { EXPECT_CALL(encoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); EXPECT_CALL(req_info_, - setDynamicMetadata("envoy.filters.http.header_to_metadata", MapEq(expected))); + setDynamicMetadata(HttpFilterNames::get().HeaderToMetadata, MapEq(expected))); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(incoming_headers, false)); EXPECT_EQ(empty_headers, incoming_headers); } @@ -388,6 +438,11 @@ TEST_F(HeaderToMetadataTest, RejectInvalidRule) { EXPECT_THROW_WITH_MESSAGE(initializeFilter(config), Envoy::EnvoyException, expected); } +TEST_F(HeaderToMetadataTest, PerRouteEmtpyRules) { + envoy::extensions::filters::http::header_to_metadata::v3::Config config_proto; + EXPECT_THROW(std::make_shared(config_proto, true), EnvoyException); +} + /** * Empty values not added to metadata. */ @@ -408,7 +463,6 @@ TEST_F(HeaderToMetadataTest, NoEmptyValues) { EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(headers, false)); } -} // namespace } // namespace HeaderToMetadataFilter } // namespace HttpFilters } // namespace Extensions diff --git a/test/extensions/filters/http/tap/BUILD b/test/extensions/filters/http/tap/BUILD index b352b03e4f20..b5aaea4b1480 100644 --- a/test/extensions/filters/http/tap/BUILD +++ b/test/extensions/filters/http/tap/BUILD @@ -26,8 +26,10 @@ envoy_extension_cc_test( extension_name = "envoy.filters.http.tap", deps = [ ":common", - "//source/extensions/filters/http/tap:tap_filter_lib", + "//source/extensions/filters/http/tap:config", + "//source/extensions/filters/http/tap:tap_config_interface", "//test/mocks/http:http_mocks", + "//test/mocks/server:server_mocks", "//test/mocks/stream_info:stream_info_mocks", "//test/test_common:utility_lib", ], diff --git a/test/extensions/filters/http/tap/tap_filter_test.cc b/test/extensions/filters/http/tap/tap_filter_test.cc index a183d6523acc..1f305ddbe79e 100644 --- a/test/extensions/filters/http/tap/tap_filter_test.cc +++ b/test/extensions/filters/http/tap/tap_filter_test.cc @@ -1,7 +1,9 @@ +#include "extensions/filters/http/tap/config.h" #include "extensions/filters/http/tap/tap_filter.h" #include "test/extensions/filters/http/tap/common.h" #include "test/mocks/http/mocks.h" +#include "test/mocks/server/mocks.h" #include "test/mocks/stream_info/mocks.h" #include "test/test_common/utility.h" @@ -126,6 +128,28 @@ TEST_F(TapFilterTest, Config) { EXPECT_TRUE(testing::Mock::VerifyAndClearExpectations(http_tap_config_.get())); } +TEST(TapFilterConfigTest, InvalidProto) { + const std::string filter_config = + R"EOF( + common_config: + static_config: + match_config: + any_match: true + output_config: + sinks: + - format: JSON_BODY_AS_STRING + streaming_admin: {} +)EOF"; + + envoy::extensions::filters::http::tap::v3::Tap config; + TestUtility::loadFromYaml(filter_config, config); + NiceMock context; + TapFilterFactory factory; + EXPECT_THROW_WITH_MESSAGE(factory.createFilterFactoryFromProto(config, "stats", context), + EnvoyException, + "Error: Specifying admin streaming output without configuring admin."); +} + } // namespace } // namespace TapFilter } // namespace HttpFilters diff --git a/test/extensions/filters/listener/proxy_protocol/proxy_protocol_test.cc b/test/extensions/filters/listener/proxy_protocol/proxy_protocol_test.cc index d3f825828b53..74dc142304e2 100644 --- a/test/extensions/filters/listener/proxy_protocol/proxy_protocol_test.cc +++ b/test/extensions/filters/listener/proxy_protocol/proxy_protocol_test.cc @@ -64,7 +64,7 @@ class ProxyProtocolTest : public testing::TestWithParamlocalAddress())); EXPECT_CALL(socket_factory_, getListenSocket()).WillOnce(Return(socket_)); - connection_handler_->addListener(*this); + connection_handler_->addListener(absl::nullopt, *this); conn_ = dispatcher_->createClientConnection(socket_->localAddress(), Network::Address::InstanceConstSharedPtr(), Network::Test::createRawBufferSocket(), nullptr); @@ -1005,7 +1005,7 @@ class WildcardProxyProtocolTest : public testing::TestWithParamlocalAddress())); EXPECT_CALL(socket_factory_, getListenSocket()).WillOnce(Return(socket_)); - connection_handler_->addListener(*this); + connection_handler_->addListener(absl::nullopt, *this); conn_ = dispatcher_->createClientConnection(local_dst_address_, Network::Address::InstanceConstSharedPtr(), Network::Test::createRawBufferSocket(), nullptr); diff --git a/test/extensions/filters/network/http_connection_manager/BUILD b/test/extensions/filters/network/http_connection_manager/BUILD index cf178eed588b..0b8e125be0c5 100644 --- a/test/extensions/filters/network/http_connection_manager/BUILD +++ b/test/extensions/filters/network/http_connection_manager/BUILD @@ -26,16 +26,17 @@ envoy_extension_cc_test( "//source/common/buffer:buffer_lib", "//source/common/event:dispatcher_lib", "//source/extensions/access_loggers/file:config", - "//source/extensions/filters/http/dynamo:config", "//source/extensions/filters/http/health_check:config", "//source/extensions/filters/http/ratelimit:config", "//source/extensions/filters/http/router:config", "//source/extensions/filters/network/http_connection_manager:config", + "//test/integration/filters:encoder_decoder_buffer_filter_lib", "//test/mocks/network:network_mocks", "//test/mocks/server:server_mocks", "//test/test_common:registry_lib", "//test/test_common:utility_lib", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", + "@envoy_api//envoy/config/trace/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/network/http_connection_manager/v3:pkg_cc_proto", "@envoy_api//envoy/type/v3:pkg_cc_proto", ], diff --git a/test/extensions/filters/network/http_connection_manager/config_test.cc b/test/extensions/filters/network/http_connection_manager/config_test.cc index 3f6e904f8e93..014459f6d73e 100644 --- a/test/extensions/filters/network/http_connection_manager/config_test.cc +++ b/test/extensions/filters/network/http_connection_manager/config_test.cc @@ -1,4 +1,7 @@ #include "envoy/config/core/v3/base.pb.h" +#include "envoy/config/trace/v3/http_tracer.pb.h" +#include "envoy/config/trace/v3/opencensus.pb.h" +#include "envoy/config/trace/v3/zipkin.pb.h" #include "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.pb.h" #include "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.pb.validate.h" #include "envoy/server/request_id_extension_config.h" @@ -1092,7 +1095,7 @@ stat_prefix: router route: cluster: cluster http_filters: -- name: envoy.filters.http.dynamo +- name: encoder-decoder-buffer-filter typed_config: "@type": type.googleapis.com/google.protobuf.Empty - name: envoy.filters.http.router @@ -1143,7 +1146,7 @@ stat_prefix: my_stat_prefix route: cluster: fake_cluster http_filters: -- name: envoy.filters.http.dynamo +- name: encoder-decoder-buffer-filter config: {} access_log: - name: accesslog @@ -1172,7 +1175,7 @@ stat_prefix: my_stat_prefix route: cluster: fake_cluster http_filters: -- name: envoy.filters.http.dynamo +- name: encoder-decoder-buffer-filter typed_config: {} access_log: - name: accesslog @@ -1202,7 +1205,7 @@ stat_prefix: my_stat_prefix route: cluster: fake_cluster http_filters: -- name: envoy.filters.http.dynamo +- name: encoder-decoder-buffer-filter typed_config: {} access_log: - name: accesslog @@ -1273,7 +1276,7 @@ stat_prefix: my_stat_prefix route: cluster: fake_cluster http_filters: -- name: envoy.http_dynamo_filter +- name: encoder-decoder-buffer-filter typed_config: {} http2_protocol_options: hpack_table_size: 2048 @@ -1363,7 +1366,7 @@ stat_prefix: my_stat_prefix route: cluster: fake_cluster http_filters: -- name: envoy.http_dynamo_filter +- name: encoder-decoder-buffer-filter typed_config: {} http2_protocol_options: custom_settings_parameters: { identifier: 2, value: 1 } @@ -1391,7 +1394,7 @@ stat_prefix: my_stat_prefix route: cluster: fake_cluster http_filters: -- name: envoy.http_dynamo_filter +- name: encoder-decoder-buffer-filter typed_config: {} http2_protocol_options: hpack_table_size: 2048 @@ -1577,7 +1580,7 @@ stat_prefix: router route: cluster: cluster http_filters: -- name: envoy.filters.http.dynamo +- name: encoder-decoder-buffer-filter - name: envoy.filters.http.router )EOF"; @@ -1589,7 +1592,7 @@ TEST_F(FilterChainTest, CreateFilterChain) { scoped_routes_config_provider_manager_, http_tracer_manager_); Http::MockFilterChainFactoryCallbacks callbacks; - EXPECT_CALL(callbacks, addStreamFilter(_)); // Dynamo + EXPECT_CALL(callbacks, addStreamFilter(_)); // Buffer EXPECT_CALL(callbacks, addStreamDecoderFilter(_)); // Router config.createFilterChain(callbacks); } @@ -1608,7 +1611,7 @@ TEST_F(FilterChainTest, CreateUpgradeFilterChain) { // config is present. We should create an upgrade filter chain for // WebSockets. { - EXPECT_CALL(callbacks, addStreamFilter(_)); // Dynamo + EXPECT_CALL(callbacks, addStreamFilter(_)); // Buffer EXPECT_CALL(callbacks, addStreamDecoderFilter(_)); // Router EXPECT_TRUE(config.createUpgradeFilterChain("WEBSOCKET", nullptr, callbacks)); } @@ -1632,7 +1635,7 @@ TEST_F(FilterChainTest, CreateUpgradeFilterChain) { // For paranoia's sake make sure route-specific enabling doesn't break // anything. { - EXPECT_CALL(callbacks, addStreamFilter(_)); // Dynamo + EXPECT_CALL(callbacks, addStreamFilter(_)); // Buffer EXPECT_CALL(callbacks, addStreamDecoderFilter(_)); // Router std::map upgrade_map; upgrade_map.emplace(std::make_pair("WebSocket", true)); @@ -1691,11 +1694,11 @@ TEST_F(FilterChainTest, CreateCustomUpgradeFilterChain) { auto foo_config = hcm_config.add_upgrade_configs(); foo_config->set_upgrade_type("foo"); foo_config->add_filters()->ParseFromString("\n" - "\x19" - "envoy.filters.http.dynamo"); + "\x1D" + "encoder-decoder-buffer-filter"); foo_config->add_filters()->ParseFromString("\n" - "\x19" - "envoy.filters.http.dynamo"); + "\x1D" + "encoder-decoder-buffer-filter"); foo_config->add_filters()->ParseFromString("\n" "\x19" "envoy.filters.http.router"); @@ -1706,7 +1709,7 @@ TEST_F(FilterChainTest, CreateCustomUpgradeFilterChain) { { Http::MockFilterChainFactoryCallbacks callbacks; - EXPECT_CALL(callbacks, addStreamFilter(_)); // Dynamo + EXPECT_CALL(callbacks, addStreamFilter(_)); // Buffer EXPECT_CALL(callbacks, addStreamDecoderFilter(_)); // Router config.createFilterChain(callbacks); } @@ -1720,7 +1723,7 @@ TEST_F(FilterChainTest, CreateCustomUpgradeFilterChain) { { Http::MockFilterChainFactoryCallbacks callbacks; EXPECT_CALL(callbacks, addStreamDecoderFilter(_)).Times(1); - EXPECT_CALL(callbacks, addStreamFilter(_)).Times(2); // Dynamo + EXPECT_CALL(callbacks, addStreamFilter(_)).Times(2); // Buffer EXPECT_TRUE(config.createUpgradeFilterChain("Foo", nullptr, callbacks)); } } @@ -1740,8 +1743,8 @@ TEST_F(FilterChainTest, CreateCustomUpgradeFilterChainWithRouterNotLast) { "\x19" "envoy.filters.http.router"); foo_config->add_filters()->ParseFromString("\n" - "\x19" - "envoy.filters.http.dynamo"); + "\x1D" + "encoder-decoder-buffer-filter"); EXPECT_THROW_WITH_MESSAGE( HttpConnectionManagerConfig(hcm_config, context_, date_provider_, diff --git a/test/extensions/filters/network/sni_dynamic_forward_proxy/BUILD b/test/extensions/filters/network/sni_dynamic_forward_proxy/BUILD new file mode 100644 index 000000000000..a931923403d2 --- /dev/null +++ b/test/extensions/filters/network/sni_dynamic_forward_proxy/BUILD @@ -0,0 +1,46 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_package", +) +load( + "//test/extensions:extensions_build_system.bzl", + "envoy_extension_cc_test", +) + +envoy_package() + +envoy_extension_cc_test( + name = "proxy_filter_test", + srcs = ["proxy_filter_test.cc"], + extension_name = "envoy.filters.network.sni_dynamic_forward_proxy", + deps = [ + "//source/extensions/filters/network:well_known_names", + "//source/extensions/filters/network/sni_dynamic_forward_proxy:config", + "//test/extensions/common/dynamic_forward_proxy:mocks", + "//test/mocks/http:http_mocks", + "//test/mocks/upstream:upstream_mocks", + "@envoy_api//envoy/extensions/filters/network/sni_dynamic_forward_proxy/v3alpha:pkg_cc_proto", + ], +) + +envoy_extension_cc_test( + name = "proxy_filter_integration_test", + srcs = ["proxy_filter_integration_test.cc"], + data = [ + "//test/config/integration/certs", + ], + extension_name = "envoy.filters.network.sni_dynamic_forward_proxy", + deps = [ + "//source/extensions/clusters/dynamic_forward_proxy:cluster", + "//source/extensions/filters/listener/tls_inspector:config", + "//source/extensions/filters/network/sni_dynamic_forward_proxy:config", + "//source/extensions/filters/network/tcp_proxy:config", + "//test/integration:http_integration_lib", + "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", + "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/filters/network/http_connection_manager/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/transport_sockets/tls/v3:pkg_cc_proto", + ], +) diff --git a/test/extensions/filters/network/sni_dynamic_forward_proxy/proxy_filter_integration_test.cc b/test/extensions/filters/network/sni_dynamic_forward_proxy/proxy_filter_integration_test.cc new file mode 100644 index 000000000000..53ff3c2fd6f2 --- /dev/null +++ b/test/extensions/filters/network/sni_dynamic_forward_proxy/proxy_filter_integration_test.cc @@ -0,0 +1,133 @@ +#include "envoy/config/bootstrap/v3/bootstrap.pb.h" +#include "envoy/config/cluster/v3/cluster.pb.h" +#include "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.pb.h" +#include "envoy/extensions/transport_sockets/tls/v3/cert.pb.h" + +#include "extensions/transport_sockets/tls/context_config_impl.h" +#include "extensions/transport_sockets/tls/ssl_socket.h" + +#include "test/integration/http_integration.h" +#include "test/integration/ssl_utility.h" + +namespace Envoy { +namespace { + +class SniDynamicProxyFilterIntegrationTest + : public testing::TestWithParam, + public Event::TestUsingSimulatedTime, + public HttpIntegrationTest { +public: + // This test is using HTTP integration test to use the utilities to pass SNI from downstream + // to upstream. The config being tested is tcp_proxy. + SniDynamicProxyFilterIntegrationTest() + : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, GetParam(), + ConfigHelper::tcpProxyConfig()) {} + + void setup(uint64_t max_hosts = 1024) { + setUpstreamProtocol(FakeHttpConnection::Type::HTTP1); + + config_helper_.addListenerFilter(ConfigHelper::tlsInspectorFilter()); + + config_helper_.addConfigModifier([this, max_hosts]( + envoy::config::bootstrap::v3::Bootstrap& bootstrap) { + // Switch predefined cluster_0 to CDS filesystem sourcing. + bootstrap.mutable_dynamic_resources()->mutable_cds_config()->set_path(cds_helper_.cds_path()); + bootstrap.mutable_static_resources()->clear_clusters(); + + const std::string filter = + fmt::format(R"EOF( +name: envoy.filters.http.dynamic_forward_proxy +typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.sni_dynamic_forward_proxy.v3alpha.FilterConfig + dns_cache_config: + name: foo + dns_lookup_family: {} + max_hosts: {} + port_value: {} +)EOF", + Network::Test::ipVersionToDnsFamily(GetParam()), max_hosts, + fake_upstreams_[0]->localAddress()->ip()->port()); + config_helper_.addNetworkFilter(filter); + }); + + // Setup the initial CDS cluster. + cluster_.mutable_connect_timeout()->CopyFrom( + Protobuf::util::TimeUtil::MillisecondsToDuration(100)); + cluster_.set_name("cluster_0"); + cluster_.set_lb_policy(envoy::config::cluster::v3::Cluster::CLUSTER_PROVIDED); + + const std::string cluster_type_config = + fmt::format(R"EOF( +name: envoy.clusters.dynamic_forward_proxy +typed_config: + "@type": type.googleapis.com/envoy.extensions.clusters.dynamic_forward_proxy.v3.ClusterConfig + dns_cache_config: + name: foo + dns_lookup_family: {} + max_hosts: {} +)EOF", + Network::Test::ipVersionToDnsFamily(GetParam()), max_hosts); + + TestUtility::loadFromYaml(cluster_type_config, *cluster_.mutable_cluster_type()); + + // Load the CDS cluster and wait for it to initialize. + cds_helper_.setCds({cluster_}); + HttpIntegrationTest::initialize(); + test_server_->waitForCounterEq("cluster_manager.cluster_added", 1); + test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 0); + } + + void createUpstreams() override { + fake_upstreams_.emplace_back(new FakeUpstream( + Ssl::createFakeUpstreamSslContext(upstream_cert_name_, context_manager_, factory_context_), + 0, FakeHttpConnection::Type::HTTP1, version_, timeSystem())); + } + + Network::ClientConnectionPtr + makeSslClientConnection(const Ssl::ClientSslTransportOptions& options) { + + Network::Address::InstanceConstSharedPtr address = + Ssl::getSslAddress(version_, lookupPort("http")); + auto client_transport_socket_factory_ptr = + Ssl::createClientSslTransportSocketFactory(options, context_manager_, *api_); + return dispatcher_->createClientConnection( + address, Network::Address::InstanceConstSharedPtr(), + client_transport_socket_factory_ptr->createTransportSocket({}), nullptr); + } + + std::string upstream_cert_name_{"server"}; + CdsHelper cds_helper_; + envoy::config::cluster::v3::Cluster cluster_; +}; + +INSTANTIATE_TEST_SUITE_P(IpVersions, SniDynamicProxyFilterIntegrationTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), + TestUtility::ipTestParamsToString); + +// Verify that upstream TLS works with auto verification for SAN as well as auto setting SNI. +TEST_P(SniDynamicProxyFilterIntegrationTest, UpstreamTls) { + setup(); + fake_upstreams_[0]->setReadDisableOnNewConnection(false); + + codec_client_ = makeHttpConnection( + makeSslClientConnection(Ssl::ClientSslTransportOptions().setSni("localhost"))); + ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection( + *dispatcher_, fake_upstream_connection_, TestUtility::DefaultTimeout, max_request_headers_kb_, + max_request_headers_count_)); + + const Http::TestRequestHeaderMapImpl request_headers{ + {":method", "POST"}, + {":path", "/test/long/url"}, + {":scheme", "http"}, + {":authority", + fmt::format("localhost:{}", fake_upstreams_[0]->localAddress()->ip()->port())}}; + + auto response = codec_client_->makeHeaderOnlyRequest(request_headers); + waitForNextUpstreamRequest(); + + upstream_request_->encodeHeaders(default_response_headers_, true); + response->waitForEndStream(); + checkSimpleRequestSuccess(0, 0, response.get()); +} +} // namespace +} // namespace Envoy diff --git a/test/extensions/filters/network/sni_dynamic_forward_proxy/proxy_filter_test.cc b/test/extensions/filters/network/sni_dynamic_forward_proxy/proxy_filter_test.cc new file mode 100644 index 000000000000..12755253776d --- /dev/null +++ b/test/extensions/filters/network/sni_dynamic_forward_proxy/proxy_filter_test.cc @@ -0,0 +1,109 @@ +#include "envoy/extensions/filters/network/sni_dynamic_forward_proxy/v3alpha/sni_dynamic_forward_proxy.pb.h" +#include "envoy/network/connection.h" + +#include "extensions/filters/network/sni_dynamic_forward_proxy/proxy_filter.h" +#include "extensions/filters/network/well_known_names.h" + +#include "test/extensions/common/dynamic_forward_proxy/mocks.h" +#include "test/mocks/http/mocks.h" +#include "test/mocks/upstream/mocks.h" +#include "test/mocks/upstream/transport_socket_match.h" + +using testing::AtLeast; +using testing::Eq; +using testing::NiceMock; +using testing::Return; +using testing::ReturnRef; + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace SniDynamicForwardProxy { +namespace { + +using LoadDnsCacheEntryStatus = Common::DynamicForwardProxy::DnsCache::LoadDnsCacheEntryStatus; +using MockLoadDnsCacheEntryResult = + Common::DynamicForwardProxy::MockDnsCache::MockLoadDnsCacheEntryResult; + +class SniDynamicProxyFilterTest + : public testing::Test, + public Extensions::Common::DynamicForwardProxy::DnsCacheManagerFactory { +public: + SniDynamicProxyFilterTest() { + FilterConfig proto_config; + proto_config.set_port_value(443); + EXPECT_CALL(*dns_cache_manager_, getCache(_)); + filter_config_ = std::make_shared(proto_config, *this, cm_); + filter_ = std::make_unique(filter_config_); + filter_->initializeReadFilterCallbacks(callbacks_); + + // Allow for an otherwise strict mock. + ON_CALL(callbacks_, connection()).WillByDefault(ReturnRef(connection_)); + EXPECT_CALL(callbacks_, connection()).Times(AtLeast(0)); + + // Configure max pending to 1 so we can test circuit breaking. + // TODO(lizan): implement circuit breaker in SNI dynamic forward proxy + cm_.thread_local_cluster_.cluster_.info_->resetResourceManager(0, 1, 0, 0, 0); + } + + ~SniDynamicProxyFilterTest() override { + EXPECT_TRUE( + cm_.thread_local_cluster_.cluster_.info_->resource_manager_->pendingRequests().canCreate()); + } + + Extensions::Common::DynamicForwardProxy::DnsCacheManagerSharedPtr get() override { + return dns_cache_manager_; + } + + std::shared_ptr dns_cache_manager_{ + new Extensions::Common::DynamicForwardProxy::MockDnsCacheManager()}; + Upstream::MockClusterManager cm_; + ProxyFilterConfigSharedPtr filter_config_; + std::unique_ptr filter_; + Network::MockReadFilterCallbacks callbacks_; + NiceMock connection_; +}; + +// No SNI handling. +TEST_F(SniDynamicProxyFilterTest, NoSNI) { + EXPECT_CALL(connection_, requestedServerName()).WillRepeatedly(Return("")); + EXPECT_EQ(Network::FilterStatus::Continue, filter_->onNewConnection()); +} + +TEST_F(SniDynamicProxyFilterTest, LoadDnsCache) { + EXPECT_CALL(connection_, requestedServerName()).WillRepeatedly(Return("foo")); + Extensions::Common::DynamicForwardProxy::MockLoadDnsCacheEntryHandle* handle = + new Extensions::Common::DynamicForwardProxy::MockLoadDnsCacheEntryHandle(); + EXPECT_CALL(*dns_cache_manager_->dns_cache_, loadDnsCacheEntry_(Eq("foo"), 443, _)) + .WillOnce(Return(MockLoadDnsCacheEntryResult{LoadDnsCacheEntryStatus::Loading, handle})); + EXPECT_EQ(Network::FilterStatus::StopIteration, filter_->onNewConnection()); + + EXPECT_CALL(callbacks_, continueReading()); + filter_->onLoadDnsCacheComplete(); + + EXPECT_CALL(*handle, onDestroy()); +} + +TEST_F(SniDynamicProxyFilterTest, LoadDnsInCache) { + EXPECT_CALL(connection_, requestedServerName()).WillRepeatedly(Return("foo")); + EXPECT_CALL(*dns_cache_manager_->dns_cache_, loadDnsCacheEntry_(Eq("foo"), 443, _)) + .WillOnce(Return(MockLoadDnsCacheEntryResult{LoadDnsCacheEntryStatus::InCache, nullptr})); + + EXPECT_EQ(Network::FilterStatus::Continue, filter_->onNewConnection()); +} + +// Cache overflow. +TEST_F(SniDynamicProxyFilterTest, CacheOverflow) { + EXPECT_CALL(connection_, requestedServerName()).WillRepeatedly(Return("foo")); + EXPECT_CALL(*dns_cache_manager_->dns_cache_, loadDnsCacheEntry_(Eq("foo"), 443, _)) + .WillOnce(Return(MockLoadDnsCacheEntryResult{LoadDnsCacheEntryStatus::Overflow, nullptr})); + EXPECT_CALL(connection_, close(Network::ConnectionCloseType::NoFlush)); + EXPECT_EQ(Network::FilterStatus::StopIteration, filter_->onNewConnection()); +} + +} // namespace + +} // namespace SniDynamicForwardProxy +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/network/thrift_proxy/mocks.cc b/test/extensions/filters/network/thrift_proxy/mocks.cc index ee57deef2893..0f177cc79e50 100644 --- a/test/extensions/filters/network/thrift_proxy/mocks.cc +++ b/test/extensions/filters/network/thrift_proxy/mocks.cc @@ -92,22 +92,21 @@ MockDecoderFilterCallbacks::MockDecoderFilterCallbacks() { } MockDecoderFilterCallbacks::~MockDecoderFilterCallbacks() = default; -MockFilterConfigFactory::MockFilterConfigFactory() - : FactoryBase("envoy.filters.thrift.mock_filter") { +MockFilterConfigFactory::MockFilterConfigFactory() : name_("envoy.filters.thrift.mock_filter") { mock_filter_ = std::make_shared>(); } MockFilterConfigFactory::~MockFilterConfigFactory() = default; -FilterFactoryCb MockFilterConfigFactory::createFilterFactoryFromProtoTyped( - const ProtobufWkt::Struct& proto_config, const std::string& stat_prefix, +FilterFactoryCb MockFilterConfigFactory::createFilterFactoryFromProto( + const Protobuf::Message& proto_config, const std::string& stats_prefix, Server::Configuration::FactoryContext& context) { UNREFERENCED_PARAMETER(context); - config_struct_ = proto_config; - config_stat_prefix_ = stat_prefix; + config_struct_ = dynamic_cast(proto_config); + config_stat_prefix_ = stats_prefix; - return [this](ThriftFilters::FilterChainFactoryCallbacks& callbacks) -> void { + return [this](FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addDecoderFilter(mock_filter_); }; } diff --git a/test/extensions/filters/network/thrift_proxy/mocks.h b/test/extensions/filters/network/thrift_proxy/mocks.h index d7d9098df2c5..05e4e88dbc98 100644 --- a/test/extensions/filters/network/thrift_proxy/mocks.h +++ b/test/extensions/filters/network/thrift_proxy/mocks.h @@ -258,19 +258,28 @@ class MockDecoderFilterCallbacks : public DecoderFilterCallbacks { std::shared_ptr route_; }; -class MockFilterConfigFactory : public ThriftFilters::FactoryBase { +class MockFilterConfigFactory : public NamedThriftFilterConfigFactory { public: MockFilterConfigFactory(); ~MockFilterConfigFactory() override; - ThriftFilters::FilterFactoryCb - createFilterFactoryFromProtoTyped(const ProtobufWkt::Struct& proto_config, - const std::string& stat_prefix, - Server::Configuration::FactoryContext& context) override; + FilterFactoryCb + createFilterFactoryFromProto(const Protobuf::Message& proto_config, + const std::string& stats_prefix, + Server::Configuration::FactoryContext& context) override; + + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return std::make_unique(); + } + + std::string name() const override { return name_; } - std::shared_ptr mock_filter_; ProtobufWkt::Struct config_struct_; std::string config_stat_prefix_; + +private: + std::shared_ptr mock_filter_; + const std::string name_; }; } // namespace ThriftFilters diff --git a/test/extensions/tracers/datadog/config_test.cc b/test/extensions/tracers/datadog/config_test.cc index 2ba4a70398e2..87d259c0f4e7 100644 --- a/test/extensions/tracers/datadog/config_test.cc +++ b/test/extensions/tracers/datadog/config_test.cc @@ -1,5 +1,6 @@ -#include "envoy/config/trace/v3/trace.pb.h" -#include "envoy/config/trace/v3/trace.pb.validate.h" +#include "envoy/config/trace/v3/datadog.pb.h" +#include "envoy/config/trace/v3/datadog.pb.validate.h" +#include "envoy/config/trace/v3/http_tracer.pb.h" #include "extensions/tracers/datadog/config.h" diff --git a/test/extensions/tracers/datadog/datadog_tracer_impl_test.cc b/test/extensions/tracers/datadog/datadog_tracer_impl_test.cc index a70ddb7e0b27..d8fec3cbadbb 100644 --- a/test/extensions/tracers/datadog/datadog_tracer_impl_test.cc +++ b/test/extensions/tracers/datadog/datadog_tracer_impl_test.cc @@ -3,7 +3,7 @@ #include #include -#include "envoy/config/trace/v3/trace.pb.h" +#include "envoy/config/trace/v3/datadog.pb.h" #include "common/common/base64.h" #include "common/http/header_map_impl.h" diff --git a/test/extensions/tracers/dynamic_ot/config_test.cc b/test/extensions/tracers/dynamic_ot/config_test.cc index 44b1ba3dbfcb..90c77529f568 100644 --- a/test/extensions/tracers/dynamic_ot/config_test.cc +++ b/test/extensions/tracers/dynamic_ot/config_test.cc @@ -1,5 +1,6 @@ -#include "envoy/config/trace/v3/trace.pb.h" -#include "envoy/config/trace/v3/trace.pb.validate.h" +#include "envoy/config/trace/v3/dynamic_ot.pb.h" +#include "envoy/config/trace/v3/dynamic_ot.pb.validate.h" +#include "envoy/config/trace/v3/http_tracer.pb.h" #include "extensions/tracers/dynamic_ot/config.h" diff --git a/test/extensions/tracers/lightstep/config_test.cc b/test/extensions/tracers/lightstep/config_test.cc index 866c31530631..ec09bf27c6c9 100644 --- a/test/extensions/tracers/lightstep/config_test.cc +++ b/test/extensions/tracers/lightstep/config_test.cc @@ -1,5 +1,6 @@ -#include "envoy/config/trace/v3/trace.pb.h" -#include "envoy/config/trace/v3/trace.pb.validate.h" +#include "envoy/config/trace/v3/http_tracer.pb.h" +#include "envoy/config/trace/v3/lightstep.pb.h" +#include "envoy/config/trace/v3/lightstep.pb.validate.h" #include "extensions/tracers/lightstep/config.h" diff --git a/test/extensions/tracers/lightstep/lightstep_tracer_impl_test.cc b/test/extensions/tracers/lightstep/lightstep_tracer_impl_test.cc index da83c6c09910..48c50dd51bbf 100644 --- a/test/extensions/tracers/lightstep/lightstep_tracer_impl_test.cc +++ b/test/extensions/tracers/lightstep/lightstep_tracer_impl_test.cc @@ -3,7 +3,7 @@ #include #include -#include "envoy/config/trace/v3/trace.pb.h" +#include "envoy/config/trace/v3/lightstep.pb.h" #include "common/common/base64.h" #include "common/grpc/common.h" diff --git a/test/extensions/tracers/opencensus/config_test.cc b/test/extensions/tracers/opencensus/config_test.cc index c526bd578feb..29888485e3de 100644 --- a/test/extensions/tracers/opencensus/config_test.cc +++ b/test/extensions/tracers/opencensus/config_test.cc @@ -1,5 +1,6 @@ -#include "envoy/config/trace/v3/trace.pb.h" -#include "envoy/config/trace/v3/trace.pb.validate.h" +#include "envoy/config/trace/v3/http_tracer.pb.h" +#include "envoy/config/trace/v3/opencensus.pb.h" +#include "envoy/config/trace/v3/opencensus.pb.validate.h" #include "envoy/registry/registry.h" #include "extensions/tracers/opencensus/config.h" diff --git a/test/extensions/tracers/opencensus/tracer_test.cc b/test/extensions/tracers/opencensus/tracer_test.cc index e90fd7756426..7ee0a23184c2 100644 --- a/test/extensions/tracers/opencensus/tracer_test.cc +++ b/test/extensions/tracers/opencensus/tracer_test.cc @@ -5,7 +5,7 @@ #include #include -#include "envoy/config/trace/v3/trace.pb.h" +#include "envoy/config/trace/v3/opencensus.pb.h" #include "common/common/base64.h" diff --git a/test/extensions/tracers/xray/config_test.cc b/test/extensions/tracers/xray/config_test.cc index be281bf47281..b71092f60eda 100644 --- a/test/extensions/tracers/xray/config_test.cc +++ b/test/extensions/tracers/xray/config_test.cc @@ -1,4 +1,4 @@ -#include "envoy/config/trace/v3/trace.pb.h" +#include "envoy/config/trace/v3/http_tracer.pb.h" #include "envoy/config/trace/v3/xray.pb.h" #include "envoy/config/trace/v3/xray.pb.validate.h" #include "envoy/registry/registry.h" diff --git a/test/extensions/tracers/zipkin/BUILD b/test/extensions/tracers/zipkin/BUILD index 32c0fbea29b0..385c4f194759 100644 --- a/test/extensions/tracers/zipkin/BUILD +++ b/test/extensions/tracers/zipkin/BUILD @@ -41,7 +41,6 @@ envoy_extension_cc_test( "//test/mocks/tracing:tracing_mocks", "//test/mocks/upstream:upstream_mocks", "//test/test_common:simulated_time_system_lib", - "//test/test_common:test_time_lib", "//test/test_common:utility_lib", "@envoy_api//envoy/config/trace/v3:pkg_cc_proto", ], diff --git a/test/extensions/tracers/zipkin/config_test.cc b/test/extensions/tracers/zipkin/config_test.cc index 7ea71eaec9d6..352a923ffe23 100644 --- a/test/extensions/tracers/zipkin/config_test.cc +++ b/test/extensions/tracers/zipkin/config_test.cc @@ -1,5 +1,6 @@ -#include "envoy/config/trace/v3/trace.pb.h" -#include "envoy/config/trace/v3/trace.pb.validate.h" +#include "envoy/config/trace/v3/http_tracer.pb.h" +#include "envoy/config/trace/v3/zipkin.pb.h" +#include "envoy/config/trace/v3/zipkin.pb.validate.h" #include "envoy/registry/registry.h" #include "extensions/tracers/zipkin/config.h" diff --git a/test/extensions/tracers/zipkin/span_buffer_test.cc b/test/extensions/tracers/zipkin/span_buffer_test.cc index f9227912ec90..05563a02de01 100644 --- a/test/extensions/tracers/zipkin/span_buffer_test.cc +++ b/test/extensions/tracers/zipkin/span_buffer_test.cc @@ -1,21 +1,34 @@ -#include "envoy/config/trace/v3/trace.pb.h" +#include "envoy/config/trace/v3/zipkin.pb.h" #include "common/network/utility.h" +#include "common/protobuf/utility.h" #include "extensions/tracers/zipkin/span_buffer.h" +#include "extensions/tracers/zipkin/util.h" -#include "test/test_common/test_time.h" +#include "test/test_common/simulated_time_system.h" #include "test/test_common/utility.h" #include "absl/strings/str_format.h" #include "gtest/gtest.h" +using testing::HasSubstr; + namespace Envoy { namespace Extensions { namespace Tracers { namespace Zipkin { namespace { +// If this default timestamp is wrapped as double (using ValueUtil::numberValue()) and then it is +// serialized using Protobuf::util::MessageToJsonString, it renders as: 1.58432429547687e+15. +constexpr uint64_t DEFAULT_TEST_TIMESTAMP = 1584324295476870; +constexpr uint64_t DEFAULT_TEST_DURATION = 2584324295476870; +const Util::Replacements DEFAULT_TEST_REPLACEMENTS = { + {"DEFAULT_TEST_TIMESTAMP", std::to_string(DEFAULT_TEST_TIMESTAMP)}}; +const Util::Replacements DEFAULT_TEST_DURATIONS = { + {"DEFAULT_TEST_DURATION", std::to_string(DEFAULT_TEST_DURATION)}}; + enum class IpType { V4, V6 }; Endpoint createEndpoint(const IpType ip_type) { @@ -31,7 +44,7 @@ Endpoint createEndpoint(const IpType ip_type) { Annotation createAnnotation(const absl::string_view value, const IpType ip_type) { Annotation annotation; annotation.setValue(value.data()); - annotation.setTimestamp(1566058071601051); + annotation.setTimestamp(DEFAULT_TEST_TIMESTAMP); annotation.setEndpoint(createEndpoint(ip_type)); return annotation; } @@ -44,11 +57,11 @@ BinaryAnnotation createTag() { } Span createSpan(const std::vector& annotation_values, const IpType ip_type) { - DangerousDeprecatedTestTime test_time; - Span span(test_time.timeSystem()); + Event::SimulatedTimeSystem simulated_time_system; + Span span(simulated_time_system); span.setId(1); span.setTraceId(1); - span.setDuration(100); + span.setDuration(DEFAULT_TEST_DURATION); std::vector annotations; annotations.reserve(annotation_values.size()); for (absl::string_view value : annotation_values) { @@ -59,15 +72,25 @@ Span createSpan(const std::vector& annotation_values, const I return span; } +// To render a string with DEFAULT_TEST_TIMESTAMP and DEFAULT_TEST_DURATION placeholder with +// DEFAULT_TEST_TIMESTAMP and DEFAULT_TEST_DURATION values. +std::string withDefaultTimestampAndDuration(const std::string& expected) { + const auto with_default_timestamp = absl::StrReplaceAll(expected, DEFAULT_TEST_REPLACEMENTS); + return absl::StrReplaceAll(with_default_timestamp, DEFAULT_TEST_DURATIONS); +} + // To wrap JSON array string in a object for JSON string comparison through JsonStringEq test -// utility. +// utility. Every DEFAULT_TEST_TIMESTAMP and DEFAULT_TEST_DURATION strings found in array_string +// will be replaced by DEFAULT_TEST_REPLACEMENTS and DEFAULT_TEST_DURATIONS respectively. i.e. to +// replace every DEFAULT_TEST_TIMESTAMP string occurrence with DEFAULT_TEST_TIMESTAMP value (the +// same with DEFAULT_TEST_DURATION). std::string wrapAsObject(absl::string_view array_string) { - return absl::StrFormat(R"({"root":%s})", array_string); + return withDefaultTimestampAndDuration(absl::StrFormat(R"({"root":%s})", array_string)); } void expectSerializedBuffer(SpanBuffer& buffer, const bool delay_allocation, const std::vector& expected_list) { - DangerousDeprecatedTestTime test_time; + Event::SimulatedTimeSystem test_time; EXPECT_EQ(0ULL, buffer.pendingSpans()); EXPECT_EQ("[]", buffer.serialize()); @@ -105,56 +128,71 @@ template std::string serializedMessageToJson(const std::string& return json; } +TEST(ZipkinSpanBufferTest, TestSerializeTimestamp) { + const std::string default_timestamp_string = std::to_string(DEFAULT_TEST_TIMESTAMP); + + ProtobufWkt::Struct object; + auto* fields = object.mutable_fields(); + Util::Replacements replacements; + (*fields)["timestamp"] = Util::uint64Value(DEFAULT_TEST_TIMESTAMP, replacements); + + ASSERT_EQ(1, replacements.size()); + EXPECT_EQ(absl::StrCat("\"", default_timestamp_string, "\""), replacements.at(0).first); + EXPECT_EQ(default_timestamp_string, replacements.at(0).second); +} + TEST(ZipkinSpanBufferTest, ConstructBuffer) { - const std::string expected1 = R"([{"traceId":"0000000000000001",)" - R"("name":"",)" - R"("id":"0000000000000001",)" - R"("duration":100,)" - R"("annotations":[{"timestamp":1566058071601051,)" - R"("value":"cs",)" - R"("endpoint":{"ipv4":"1.2.3.4",)" - R"("port":8080,)" - R"("serviceName":"service1"}},)" - R"({"timestamp":1566058071601051,)" - R"("value":"sr",)" - R"("endpoint":{"ipv4":"1.2.3.4",)" - R"("port":8080,)" - R"("serviceName":"service1"}}],)" - R"("binaryAnnotations":[{"key":"component",)" - R"("value":"proxy"}]}])"; - - const std::string expected2 = R"([{"traceId":"0000000000000001",)" - R"("name":"",)" - R"("id":"0000000000000001",)" - R"("duration":100,)" - R"("annotations":[{"timestamp":1566058071601051,)" - R"("value":"cs",)" - R"("endpoint":{"ipv4":"1.2.3.4",)" - R"("port":8080,)" - R"("serviceName":"service1"}},)" - R"({"timestamp":1566058071601051,)" - R"("value":"sr",)" - R"("endpoint":{"ipv4":"1.2.3.4",)" - R"("port":8080,)" - R"("serviceName":"service1"}}],)" - R"("binaryAnnotations":[{"key":"component",)" - R"("value":"proxy"}]},)" - R"({"traceId":"0000000000000001",)" - R"("name":"",)" - R"("id":"0000000000000001",)" - R"("duration":100,)" - R"("annotations":[{"timestamp":1566058071601051,)" - R"("value":"cs",)" - R"("endpoint":{"ipv4":"1.2.3.4",)" - R"("port":8080,)" - R"("serviceName":"service1"}},)" - R"({"timestamp":1566058071601051,)" - R"("value":"sr",)" - R"("endpoint":{"ipv4":"1.2.3.4",)" - R"("port":8080,)" - R"("serviceName":"service1"}}],)" - R"("binaryAnnotations":[{"key":"component",)" - R"("value":"proxy"}]}])"; + const std::string expected1 = + withDefaultTimestampAndDuration(R"([{"traceId":"0000000000000001",)" + R"("name":"",)" + R"("id":"0000000000000001",)" + R"("duration":DEFAULT_TEST_DURATION,)" + R"("annotations":[{"timestamp":DEFAULT_TEST_TIMESTAMP,)" + R"("value":"cs",)" + R"("endpoint":{"ipv4":"1.2.3.4",)" + R"("port":8080,)" + R"("serviceName":"service1"}},)" + R"({"timestamp":DEFAULT_TEST_TIMESTAMP,)" + R"("value":"sr",)" + R"("endpoint":{"ipv4":"1.2.3.4",)" + R"("port":8080,)" + R"("serviceName":"service1"}}],)" + R"("binaryAnnotations":[{"key":"component",)" + R"("value":"proxy"}]}])"); + + const std::string expected2 = + withDefaultTimestampAndDuration(R"([{"traceId":"0000000000000001",)" + R"("name":"",)" + R"("id":"0000000000000001",)" + R"("duration":DEFAULT_TEST_DURATION,)" + R"("annotations":[{"timestamp":DEFAULT_TEST_TIMESTAMP,)" + R"("value":"cs",)" + R"("endpoint":{"ipv4":"1.2.3.4",)" + R"("port":8080,)" + R"("serviceName":"service1"}},)" + R"({"timestamp":DEFAULT_TEST_TIMESTAMP,)" + R"("value":"sr",)" + R"("endpoint":{"ipv4":"1.2.3.4",)" + R"("port":8080,)" + R"("serviceName":"service1"}}],)" + R"("binaryAnnotations":[{"key":"component",)" + R"("value":"proxy"}]},)" + R"({"traceId":"0000000000000001",)" + R"("name":"",)" + R"("id":"0000000000000001",)" + R"("duration":DEFAULT_TEST_DURATION,)" + R"("annotations":[{"timestamp":DEFAULT_TEST_TIMESTAMP,)" + R"("value":"cs",)" + R"("endpoint":{"ipv4":"1.2.3.4",)" + R"("port":8080,)" + R"("serviceName":"service1"}},)" + R"({"timestamp":DEFAULT_TEST_TIMESTAMP,)" + R"("value":"sr",)" + R"("endpoint":{"ipv4":"1.2.3.4",)" + R"("port":8080,)" + R"("serviceName":"service1"}}],)" + R"("binaryAnnotations":[{"key":"component",)" + R"("value":"proxy"}]}])"); const bool shared = true; const bool delay_allocation = true; @@ -176,8 +214,8 @@ TEST(ZipkinSpanBufferTest, SerializeSpan) { R"("traceId":"0000000000000001",)" R"("id":"0000000000000001",)" R"("kind":"CLIENT",)" - R"("timestamp":1566058071601051,)" - R"("duration":100,)" + R"("timestamp":DEFAULT_TEST_TIMESTAMP,)" + R"("duration":DEFAULT_TEST_DURATION,)" R"("localEndpoint":{)" R"("serviceName":"service1",)" R"("ipv4":"1.2.3.4",)" @@ -193,8 +231,8 @@ TEST(ZipkinSpanBufferTest, SerializeSpan) { R"("traceId":"0000000000000001",)" R"("id":"0000000000000001",)" R"("kind":"CLIENT",)" - R"("timestamp":1566058071601051,)" - R"("duration":100,)" + R"("timestamp":DEFAULT_TEST_TIMESTAMP,)" + R"("duration":DEFAULT_TEST_DURATION,)" R"("localEndpoint":{)" R"("serviceName":"service1",)" R"("ipv6":"2001:db8:85a3::8a2e:370:4444",)" @@ -210,8 +248,8 @@ TEST(ZipkinSpanBufferTest, SerializeSpan) { R"("traceId":"0000000000000001",)" R"("id":"0000000000000001",)" R"("kind":"CLIENT",)" - R"("timestamp":1566058071601051,)" - R"("duration":100,)" + R"("timestamp":DEFAULT_TEST_TIMESTAMP,)" + R"("duration":DEFAULT_TEST_DURATION,)" R"("localEndpoint":{)" R"("serviceName":"service1",)" R"("ipv4":"1.2.3.4",)" @@ -222,8 +260,8 @@ TEST(ZipkinSpanBufferTest, SerializeSpan) { R"("traceId":"0000000000000001",)" R"("id":"0000000000000001",)" R"("kind":"SERVER",)" - R"("timestamp":1566058071601051,)" - R"("duration":100,)" + R"("timestamp":DEFAULT_TEST_TIMESTAMP,)" + R"("duration":DEFAULT_TEST_DURATION,)" R"("localEndpoint":{)" R"("serviceName":"service1",)" R"("ipv4":"1.2.3.4",)" @@ -240,8 +278,8 @@ TEST(ZipkinSpanBufferTest, SerializeSpan) { R"("traceId":"0000000000000001",)" R"("id":"0000000000000001",)" R"("kind":"CLIENT",)" - R"("timestamp":1566058071601051,)" - R"("duration":100,)" + R"("timestamp":DEFAULT_TEST_TIMESTAMP,)" + R"("duration":DEFAULT_TEST_DURATION,)" R"("localEndpoint":{)" R"("serviceName":"service1",)" R"("ipv4":"1.2.3.4",)" @@ -252,8 +290,8 @@ TEST(ZipkinSpanBufferTest, SerializeSpan) { R"("traceId":"0000000000000001",)" R"("id":"0000000000000001",)" R"("kind":"SERVER",)" - R"("timestamp":1566058071601051,)" - R"("duration":100,)" + R"("timestamp":DEFAULT_TEST_TIMESTAMP,)" + R"("duration":DEFAULT_TEST_DURATION,)" R"("localEndpoint":{)" R"("serviceName":"service1",)" R"("ipv4":"1.2.3.4",)" @@ -265,102 +303,150 @@ TEST(ZipkinSpanBufferTest, SerializeSpan) { SpanBuffer buffer4(envoy::config::trace::v3::ZipkinConfig::HTTP_PROTO, shared, 2); buffer4.addSpan(createSpan({"cs"}, IpType::V4)); - EXPECT_EQ("{" - R"("spans":[{)" - R"("traceId":"AAAAAAAAAAE=",)" - R"("id":"AQAAAAAAAAA=",)" - R"("kind":"CLIENT",)" - R"("timestamp":"1566058071601051",)" - R"("duration":"100",)" - R"("localEndpoint":{)" - R"("serviceName":"service1",)" - R"("ipv4":"AQIDBA==",)" - R"("port":8080},)" - R"("tags":{)" - R"("component":"proxy"})" - "}]}", + EXPECT_EQ(withDefaultTimestampAndDuration("{" + R"("spans":[{)" + R"("traceId":"AAAAAAAAAAE=",)" + R"("id":"AQAAAAAAAAA=",)" + R"("kind":"CLIENT",)" + R"("timestamp":"DEFAULT_TEST_TIMESTAMP",)" + R"("duration":"DEFAULT_TEST_DURATION",)" + R"("localEndpoint":{)" + R"("serviceName":"service1",)" + R"("ipv4":"AQIDBA==",)" + R"("port":8080},)" + R"("tags":{)" + R"("component":"proxy"})" + "}]}"), serializedMessageToJson(buffer4.serialize())); SpanBuffer buffer4_v6(envoy::config::trace::v3::ZipkinConfig::HTTP_PROTO, shared, 2); buffer4_v6.addSpan(createSpan({"cs"}, IpType::V6)); - EXPECT_EQ("{" - R"("spans":[{)" - R"("traceId":"AAAAAAAAAAE=",)" - R"("id":"AQAAAAAAAAA=",)" - R"("kind":"CLIENT",)" - R"("timestamp":"1566058071601051",)" - R"("duration":"100",)" - R"("localEndpoint":{)" - R"("serviceName":"service1",)" - R"("ipv6":"IAENuIWjAAAAAIouA3BERA==",)" - R"("port":7334},)" - R"("tags":{)" - R"("component":"proxy"})" - "}]}", + EXPECT_EQ(withDefaultTimestampAndDuration("{" + R"("spans":[{)" + R"("traceId":"AAAAAAAAAAE=",)" + R"("id":"AQAAAAAAAAA=",)" + R"("kind":"CLIENT",)" + R"("timestamp":"DEFAULT_TEST_TIMESTAMP",)" + R"("duration":"DEFAULT_TEST_DURATION",)" + R"("localEndpoint":{)" + R"("serviceName":"service1",)" + R"("ipv6":"IAENuIWjAAAAAIouA3BERA==",)" + R"("port":7334},)" + R"("tags":{)" + R"("component":"proxy"})" + "}]}"), serializedMessageToJson(buffer4_v6.serialize())); SpanBuffer buffer5(envoy::config::trace::v3::ZipkinConfig::HTTP_PROTO, shared, 2); buffer5.addSpan(createSpan({"cs", "sr"}, IpType::V4)); - EXPECT_EQ("{" - R"("spans":[{)" - R"("traceId":"AAAAAAAAAAE=",)" - R"("id":"AQAAAAAAAAA=",)" - R"("kind":"CLIENT",)" - R"("timestamp":"1566058071601051",)" - R"("duration":"100",)" - R"("localEndpoint":{)" - R"("serviceName":"service1",)" - R"("ipv4":"AQIDBA==",)" - R"("port":8080},)" - R"("tags":{)" - R"("component":"proxy"}},)" - R"({)" - R"("traceId":"AAAAAAAAAAE=",)" - R"("id":"AQAAAAAAAAA=",)" - R"("kind":"SERVER",)" - R"("timestamp":"1566058071601051",)" - R"("duration":"100",)" - R"("localEndpoint":{)" - R"("serviceName":"service1",)" - R"("ipv4":"AQIDBA==",)" - R"("port":8080},)" - R"("tags":{)" - R"("component":"proxy"},)" - R"("shared":true)" - "}]}", + EXPECT_EQ(withDefaultTimestampAndDuration("{" + R"("spans":[{)" + R"("traceId":"AAAAAAAAAAE=",)" + R"("id":"AQAAAAAAAAA=",)" + R"("kind":"CLIENT",)" + R"("timestamp":"DEFAULT_TEST_TIMESTAMP",)" + R"("duration":"DEFAULT_TEST_DURATION",)" + R"("localEndpoint":{)" + R"("serviceName":"service1",)" + R"("ipv4":"AQIDBA==",)" + R"("port":8080},)" + R"("tags":{)" + R"("component":"proxy"}},)" + R"({)" + R"("traceId":"AAAAAAAAAAE=",)" + R"("id":"AQAAAAAAAAA=",)" + R"("kind":"SERVER",)" + R"("timestamp":"DEFAULT_TEST_TIMESTAMP",)" + R"("duration":"DEFAULT_TEST_DURATION",)" + R"("localEndpoint":{)" + R"("serviceName":"service1",)" + R"("ipv4":"AQIDBA==",)" + R"("port":8080},)" + R"("tags":{)" + R"("component":"proxy"},)" + R"("shared":true)" + "}]}"), serializedMessageToJson(buffer5.serialize())); SpanBuffer buffer6(envoy::config::trace::v3::ZipkinConfig::HTTP_PROTO, !shared, 2); buffer6.addSpan(createSpan({"cs", "sr"}, IpType::V4)); - EXPECT_EQ("{" - R"("spans":[{)" - R"("traceId":"AAAAAAAAAAE=",)" - R"("id":"AQAAAAAAAAA=",)" - R"("kind":"CLIENT",)" - R"("timestamp":"1566058071601051",)" - R"("duration":"100",)" - R"("localEndpoint":{)" - R"("serviceName":"service1",)" - R"("ipv4":"AQIDBA==",)" - R"("port":8080},)" - R"("tags":{)" - R"("component":"proxy"}},)" - R"({)" - R"("traceId":"AAAAAAAAAAE=",)" - R"("id":"AQAAAAAAAAA=",)" - R"("kind":"SERVER",)" - R"("timestamp":"1566058071601051",)" - R"("duration":"100",)" - R"("localEndpoint":{)" - R"("serviceName":"service1",)" - R"("ipv4":"AQIDBA==",)" - R"("port":8080},)" - R"("tags":{)" - R"("component":"proxy"})" - "}]}", + EXPECT_EQ(withDefaultTimestampAndDuration("{" + R"("spans":[{)" + R"("traceId":"AAAAAAAAAAE=",)" + R"("id":"AQAAAAAAAAA=",)" + R"("kind":"CLIENT",)" + R"("timestamp":"DEFAULT_TEST_TIMESTAMP",)" + R"("duration":"DEFAULT_TEST_DURATION",)" + R"("localEndpoint":{)" + R"("serviceName":"service1",)" + R"("ipv4":"AQIDBA==",)" + R"("port":8080},)" + R"("tags":{)" + R"("component":"proxy"}},)" + R"({)" + R"("traceId":"AAAAAAAAAAE=",)" + R"("id":"AQAAAAAAAAA=",)" + R"("kind":"SERVER",)" + R"("timestamp":"DEFAULT_TEST_TIMESTAMP",)" + R"("duration":"DEFAULT_TEST_DURATION",)" + R"("localEndpoint":{)" + R"("serviceName":"service1",)" + R"("ipv4":"AQIDBA==",)" + R"("port":8080},)" + R"("tags":{)" + R"("component":"proxy"})" + "}]}"), serializedMessageToJson(buffer6.serialize())); } +TEST(ZipkinSpanBufferTest, TestSerializeTimestampInTheFuture) { + ProtobufWkt::Struct objectWithScientificNotation; + auto* objectWithScientificNotationFields = objectWithScientificNotation.mutable_fields(); + (*objectWithScientificNotationFields)["timestamp"] = ValueUtil::numberValue( + DEFAULT_TEST_TIMESTAMP); // the value of DEFAULT_TEST_TIMESTAMP is 1584324295476870. + const auto objectWithScientificNotationJson = + MessageUtil::getJsonStringFromMessage(objectWithScientificNotation, false, true); + // Since we use ValueUtil::numberValue to set the timestamp, we expect to + // see the value is rendered with scientific notation (1.58432429547687e+15). + EXPECT_EQ(R"({"timestamp":1.58432429547687e+15})", objectWithScientificNotationJson); + + ProtobufWkt::Struct object; + auto* objectFields = object.mutable_fields(); + Util::Replacements replacements; + (*objectFields)["timestamp"] = Util::uint64Value(DEFAULT_TEST_TIMESTAMP, replacements); + const auto objectJson = MessageUtil::getJsonStringFromMessage(object, false, true); + // We still have "1584324295476870" from MessageUtil::getJsonStringFromMessage here. + EXPECT_EQ(R"({"timestamp":"1584324295476870"})", objectJson); + // However, then the replacement correctly replaces "1584324295476870" with 1584324295476870 + // (without quotes). + EXPECT_EQ(R"({"timestamp":1584324295476870})", absl::StrReplaceAll(objectJson, replacements)); + + SpanBuffer bufferDeprecatedJsonV1(envoy::config::trace::v3::ZipkinConfig::HTTP_JSON, true, 2); + bufferDeprecatedJsonV1.addSpan(createSpan({"cs"}, IpType::V4)); + // We do "HasSubstr" here since we could not compare the serialized JSON of a ProtobufWkt::Struct + // object, since the positions of keys are not consistent between calls. + EXPECT_THAT(bufferDeprecatedJsonV1.serialize(), HasSubstr(R"("timestamp":1584324295476870)")); + EXPECT_THAT(bufferDeprecatedJsonV1.serialize(), + Not(HasSubstr(R"("timestamp":1.58432429547687e+15)"))); + EXPECT_THAT(bufferDeprecatedJsonV1.serialize(), + Not(HasSubstr(R"("timestamp":"1584324295476870")"))); + EXPECT_THAT(bufferDeprecatedJsonV1.serialize(), HasSubstr(R"("duration":2584324295476870)")); + EXPECT_THAT(bufferDeprecatedJsonV1.serialize(), + Not(HasSubstr(R"("duration":2.584324295476870e+15)"))); + EXPECT_THAT(bufferDeprecatedJsonV1.serialize(), + Not(HasSubstr(R"("duration":"2584324295476870")"))); + + SpanBuffer bufferJsonV2( + envoy::config::trace::v3::ZipkinConfig::hidden_envoy_deprecated_HTTP_JSON_V1, true, 2); + bufferJsonV2.addSpan(createSpan({"cs"}, IpType::V4)); + EXPECT_THAT(bufferJsonV2.serialize(), HasSubstr(R"("timestamp":1584324295476870)")); + EXPECT_THAT(bufferJsonV2.serialize(), Not(HasSubstr(R"("timestamp":1.58432429547687e+15)"))); + EXPECT_THAT(bufferJsonV2.serialize(), Not(HasSubstr(R"("timestamp":"1584324295476870")"))); + EXPECT_THAT(bufferJsonV2.serialize(), HasSubstr(R"("duration":2584324295476870)")); + EXPECT_THAT(bufferJsonV2.serialize(), Not(HasSubstr(R"("duration":2.584324295476870e+15)"))); + EXPECT_THAT(bufferJsonV2.serialize(), Not(HasSubstr(R"("duration":"2584324295476870")"))); +} + } // namespace } // namespace Zipkin } // namespace Tracers diff --git a/test/extensions/tracers/zipkin/zipkin_core_types_test.cc b/test/extensions/tracers/zipkin/zipkin_core_types_test.cc index 97ef5254db05..cc4470820f24 100644 --- a/test/extensions/tracers/zipkin/zipkin_core_types_test.cc +++ b/test/extensions/tracers/zipkin/zipkin_core_types_test.cc @@ -5,7 +5,7 @@ #include "extensions/tracers/zipkin/zipkin_core_constants.h" #include "extensions/tracers/zipkin/zipkin_core_types.h" -#include "test/test_common/test_time.h" +#include "test/test_common/simulated_time_system.h" #include "test/test_common/utility.h" #include "gtest/gtest.h" @@ -18,17 +18,19 @@ namespace { TEST(ZipkinCoreTypesEndpointTest, defaultConstructor) { Endpoint ep; + Util::Replacements replacements; EXPECT_EQ("", ep.serviceName()); - EXPECT_TRUE(TestUtility::protoEqual( - TestUtility::jsonToStruct(R"({"ipv4":"","port":0,"serviceName":""})"), ep.toStruct())); + EXPECT_TRUE( + TestUtility::protoEqual(TestUtility::jsonToStruct(R"({"ipv4":"","port":0,"serviceName":""})"), + ep.toStruct(replacements))); Network::Address::InstanceConstSharedPtr addr = Network::Utility::parseInternetAddress("127.0.0.1"); ep.setAddress(addr); EXPECT_TRUE(TestUtility::protoEqual( TestUtility::jsonToStruct(R"({"ipv4":"127.0.0.1","port":0,"serviceName":""})"), - ep.toStruct())); + ep.toStruct(replacements))); addr = Network::Utility::parseInternetAddressAndPort( "[2001:0db8:85a3:0000:0000:8a2e:0370:4444]:7334"); @@ -36,7 +38,8 @@ TEST(ZipkinCoreTypesEndpointTest, defaultConstructor) { EXPECT_TRUE(TestUtility::protoEqual( TestUtility::jsonToStruct( R"({"ipv6":"2001:db8:85a3::8a2e:370:4444","port":7334,"serviceName":""})"), - ep.toStruct())); + ep.toStruct(replacements))); + EXPECT_TRUE(replacements.empty()); ep.setServiceName("my_service"); EXPECT_EQ("my_service", ep.serviceName()); @@ -44,18 +47,21 @@ TEST(ZipkinCoreTypesEndpointTest, defaultConstructor) { EXPECT_TRUE(TestUtility::protoEqual( TestUtility::jsonToStruct( R"({"ipv6":"2001:db8:85a3::8a2e:370:4444","port":7334,"serviceName":"my_service"})"), - ep.toStruct())); + ep.toStruct(replacements))); + EXPECT_TRUE(replacements.empty()); } TEST(ZipkinCoreTypesEndpointTest, customConstructor) { Network::Address::InstanceConstSharedPtr addr = Network::Utility::parseInternetAddressAndPort("127.0.0.1:3306"); Endpoint ep(std::string("my_service"), addr); + Util::Replacements replacements; EXPECT_EQ("my_service", ep.serviceName()); EXPECT_TRUE(TestUtility::protoEqual( TestUtility::jsonToStruct(R"({"ipv4":"127.0.0.1","port":3306,"serviceName":"my_service"})"), - ep.toStruct())); + ep.toStruct(replacements))); + EXPECT_TRUE(replacements.empty()); addr = Network::Utility::parseInternetAddressAndPort( "[2001:0db8:85a3:0000:0000:8a2e:0370:4444]:7334"); @@ -64,47 +70,53 @@ TEST(ZipkinCoreTypesEndpointTest, customConstructor) { EXPECT_TRUE(TestUtility::protoEqual( TestUtility::jsonToStruct( R"({"ipv6":"2001:db8:85a3::8a2e:370:4444","port":7334,"serviceName":"my_service"})"), - ep.toStruct())); + ep.toStruct(replacements))); + EXPECT_TRUE(replacements.empty()); } TEST(ZipkinCoreTypesEndpointTest, copyOperator) { Network::Address::InstanceConstSharedPtr addr = Network::Utility::parseInternetAddressAndPort("127.0.0.1:3306"); Endpoint ep1(std::string("my_service"), addr); - const Endpoint& ep2(ep1); + Endpoint& ep2(ep1); + Util::Replacements replacements; EXPECT_EQ("my_service", ep1.serviceName()); EXPECT_TRUE(TestUtility::protoEqual( TestUtility::jsonToStruct(R"({"ipv4":"127.0.0.1","port":3306,"serviceName":"my_service"})"), - ep1.toStruct())); + ep1.toStruct(replacements))); EXPECT_EQ(ep1.serviceName(), ep2.serviceName()); - EXPECT_TRUE(TestUtility::protoEqual(ep1.toStruct(), ep2.toStruct())); + EXPECT_TRUE(TestUtility::protoEqual(ep1.toStruct(replacements), ep2.toStruct(replacements))); + EXPECT_TRUE(replacements.empty()); } TEST(ZipkinCoreTypesEndpointTest, assignmentOperator) { Network::Address::InstanceConstSharedPtr addr = Network::Utility::parseInternetAddressAndPort("127.0.0.1:3306"); Endpoint ep1(std::string("my_service"), addr); - const Endpoint& ep2 = ep1; + Endpoint& ep2 = ep1; + Util::Replacements replacements; EXPECT_EQ("my_service", ep1.serviceName()); EXPECT_TRUE(TestUtility::protoEqual( TestUtility::jsonToStruct(R"({"ipv4":"127.0.0.1","port":3306,"serviceName":"my_service"})"), - ep1.toStruct())); + ep1.toStruct(replacements))); EXPECT_EQ(ep1.serviceName(), ep2.serviceName()); - EXPECT_TRUE(TestUtility::protoEqual(ep1.toStruct(), ep2.toStruct())); + EXPECT_TRUE(TestUtility::protoEqual(ep1.toStruct(replacements), ep2.toStruct(replacements))); + EXPECT_TRUE(replacements.empty()); } TEST(ZipkinCoreTypesAnnotationTest, defaultConstructor) { Annotation ann; + Util::Replacements replacements; EXPECT_EQ(0ULL, ann.timestamp()); EXPECT_EQ("", ann.value()); EXPECT_FALSE(ann.isSetEndpoint()); - DangerousDeprecatedTestTime test_time; + Event::SimulatedTimeSystem test_time; uint64_t timestamp = std::chrono::duration_cast( test_time.timeSystem().systemTime().time_since_epoch()) .count(); @@ -114,10 +126,13 @@ TEST(ZipkinCoreTypesAnnotationTest, defaultConstructor) { ann.setValue(CLIENT_SEND); EXPECT_EQ(CLIENT_SEND, ann.value()); - std::string expected_json = - R"({"timestamp":)" + std::to_string(timestamp) + R"(,"value":")" + CLIENT_SEND + R"("})"; - EXPECT_TRUE(TestUtility::protoEqual(TestUtility::jsonToStruct(expected_json), ann.toStruct())); + std::string expected_json = R"({"timestamp":")" + std::to_string(timestamp) + R"(")" + + R"(,"value":")" + CLIENT_SEND + R"("})"; + EXPECT_TRUE(TestUtility::protoEqual(TestUtility::jsonToStruct(expected_json), + ann.toStruct(replacements))); + EXPECT_EQ(1, replacements.size()); + replacements.clear(); // Test the copy-semantics flavor of setEndpoint Network::Address::InstanceConstSharedPtr addr = Network::Utility::parseInternetAddressAndPort("127.0.0.1:3306"); @@ -127,13 +142,18 @@ TEST(ZipkinCoreTypesAnnotationTest, defaultConstructor) { EXPECT_EQ("my_service", ann.endpoint().serviceName()); EXPECT_TRUE(TestUtility::protoEqual( TestUtility::jsonToStruct(R"({"ipv4":"127.0.0.1","port":3306,"serviceName":"my_service"})"), - (const_cast(ann.endpoint())).toStruct())); + (const_cast(ann.endpoint())).toStruct(replacements))); - expected_json = R"({"timestamp":)" + std::to_string(timestamp) + R"(,"value":")" + CLIENT_SEND + + expected_json = R"({"timestamp":")" + std::to_string(timestamp) + R"(")" + R"(,"value":")" + + CLIENT_SEND + R"(","endpoint":{"ipv4":)" R"("127.0.0.1","port":3306,"serviceName":"my_service"}})"; - EXPECT_TRUE(TestUtility::protoEqual(TestUtility::jsonToStruct(expected_json), ann.toStruct())); + EXPECT_TRUE(TestUtility::protoEqual(TestUtility::jsonToStruct(expected_json), + ann.toStruct(replacements))); + + EXPECT_EQ(1, replacements.size()); + replacements.clear(); // Test the move-semantics flavor of setEndpoint addr = Network::Utility::parseInternetAddressAndPort("192.168.1.1:5555"); Endpoint ep2(std::string("my_service_2"), addr); @@ -143,31 +163,41 @@ TEST(ZipkinCoreTypesAnnotationTest, defaultConstructor) { EXPECT_TRUE(TestUtility::protoEqual( TestUtility::jsonToStruct( R"({"ipv4":"192.168.1.1","port":5555,"serviceName":"my_service_2"})"), - (const_cast(ann.endpoint())).toStruct())); + (const_cast(ann.endpoint())).toStruct(replacements))); + EXPECT_TRUE(replacements.empty()); - expected_json = R"({"timestamp":)" + std::to_string(timestamp) + R"(,"value":")" + CLIENT_SEND + + replacements.clear(); + expected_json = R"({"timestamp":")" + std::to_string(timestamp) + R"(")" + R"(,"value":")" + + CLIENT_SEND + R"(","endpoint":{"ipv4":"192.168.1.1",)" R"("port":5555,"serviceName":"my_service_2"}})"; - EXPECT_TRUE(TestUtility::protoEqual(TestUtility::jsonToStruct(expected_json), ann.toStruct())); + EXPECT_TRUE(TestUtility::protoEqual(TestUtility::jsonToStruct(expected_json), + ann.toStruct(replacements))); + EXPECT_EQ(1, replacements.size()); + replacements.clear(); // Test change endpoint service name. ann.changeEndpointServiceName("NEW_SERVICE_NAME"); EXPECT_EQ("NEW_SERVICE_NAME", ann.endpoint().serviceName()); - expected_json = R"({"timestamp":)" + std::to_string(timestamp) + R"(,"value":")" + CLIENT_SEND + + expected_json = R"({"timestamp":")" + std::to_string(timestamp) + R"(")" + R"(,"value":")" + + CLIENT_SEND + R"(","endpoint":{"ipv4":"192.168.1.1",)" R"("port":5555,"serviceName":"NEW_SERVICE_NAME"}})"; - EXPECT_TRUE(TestUtility::protoEqual(TestUtility::jsonToStruct(expected_json), ann.toStruct())); + EXPECT_TRUE(TestUtility::protoEqual(TestUtility::jsonToStruct(expected_json), + ann.toStruct(replacements))); + EXPECT_EQ(1, replacements.size()); } TEST(ZipkinCoreTypesAnnotationTest, customConstructor) { Network::Address::InstanceConstSharedPtr addr = Network::Utility::parseInternetAddressAndPort("127.0.0.1:3306"); Endpoint ep(std::string("my_service"), addr); - DangerousDeprecatedTestTime test_time; + Event::SimulatedTimeSystem test_time; uint64_t timestamp = std::chrono::duration_cast( test_time.timeSystem().systemTime().time_since_epoch()) .count(); Annotation ann(timestamp, CLIENT_SEND, ep); + Util::Replacements replacements; EXPECT_EQ(timestamp, ann.timestamp()); EXPECT_EQ(CLIENT_SEND, ann.value()); @@ -176,30 +206,34 @@ TEST(ZipkinCoreTypesAnnotationTest, customConstructor) { EXPECT_EQ("my_service", ann.endpoint().serviceName()); EXPECT_TRUE(TestUtility::protoEqual( TestUtility::jsonToStruct(R"({"ipv4":"127.0.0.1","port":3306,"serviceName":"my_service"})"), - (const_cast(ann.endpoint())).toStruct())); + (const_cast(ann.endpoint())).toStruct(replacements))); + EXPECT_TRUE(replacements.empty()); - std::string expected_json = R"({"timestamp":)" + std::to_string(timestamp) + R"(,"value":")" + - CLIENT_SEND + + std::string expected_json = R"({"timestamp":")" + std::to_string(timestamp) + R"(")" + + R"(,"value":")" + CLIENT_SEND + R"(","endpoint":{"ipv4":"127.0.0.1",)" R"("port":3306,"serviceName":"my_service"}})"; - EXPECT_TRUE(TestUtility::protoEqual(TestUtility::jsonToStruct(expected_json), ann.toStruct())); + EXPECT_TRUE(TestUtility::protoEqual(TestUtility::jsonToStruct(expected_json), + ann.toStruct(replacements))); + EXPECT_EQ(1, replacements.size()); } TEST(ZipkinCoreTypesAnnotationTest, copyConstructor) { Network::Address::InstanceConstSharedPtr addr = Network::Utility::parseInternetAddressAndPort("127.0.0.1:3306"); Endpoint ep(std::string("my_service"), addr); - DangerousDeprecatedTestTime test_time; + Event::SimulatedTimeSystem test_time; uint64_t timestamp = std::chrono::duration_cast( test_time.timeSystem().systemTime().time_since_epoch()) .count(); Annotation ann(timestamp, CLIENT_SEND, ep); - const Annotation& ann2(ann); + Annotation& ann2(ann); + Util::Replacements replacements; EXPECT_EQ(ann.value(), ann2.value()); EXPECT_EQ(ann.timestamp(), ann2.timestamp()); EXPECT_EQ(ann.isSetEndpoint(), ann2.isSetEndpoint()); - EXPECT_TRUE(TestUtility::protoEqual(ann.toStruct(), ann2.toStruct())); + EXPECT_TRUE(TestUtility::protoEqual(ann.toStruct(replacements), ann2.toStruct(replacements))); EXPECT_EQ(ann.endpoint().serviceName(), ann2.endpoint().serviceName()); } @@ -207,22 +241,24 @@ TEST(ZipkinCoreTypesAnnotationTest, assignmentOperator) { Network::Address::InstanceConstSharedPtr addr = Network::Utility::parseInternetAddressAndPort("127.0.0.1:3306"); Endpoint ep(std::string("my_service"), addr); - DangerousDeprecatedTestTime test_time; + Event::SimulatedTimeSystem test_time; uint64_t timestamp = std::chrono::duration_cast( test_time.timeSystem().systemTime().time_since_epoch()) .count(); Annotation ann(timestamp, CLIENT_SEND, ep); - const Annotation& ann2 = ann; + Annotation& ann2 = ann; + Util::Replacements replacements; EXPECT_EQ(ann.value(), ann2.value()); EXPECT_EQ(ann.timestamp(), ann2.timestamp()); EXPECT_EQ(ann.isSetEndpoint(), ann2.isSetEndpoint()); - EXPECT_TRUE(TestUtility::protoEqual(ann.toStruct(), ann2.toStruct())); + EXPECT_TRUE(TestUtility::protoEqual(ann.toStruct(replacements), ann2.toStruct(replacements))); EXPECT_EQ(ann.endpoint().serviceName(), ann2.endpoint().serviceName()); } TEST(ZipkinCoreTypesBinaryAnnotationTest, defaultConstructor) { BinaryAnnotation ann; + Util::Replacements replacements; EXPECT_EQ("", ann.key()); EXPECT_EQ("", ann.value()); @@ -236,7 +272,8 @@ TEST(ZipkinCoreTypesBinaryAnnotationTest, defaultConstructor) { EXPECT_EQ("value", ann.value()); std::string expected_json = R"({"key":"key","value":"value"})"; - EXPECT_TRUE(TestUtility::protoEqual(TestUtility::jsonToStruct(expected_json), ann.toStruct())); + EXPECT_TRUE(TestUtility::protoEqual(TestUtility::jsonToStruct(expected_json), + ann.toStruct(replacements))); // Test the copy-semantics flavor of setEndpoint @@ -248,14 +285,15 @@ TEST(ZipkinCoreTypesBinaryAnnotationTest, defaultConstructor) { EXPECT_EQ("my_service", ann.endpoint().serviceName()); EXPECT_TRUE(TestUtility::protoEqual( TestUtility::jsonToStruct(R"({"ipv4":"127.0.0.1","port":3306,"serviceName":"my_service"})"), - (const_cast(ann.endpoint())).toStruct())); + (const_cast(ann.endpoint())).toStruct(replacements))); expected_json = "{" R"("key":"key","value":"value",)" R"("endpoint":)" R"({"ipv4":"127.0.0.1","port":3306,"serviceName":"my_service"})" "}"; - EXPECT_TRUE(TestUtility::protoEqual(TestUtility::jsonToStruct(expected_json), ann.toStruct())); + EXPECT_TRUE(TestUtility::protoEqual(TestUtility::jsonToStruct(expected_json), + ann.toStruct(replacements))); // Test the move-semantics flavor of setEndpoint addr = Network::Utility::parseInternetAddressAndPort("192.168.1.1:5555"); @@ -266,51 +304,57 @@ TEST(ZipkinCoreTypesBinaryAnnotationTest, defaultConstructor) { EXPECT_TRUE(TestUtility::protoEqual( TestUtility::jsonToStruct( R"({"ipv4":"192.168.1.1","port":5555,"serviceName":"my_service_2"})"), - (const_cast(ann.endpoint())).toStruct())); + (const_cast(ann.endpoint())).toStruct(replacements))); expected_json = "{" R"("key":"key","value":"value",)" R"("endpoint":)" R"({"ipv4":"192.168.1.1","port":5555,"serviceName":"my_service_2"})" "}"; - EXPECT_TRUE(TestUtility::protoEqual(TestUtility::jsonToStruct(expected_json), ann.toStruct())); + EXPECT_TRUE(TestUtility::protoEqual(TestUtility::jsonToStruct(expected_json), + ann.toStruct(replacements))); } TEST(ZipkinCoreTypesBinaryAnnotationTest, customConstructor) { BinaryAnnotation ann("key", "value"); + Util::Replacements replacements; EXPECT_EQ("key", ann.key()); EXPECT_EQ("value", ann.value()); EXPECT_FALSE(ann.isSetEndpoint()); EXPECT_EQ(AnnotationType::STRING, ann.annotationType()); std::string expected_json = R"({"key":"key","value":"value"})"; - EXPECT_TRUE(TestUtility::protoEqual(TestUtility::jsonToStruct(expected_json), ann.toStruct())); + EXPECT_TRUE(TestUtility::protoEqual(TestUtility::jsonToStruct(expected_json), + ann.toStruct(replacements))); } TEST(ZipkinCoreTypesBinaryAnnotationTest, copyConstructor) { BinaryAnnotation ann("key", "value"); - const BinaryAnnotation& ann2(ann); + BinaryAnnotation& ann2(ann); + Util::Replacements replacements; EXPECT_EQ(ann.value(), ann2.value()); EXPECT_EQ(ann.key(), ann2.key()); EXPECT_EQ(ann.isSetEndpoint(), ann2.isSetEndpoint()); - EXPECT_TRUE(TestUtility::protoEqual(ann.toStruct(), ann2.toStruct())); + EXPECT_TRUE(TestUtility::protoEqual(ann.toStruct(replacements), ann2.toStruct(replacements))); EXPECT_EQ(ann.annotationType(), ann2.annotationType()); } TEST(ZipkinCoreTypesBinaryAnnotationTest, assignmentOperator) { BinaryAnnotation ann("key", "value"); - const BinaryAnnotation& ann2 = ann; + BinaryAnnotation& ann2 = ann; + Util::Replacements replacements; EXPECT_EQ(ann.value(), ann2.value()); EXPECT_EQ(ann.key(), ann2.key()); EXPECT_EQ(ann.isSetEndpoint(), ann2.isSetEndpoint()); - EXPECT_TRUE(TestUtility::protoEqual(ann.toStruct(), ann2.toStruct())); + EXPECT_TRUE(TestUtility::protoEqual(ann.toStruct(replacements), ann2.toStruct(replacements))); EXPECT_EQ(ann.annotationType(), ann2.annotationType()); } TEST(ZipkinCoreTypesSpanTest, defaultConstructor) { - DangerousDeprecatedTestTime test_time; + Event::SimulatedTimeSystem test_time; Span span(test_time.timeSystem()); + Util::Replacements replacements; EXPECT_EQ(0ULL, span.id()); EXPECT_EQ(0ULL, span.traceId()); @@ -329,7 +373,7 @@ TEST(ZipkinCoreTypesSpanTest, defaultConstructor) { EXPECT_TRUE(TestUtility::protoEqual( TestUtility::jsonToStruct( R"({"traceId":"0000000000000000","name":"","id":"0000000000000000"})"), - span.toStruct())); + span.toStruct(replacements))); uint64_t id = Util::generateRandom64(test_time.timeSystem()); std::string id_hex = Hex::uint64ToHex(id); @@ -410,16 +454,19 @@ TEST(ZipkinCoreTypesSpanTest, defaultConstructor) { TestUtility::jsonToStruct( R"({"traceId":")" + span.traceIdAsHexString() + R"(","name":"span_name","id":")" + span.idAsHexString() + R"(","parentId":")" + span.parentIdAsHexString() + - R"(","timestamp":)" + std::to_string(span.timestamp()) + - R"(,"duration":3000,)" + R"(","timestamp":")" + std::to_string(span.timestamp()) + + R"(")" + R"(,"duration":"3000",)" R"("annotations":[)" - R"({"timestamp":)" + + R"({"timestamp":")" + std::to_string(span.timestamp()) + + R"(")" R"(,"value":"cs","endpoint":)" R"({"ipv4":"192.168.1.2","port":3306,"serviceName":"my_service_name"}}],)" R"("binaryAnnotations":[{"key":"lc","value":"my_component_name","endpoint":)" R"({"ipv4":"192.168.1.2","port":3306,"serviceName":"my_service_name"}}]})"), - span.toStruct())); + span.toStruct(replacements))); + EXPECT_EQ(3, replacements.size()); // Test the copy-semantics flavor of addAnnotation and addBinaryAnnotation @@ -444,24 +491,29 @@ TEST(ZipkinCoreTypesSpanTest, defaultConstructor) { EXPECT_EQ(3ULL, span.annotations().size()); EXPECT_EQ(3ULL, span.binaryAnnotations().size()); + replacements.clear(); EXPECT_TRUE(TestUtility::protoEqual( TestUtility::jsonToStruct( R"({"traceId":")" + span.traceIdAsHexString() + R"(","name":"span_name","id":")" + span.idAsHexString() + R"(","parentId":")" + span.parentIdAsHexString() + - R"(","timestamp":)" + std::to_string(span.timestamp()) + - R"(,"duration":3000,)" + R"(","timestamp":")" + std::to_string(span.timestamp()) + + R"(")" + R"(,"duration":"3000",)" R"("annotations":[)" - R"({"timestamp":)" + + R"({"timestamp":")" + std::to_string(timestamp) + + R"(")" R"(,"value":"cs","endpoint":)" R"({"ipv4":"192.168.1.2","port":3306,"serviceName":"my_service_name"}},)" - R"({"timestamp":)" + + R"({"timestamp":")" + std::to_string(timestamp) + + R"(")" R"(,"value":"ss",)" R"("endpoint":{"ipv4":"192.168.1.2","port":3306,)" R"("serviceName":"my_service_name"}},)" - R"({"timestamp":)" + + R"({"timestamp":")" + std::to_string(timestamp) + + R"(")" R"(,"value":"sr","endpoint":{"ipv4":"192.168.1.2","port":3306,)" R"("serviceName":"my_service_name"}}],)" R"("binaryAnnotations":[{"key":"lc","value":"my_component_name",)" @@ -473,35 +525,42 @@ TEST(ZipkinCoreTypesSpanTest, defaultConstructor) { R"({"key":"http.return_code","value":"400",)" R"("endpoint":{"ipv4":"192.168.1.2","port":3306,)" R"("serviceName":"my_service_name"}}]})"), - span.toStruct())); + span.toStruct(replacements))); + EXPECT_EQ(5, replacements.size()); // Test setSourceServiceName and setDestinationServiceName ann_copy.setValue(CLIENT_RECV); span.addAnnotation(ann_copy); span.setServiceName("NEW_SERVICE_NAME"); + replacements.clear(); EXPECT_TRUE(TestUtility::protoEqual( TestUtility::jsonToStruct( R"({"traceId":")" + span.traceIdAsHexString() + R"(","name":"span_name","id":")" + span.idAsHexString() + R"(","parentId":")" + span.parentIdAsHexString() + - R"(","timestamp":)" + std::to_string(span.timestamp()) + - R"(,"duration":3000,)" + R"(","timestamp":")" + std::to_string(span.timestamp()) + + R"(")" + R"(,"duration":"3000",)" R"("annotations":[)" - R"({"timestamp":)" + + R"({"timestamp":")" + std::to_string(timestamp) + + R"(")" R"(,"value":"cs","endpoint":)" R"({"ipv4":"192.168.1.2","port":3306,"serviceName":"NEW_SERVICE_NAME"}},)" - R"({"timestamp":)" + + R"({"timestamp":")" + std::to_string(timestamp) + + R"(")" R"(,"value":"ss",)" R"("endpoint":{"ipv4":"192.168.1.2","port":3306,)" R"("serviceName":"NEW_SERVICE_NAME"}},)" - R"({"timestamp":)" + + R"({"timestamp":")" + std::to_string(timestamp) + + R"(")" R"(,"value":"sr","endpoint":{"ipv4":"192.168.1.2","port":3306,)" R"("serviceName":"NEW_SERVICE_NAME"}},)" - R"({"timestamp":)" + + R"({"timestamp":")" + std::to_string(timestamp) + + R"(")" R"(,"value":"cr","endpoint":)" R"({"ipv4":"192.168.1.2","port":3306,"serviceName":"NEW_SERVICE_NAME"}}],)" R"("binaryAnnotations":[{"key":"lc","value":"my_component_name",)" @@ -513,12 +572,14 @@ TEST(ZipkinCoreTypesSpanTest, defaultConstructor) { R"({"key":"http.return_code","value":"400",)" R"("endpoint":{"ipv4":"192.168.1.2","port":3306,)" R"("serviceName":"my_service_name"}}]})"), - span.toStruct())); + span.toStruct(replacements))); + EXPECT_EQ(6, replacements.size()); } TEST(ZipkinCoreTypesSpanTest, copyConstructor) { - DangerousDeprecatedTestTime test_time; + Event::SimulatedTimeSystem test_time; Span span(test_time.timeSystem()); + Util::Replacements replacements; uint64_t id = Util::generateRandom64(test_time.timeSystem()); std::string id_hex = Hex::uint64ToHex(id); @@ -554,8 +615,9 @@ TEST(ZipkinCoreTypesSpanTest, copyConstructor) { } TEST(ZipkinCoreTypesSpanTest, assignmentOperator) { - DangerousDeprecatedTestTime test_time; + Event::SimulatedTimeSystem test_time; Span span(test_time.timeSystem()); + Util::Replacements replacements; uint64_t id = Util::generateRandom64(test_time.timeSystem()); std::string id_hex = Hex::uint64ToHex(id); @@ -591,7 +653,7 @@ TEST(ZipkinCoreTypesSpanTest, assignmentOperator) { } TEST(ZipkinCoreTypesSpanTest, setTag) { - DangerousDeprecatedTestTime test_time; + Event::SimulatedTimeSystem test_time; Span span(test_time.timeSystem()); span.setTag("key1", "value1"); diff --git a/test/extensions/tracers/zipkin/zipkin_tracer_impl_test.cc b/test/extensions/tracers/zipkin/zipkin_tracer_impl_test.cc index 477d8ce6adef..96078b70c898 100644 --- a/test/extensions/tracers/zipkin/zipkin_tracer_impl_test.cc +++ b/test/extensions/tracers/zipkin/zipkin_tracer_impl_test.cc @@ -4,7 +4,7 @@ #include #include -#include "envoy/config/trace/v3/trace.pb.h" +#include "envoy/config/trace/v3/zipkin.pb.h" #include "common/http/header_map_impl.h" #include "common/http/headers.h" diff --git a/test/integration/BUILD b/test/integration/BUILD index 0f59370e8741..24fbc6f4b806 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -270,7 +270,6 @@ envoy_cc_test( "//source/common/buffer:buffer_lib", "//source/common/http:header_map_lib", "//source/extensions/filters/http/buffer:config", - "//source/extensions/filters/http/dynamo:config", "//source/extensions/filters/http/health_check:config", "//test/common/http/http2:http2_frame", "//test/integration/filters:metadata_stop_all_filter_config_lib", @@ -357,8 +356,8 @@ envoy_cc_test( ":http_protocol_integration_lib", "//source/common/http:header_map_lib", "//source/extensions/filters/http/buffer:config", - "//source/extensions/filters/http/dynamo:config", "//source/extensions/filters/http/health_check:config", + "//test/integration/filters:encoder_decoder_buffer_filter_lib", "//test/integration/filters:random_pause_filter_lib", "//test/test_common:utility_lib", "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", @@ -378,8 +377,8 @@ envoy_cc_test( "//source/common/http:header_map_lib", "//source/extensions/access_loggers/grpc:http_config", "//source/extensions/filters/http/buffer:config", - "//source/extensions/filters/http/dynamo:config", "//source/extensions/filters/http/health_check:config", + "//test/integration/filters:encoder_decoder_buffer_filter_lib", "//test/integration/filters:random_pause_filter_lib", "//test/test_common:utility_lib", "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", @@ -583,10 +582,10 @@ envoy_cc_test( "//source/common/http:header_map_lib", "//source/common/http:headers_lib", "//source/extensions/filters/http/cors:config", - "//source/extensions/filters/http/dynamo:config", "//source/extensions/filters/http/grpc_http1_bridge:config", "//source/extensions/filters/http/health_check:config", "//test/integration/filters:clear_route_cache_filter_lib", + "//test/integration/filters:encoder_decoder_buffer_filter_lib", "//test/integration/filters:process_context_lib", "//test/mocks/http:http_mocks", "//test/test_common:utility_lib", diff --git a/test/integration/fake_upstream.cc b/test/integration/fake_upstream.cc index b65a4ce33302..8562edc01ca3 100644 --- a/test/integration/fake_upstream.cc +++ b/test/integration/fake_upstream.cc @@ -463,7 +463,7 @@ void FakeUpstream::createUdpListenerFilterChain(Network::UdpListenerFilterManage } void FakeUpstream::threadRoutine() { - handler_->addListener(listener_); + handler_->addListener(absl::nullopt, listener_); server_initialized_.setReady(); dispatcher_->run(Event::Dispatcher::RunType::Block); handler_.reset(); @@ -502,7 +502,9 @@ AssertionResult FakeUpstream::waitForHttpConnection( max_request_headers_count, headers_with_underscores_action); } VERIFY_ASSERTION(connection->initialize()); - VERIFY_ASSERTION(connection->readDisable(false)); + if (read_disable_on_new_connection_) { + VERIFY_ASSERTION(connection->readDisable(false)); + } return AssertionSuccess(); } diff --git a/test/integration/filters/BUILD b/test/integration/filters/BUILD index 5c8e5aa3584d..b26d5ffb1835 100644 --- a/test/integration/filters/BUILD +++ b/test/integration/filters/BUILD @@ -261,3 +261,17 @@ envoy_cc_test_library( "//test/extensions/filters/http/common:empty_http_filter_config_lib", ], ) + +envoy_cc_test_library( + name = "encoder_decoder_buffer_filter_lib", + srcs = [ + "encoder_decoder_buffer_filter.cc", + ], + deps = [ + "//include/envoy/http:filter_interface", + "//include/envoy/registry", + "//include/envoy/server:filter_config_interface", + "//source/extensions/filters/http/common:pass_through_filter_lib", + "//test/extensions/filters/http/common:empty_http_filter_config_lib", + ], +) diff --git a/test/integration/filters/encoder_decoder_buffer_filter.cc b/test/integration/filters/encoder_decoder_buffer_filter.cc new file mode 100644 index 000000000000..22050f5cfcf3 --- /dev/null +++ b/test/integration/filters/encoder_decoder_buffer_filter.cc @@ -0,0 +1,55 @@ +#include + +#include "envoy/http/filter.h" +#include "envoy/registry/registry.h" +#include "envoy/server/filter_config.h" + +#include "extensions/filters/http/common/pass_through_filter.h" + +#include "test/extensions/filters/http/common/empty_http_filter_config.h" + +namespace Envoy { + +// A filter that buffers the entire request/response. +class EncoderDecoderBufferStreamFilter : public Http::PassThroughFilter { +public: + Http::FilterHeadersStatus decodeHeaders(Http::RequestHeaderMap&, bool end_stream) override { + return end_stream ? Http::FilterHeadersStatus::Continue + : Http::FilterHeadersStatus::StopIteration; + } + + Http::FilterDataStatus decodeData(Buffer::Instance&, bool end_stream) override { + return end_stream ? Http::FilterDataStatus::Continue + : Http::FilterDataStatus::StopIterationAndBuffer; + } + + Http::FilterHeadersStatus encodeHeaders(Http::ResponseHeaderMap&, bool end_stream) override { + return end_stream ? Http::FilterHeadersStatus::Continue + : Http::FilterHeadersStatus::StopIteration; + } + + Http::FilterDataStatus encodeData(Buffer::Instance&, bool end_stream) override { + return end_stream ? Http::FilterDataStatus::Continue + : Http::FilterDataStatus::StopIterationAndBuffer; + } +}; + +class EncoderDecoderBuffferFilterConfig + : public Extensions::HttpFilters::Common::EmptyHttpFilterConfig { +public: + EncoderDecoderBuffferFilterConfig() : EmptyHttpFilterConfig("encoder-decoder-buffer-filter") {} + + Http::FilterFactoryCb createFilter(const std::string&, + Server::Configuration::FactoryContext&) override { + return [](Http::FilterChainFactoryCallbacks& callbacks) -> void { + callbacks.addStreamFilter(std::make_shared<::Envoy::EncoderDecoderBufferStreamFilter>()); + }; + } +}; + +// perform static registration +static Registry::RegisterFactory + register_; + +} // namespace Envoy diff --git a/test/integration/http2_upstream_integration_test.cc b/test/integration/http2_upstream_integration_test.cc index 70e05547cb1f..37cda22965f2 100644 --- a/test/integration/http2_upstream_integration_test.cc +++ b/test/integration/http2_upstream_integration_test.cc @@ -345,7 +345,7 @@ name: router // As with ProtocolIntegrationTest.HittingEncoderFilterLimit use a filter // which buffers response data but in this case, make sure the sendLocalReply // is gRPC. - config_helper_.addFilter("{ name: envoy.filters.http.dynamo, typed_config: { \"@type\": " + config_helper_.addFilter("{ name: encoder-decoder-buffer-filter, typed_config: { \"@type\": " "type.googleapis.com/google.protobuf.Empty } }"); config_helper_.setBufferLimits(1024, 1024); initialize(); @@ -360,7 +360,7 @@ name: router {"te", "trailers"}}); auto downstream_request = &encoder_decoder.first; auto response = std::move(encoder_decoder.second); - Buffer::OwnedImpl data(R"({"TableName":"locations"})"); + Buffer::OwnedImpl data("HTTP body content goes here"); codec_client_->sendData(*downstream_request, data, true); waitForNextUpstreamRequest(); diff --git a/test/integration/integration_test.cc b/test/integration/integration_test.cc index ec5125a1cd95..a38c97817102 100644 --- a/test/integration/integration_test.cc +++ b/test/integration/integration_test.cc @@ -954,8 +954,8 @@ TEST_P(IntegrationTest, ViaAppendWith100Continue) { TEST_P(IntegrationTest, TestDelayedConnectionTeardownOnGracefulClose) { // This test will trigger an early 413 Payload Too Large response due to buffer limits being // exceeded. The following filter is needed since the router filter will never trigger a 413. - config_helper_.addFilter("{ name: http_dynamo_filter, typed_config: { \"@type\": " - "type.googleapis.com/envoy.config.filter.http.dynamo.v2.Dynamo } }"); + config_helper_.addFilter("{ name: encoder-decoder-buffer-filter, typed_config: { \"@type\": " + "type.googleapis.com/google.protobuf.Empty } }"); config_helper_.setBufferLimits(1024, 1024); initialize(); @@ -989,8 +989,8 @@ TEST_P(IntegrationTest, TestDelayedConnectionTeardownOnGracefulClose) { // Test configuration of the delayed close timeout on downstream HTTP/1.1 connections. A value of 0 // disables delayed close processing. TEST_P(IntegrationTest, TestDelayedConnectionTeardownConfig) { - config_helper_.addFilter("{ name: http_dynamo_filter, typed_config: { \"@type\": " - "type.googleapis.com/envoy.config.filter.http.dynamo.v2.Dynamo } }"); + config_helper_.addFilter("{ name: encoder-decoder-buffer-filter, typed_config: { \"@type\": " + "type.googleapis.com/google.protobuf.Empty } }"); config_helper_.setBufferLimits(1024, 1024); config_helper_.addConfigModifier( [](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& @@ -1025,8 +1025,8 @@ TEST_P(IntegrationTest, TestDelayedConnectionTeardownConfig) { // Test that delay closed connections are eventually force closed when the timeout triggers. TEST_P(IntegrationTest, TestDelayedConnectionTeardownTimeoutTrigger) { - config_helper_.addFilter("{ name: http_dynamo_filter, typed_config: { \"@type\": " - "type.googleapis.com/envoy.config.filter.http.dynamo.v2.Dynamo } }"); + config_helper_.addFilter("{ name: encoder-decoder-buffer-filter, typed_config: { \"@type\": " + "type.googleapis.com/google.protobuf.Empty } }"); config_helper_.setBufferLimits(1024, 1024); config_helper_.addConfigModifier( [](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& diff --git a/test/integration/protocol_integration_test.cc b/test/integration/protocol_integration_test.cc index d051884e2629..e1bc4a9ef727 100644 --- a/test/integration/protocol_integration_test.cc +++ b/test/integration/protocol_integration_test.cc @@ -299,6 +299,117 @@ TEST_P(ProtocolIntegrationTest, Retry) { EXPECT_EQ(512U, response->body().size()); } +TEST_P(ProtocolIntegrationTest, RetryStreaming) { + initialize(); + codec_client_ = makeHttpConnection(lookupPort("http")); + auto encoder_decoder = + codec_client_->startRequest(Http::TestRequestHeaderMapImpl{{":method", "POST"}, + {":path", "/test/long/url"}, + {":scheme", "http"}, + {":authority", "host"}, + {"x-forwarded-for", "10.0.0.1"}, + {"x-envoy-retry-on", "5xx"}}); + auto& encoder = encoder_decoder.first; + auto& response = encoder_decoder.second; + + // Send some data, but not the entire body. + std::string data(1024, 'a'); + Buffer::OwnedImpl send1(data); + encoder.encodeData(send1, false); + + ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_)); + ASSERT_TRUE(fake_upstream_connection_->waitForNewStream(*dispatcher_, upstream_request_)); + + // Send back an upstream failure. + upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "503"}}, false); + + if (fake_upstreams_[0]->httpType() == FakeHttpConnection::Type::HTTP1) { + ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); + ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_)); + } else { + ASSERT_TRUE(upstream_request_->waitForReset()); + } + + // Wait for a retry. Ensure all data, both before and after the retry, is received. + ASSERT_TRUE(fake_upstream_connection_->waitForNewStream(*dispatcher_, upstream_request_)); + + // Finish the request. + std::string data2(512, 'b'); + Buffer::OwnedImpl send2(data2); + encoder.encodeData(send2, true); + std::string combined_request_data = data + data2; + ASSERT_TRUE(upstream_request_->waitForData(*dispatcher_, combined_request_data)); + + upstream_request_->encodeHeaders(default_response_headers_, false); + upstream_request_->encodeData(512, true); + + response->waitForEndStream(); + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_EQ(combined_request_data.size(), upstream_request_->bodyLength()); + + EXPECT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().Status()->value().getStringView()); + EXPECT_EQ(512U, response->body().size()); +} + +TEST_P(ProtocolIntegrationTest, RetryStreamingCancelDueToBufferOverflow) { + config_helper_.addConfigModifier( + [](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& + hcm) { + auto* route = hcm.mutable_route_config()->mutable_virtual_hosts(0)->mutable_routes(0); + + route->mutable_per_request_buffer_limit_bytes()->set_value(1024); + route->mutable_route() + ->mutable_retry_policy() + ->mutable_retry_back_off() + ->mutable_base_interval() + ->MergeFrom( + ProtobufUtil::TimeUtil::MillisecondsToDuration(100000000)); // Effectively infinity. + }); + initialize(); + + codec_client_ = makeHttpConnection(lookupPort("http")); + auto encoder_decoder = + codec_client_->startRequest(Http::TestRequestHeaderMapImpl{{":method", "POST"}, + {":path", "/test/long/url"}, + {":scheme", "http"}, + {":authority", "host"}, + {"x-forwarded-for", "10.0.0.1"}, + {"x-envoy-retry-on", "5xx"}}); + auto& encoder = encoder_decoder.first; + auto& response = encoder_decoder.second; + + // Send some data, but less than the buffer limit, and not end-stream + std::string data(64, 'a'); + Buffer::OwnedImpl send1(data); + encoder.encodeData(send1, false); + + ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_)); + ASSERT_TRUE(fake_upstream_connection_->waitForNewStream(*dispatcher_, upstream_request_)); + + // Send back an upstream failure. + upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "503"}}, false); + + if (fake_upstreams_[0]->httpType() == FakeHttpConnection::Type::HTTP1) { + ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); + } else { + ASSERT_TRUE(upstream_request_->waitForReset()); + } + + // Overflow the request buffer limit. Because the retry base interval is infinity, no + // request will be in progress. This will cause the request to be aborted and an error + // to be returned to the client. + std::string data2(2048, 'b'); + Buffer::OwnedImpl send2(data2); + encoder.encodeData(send2, false); + + response->waitForEndStream(); + + EXPECT_TRUE(response->complete()); + EXPECT_EQ("507", response->headers().Status()->value().getStringView()); + test_server_->waitForCounterEq("cluster.cluster_0.retry_or_shadow_abandoned", 1); +} + // Tests that the x-envoy-attempt-count header is properly set on the upstream request and the // downstream response, and updated after the request is retried. TEST_P(DownstreamProtocolIntegrationTest, RetryAttemptCountHeader) { @@ -539,10 +650,10 @@ TEST_P(ProtocolIntegrationTest, RetryHittingRouteLimits) { EXPECT_EQ("503", response->headers().Status()->value().getStringView()); } -// Test hitting the dynamo filter with too many request bytes to buffer. Ensure the connection -// manager sends a 413. +// Test hitting the decoder buffer filter with too many request bytes to buffer. Ensure the +// connection manager sends a 413. TEST_P(DownstreamProtocolIntegrationTest, HittingDecoderFilterLimit) { - config_helper_.addFilter("{ name: envoy.filters.http.dynamo, typed_config: { \"@type\": " + config_helper_.addFilter("{ name: encoder-decoder-buffer-filter, typed_config: { \"@type\": " "type.googleapis.com/google.protobuf.Empty } }"); config_helper_.setBufferLimits(1024, 1024); initialize(); @@ -574,11 +685,11 @@ TEST_P(DownstreamProtocolIntegrationTest, HittingDecoderFilterLimit) { } } -// Test hitting the dynamo filter with too many response bytes to buffer. Given the request headers -// are sent on early, the stream/connection will be reset. +// Test hitting the encoder buffer filter with too many response bytes to buffer. Given the request +// headers are sent on early, the stream/connection will be reset. TEST_P(ProtocolIntegrationTest, HittingEncoderFilterLimit) { useAccessLog(); - config_helper_.addFilter("{ name: envoy.filters.http.dynamo, typed_config: { \"@type\": " + config_helper_.addFilter("{ name: encoder-decoder-buffer-filter, typed_config: { \"@type\": " "type.googleapis.com/google.protobuf.Empty } }"); config_helper_.setBufferLimits(1024, 1024); initialize(); @@ -588,7 +699,7 @@ TEST_P(ProtocolIntegrationTest, HittingEncoderFilterLimit) { auto encoder_decoder = codec_client_->startRequest(default_request_headers_); auto downstream_request = &encoder_decoder.first; auto response = std::move(encoder_decoder.second); - Buffer::OwnedImpl data(R"({"TableName":"locations"})"); + Buffer::OwnedImpl data("HTTP body content goes here"); codec_client_->sendData(*downstream_request, data, true); waitForNextUpstreamRequest(); diff --git a/test/integration/server.h b/test/integration/server.h index 13dfbe451147..22f80a09b57b 100644 --- a/test/integration/server.h +++ b/test/integration/server.h @@ -98,6 +98,13 @@ class TestScopeWrapper : public Scope { Thread::LockGuard lock(lock_); return wrapped_scope_->histogramFromStatNameWithTags(name, tags, unit); } + + TextReadout& textReadoutFromStatNameWithTags(const StatName& name, + StatNameTagVectorOptConstRef tags) override { + Thread::LockGuard lock(lock_); + return wrapped_scope_->textReadoutFromStatNameWithTags(name, tags); + } + NullGaugeImpl& nullGauge(const std::string& str) override { return wrapped_scope_->nullGauge(str); } @@ -114,6 +121,10 @@ class TestScopeWrapper : public Scope { StatNameManagedStorage storage(name, symbolTable()); return histogramFromStatName(storage.statName(), unit); } + TextReadout& textReadoutFromString(const std::string& name) override { + StatNameManagedStorage storage(name, symbolTable()); + return textReadoutFromStatName(storage.statName()); + } CounterOptConstRef findCounter(StatName name) const override { Thread::LockGuard lock(lock_); @@ -127,6 +138,10 @@ class TestScopeWrapper : public Scope { Thread::LockGuard lock(lock_); return wrapped_scope_->findHistogram(name); } + TextReadoutOptConstRef findTextReadout(StatName name) const override { + Thread::LockGuard lock(lock_); + return wrapped_scope_->findTextReadout(name); + } const SymbolTable& constSymbolTable() const override { return wrapped_scope_->constSymbolTable(); @@ -178,6 +193,15 @@ class TestIsolatedStoreImpl : public StoreRoot { Thread::LockGuard lock(lock_); return store_.histogramFromString(name, unit); } + TextReadout& textReadoutFromStatNameWithTags(const StatName& name, + StatNameTagVectorOptConstRef tags) override { + Thread::LockGuard lock(lock_); + return store_.textReadoutFromStatNameWithTags(name, tags); + } + TextReadout& textReadoutFromString(const std::string& name) override { + Thread::LockGuard lock(lock_); + return store_.textReadoutFromString(name); + } CounterOptConstRef findCounter(StatName name) const override { Thread::LockGuard lock(lock_); return store_.findCounter(name); @@ -190,6 +214,10 @@ class TestIsolatedStoreImpl : public StoreRoot { Thread::LockGuard lock(lock_); return store_.findHistogram(name); } + TextReadoutOptConstRef findTextReadout(StatName name) const override { + Thread::LockGuard lock(lock_); + return store_.findTextReadout(name); + } const SymbolTable& constSymbolTable() const override { return store_.constSymbolTable(); } SymbolTable& symbolTable() override { return store_.symbolTable(); } @@ -202,11 +230,14 @@ class TestIsolatedStoreImpl : public StoreRoot { Thread::LockGuard lock(lock_); return store_.gauges(); } - std::vector histograms() const override { Thread::LockGuard lock(lock_); return store_.histograms(); } + std::vector textReadouts() const override { + Thread::LockGuard lock(lock_); + return store_.textReadouts(); + } // Stats::StoreRoot void addSink(Sink&) override {} diff --git a/test/integration/ssl_utility.cc b/test/integration/ssl_utility.cc index 7d9b710e16ca..af54c2cdb40a 100644 --- a/test/integration/ssl_utility.cc +++ b/test/integration/ssl_utility.cc @@ -64,6 +64,9 @@ createClientSslTransportSocketFactory(const ClientSslTransportOptions& options, for (const std::string& cipher_suite : options.cipher_suites_) { common_context->mutable_tls_params()->add_cipher_suites(cipher_suite); } + if (!options.sni_.empty()) { + tls_context.set_sni(options.sni_); + } common_context->mutable_tls_params()->set_tls_minimum_protocol_version(options.tls_version_); common_context->mutable_tls_params()->set_tls_maximum_protocol_version(options.tls_version_); @@ -93,6 +96,24 @@ Network::TransportSocketFactoryPtr createUpstreamSslContext(ContextManager& cont std::move(cfg), context_manager, *upstream_stats_store, std::vector{}); } +Network::TransportSocketFactoryPtr createFakeUpstreamSslContext( + const std::string& upstream_cert_name, ContextManager& context_manager, + Server::Configuration::TransportSocketFactoryContext& factory_context) { + envoy::extensions::transport_sockets::tls::v3::DownstreamTlsContext tls_context; + auto* common_tls_context = tls_context.mutable_common_tls_context(); + auto* tls_cert = common_tls_context->add_tls_certificates(); + tls_cert->mutable_certificate_chain()->set_filename(TestEnvironment::runfilesPath( + fmt::format("test/config/integration/certs/{}cert.pem", upstream_cert_name))); + tls_cert->mutable_private_key()->set_filename(TestEnvironment::runfilesPath( + fmt::format("test/config/integration/certs/{}key.pem", upstream_cert_name))); + + auto cfg = std::make_unique( + tls_context, factory_context); + + static Stats::Scope* upstream_stats_store = new Stats::IsolatedStoreImpl(); + return std::make_unique( + std::move(cfg), context_manager, *upstream_stats_store, std::vector{}); +} Network::Address::InstanceConstSharedPtr getSslAddress(const Network::Address::IpVersion& version, int port) { std::string url = diff --git a/test/integration/ssl_utility.h b/test/integration/ssl_utility.h index 8826db3cb4e3..afaf57eae00c 100644 --- a/test/integration/ssl_utility.h +++ b/test/integration/ssl_utility.h @@ -42,11 +42,17 @@ struct ClientSslTransportOptions { return *this; } + ClientSslTransportOptions& setSni(absl::string_view sni) { + sni_ = std::string(sni); + return *this; + } + bool alpn_{}; bool san_{}; bool client_ecdsa_cert_{}; std::vector cipher_suites_{}; std::string sigalgs_; + std::string sni_; envoy::extensions::transport_sockets::tls::v3::TlsParameters::TlsProtocol tls_version_{ envoy::extensions::transport_sockets::tls::v3::TlsParameters::TLS_AUTO}; }; @@ -54,9 +60,14 @@ struct ClientSslTransportOptions { Network::TransportSocketFactoryPtr createClientSslTransportSocketFactory(const ClientSslTransportOptions& options, ContextManager& context_manager, Api::Api& api); + Network::TransportSocketFactoryPtr createUpstreamSslContext(ContextManager& context_manager, Api::Api& api); +Network::TransportSocketFactoryPtr +createFakeUpstreamSslContext(const std::string& upstream_cert_name, ContextManager& context_manager, + Server::Configuration::TransportSocketFactoryContext& factory_context); + Network::Address::InstanceConstSharedPtr getSslAddress(const Network::Address::IpVersion& version, int port); diff --git a/test/integration/stats_integration_test.cc b/test/integration/stats_integration_test.cc index c08092a4b47d..75012bd02116 100644 --- a/test/integration/stats_integration_test.cc +++ b/test/integration/stats_integration_test.cc @@ -284,8 +284,8 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithFakeSymbolTable) { // If you encounter a failure here, please see // https://github.com/envoyproxy/envoy/blob/master/source/docs/stats.md#stats-memory-tests // for details on how to fix. - EXPECT_MEMORY_EQ(m_per_cluster, 43349); - EXPECT_MEMORY_LE(m_per_cluster, 44000); + EXPECT_MEMORY_EQ(m_per_cluster, 43993); + EXPECT_MEMORY_LE(m_per_cluster, 44100); } TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithRealSymbolTable) { @@ -342,8 +342,8 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithRealSymbolTable) { // If you encounter a failure here, please see // https://github.com/envoyproxy/envoy/blob/master/source/docs/stats.md#stats-memory-tests // for details on how to fix. - EXPECT_MEMORY_EQ(m_per_cluster, 35557); - EXPECT_MEMORY_LE(m_per_cluster, 36000); + EXPECT_MEMORY_EQ(m_per_cluster, 36201); + EXPECT_MEMORY_LE(m_per_cluster, 36300); } TEST_P(ClusterMemoryTestRunner, MemoryLargeHostSizeWithStats) { diff --git a/test/integration/websocket_integration_test.cc b/test/integration/websocket_integration_test.cc index 828dd658f2c8..f66eddbf02de 100644 --- a/test/integration/websocket_integration_test.cc +++ b/test/integration/websocket_integration_test.cc @@ -31,19 +31,12 @@ Http::TestRequestHeaderMapImpl upgradeRequestHeaders(const char* upgrade_type = } Http::TestResponseHeaderMapImpl upgradeResponseHeaders(const char* upgrade_type = "websocket") { - return Http::TestResponseHeaderMapImpl{{":status", "101"}, - {"connection", "upgrade"}, - {"upgrade", upgrade_type}, - {"content-length", "0"}}; + return Http::TestResponseHeaderMapImpl{ + {":status", "101"}, {"connection", "upgrade"}, {"upgrade", upgrade_type}}; } template void commonValidate(ProxiedHeaders& proxied_headers, const OriginalHeaders& original_headers) { - // 0 byte content lengths may be stripped on the H2 path - ignore that as a difference by adding - // it back to the proxied headers. - if (original_headers.ContentLength() && proxied_headers.ContentLength() == nullptr) { - proxied_headers.setContentLength(size_t(0)); - } // If no content length is specified, the HTTP1 codec will add a chunked encoding header. if (original_headers.ContentLength() == nullptr && proxied_headers.TransferEncoding() != nullptr) { @@ -82,6 +75,13 @@ void WebsocketIntegrationTest::validateUpgradeRequestHeaders( proxied_request_headers.setScheme("http"); } + // 0 byte content lengths may be stripped on the H2 path - ignore that as a difference by adding + // it back to the proxied headers. + if (original_request_headers.ContentLength() && + proxied_request_headers.ContentLength() == nullptr) { + proxied_request_headers.setContentLength(size_t(0)); + } + commonValidate(proxied_request_headers, original_request_headers); proxied_request_headers.removeRequestId(); diff --git a/test/mocks/http/stream_decoder.cc b/test/mocks/http/stream_decoder.cc index 1c40a2502f11..e4bf5ef3958d 100644 --- a/test/mocks/http/stream_decoder.cc +++ b/test/mocks/http/stream_decoder.cc @@ -13,7 +13,7 @@ MockRequestDecoder::MockRequestDecoder() { ON_CALL(*this, decodeHeaders_(_, _)).WillByDefault(Invoke([](RequestHeaderMapPtr& headers, bool) { // Check to see that method is not-null. Path can be null for CONNECT and authority can be null // at the codec level. - ASSERT(headers->Method() != nullptr); + ASSERT_NE(nullptr, headers->Method()); })); } MockRequestDecoder::~MockRequestDecoder() = default; @@ -21,7 +21,7 @@ MockRequestDecoder::~MockRequestDecoder() = default; MockResponseDecoder::MockResponseDecoder() { ON_CALL(*this, decodeHeaders_(_, _)) .WillByDefault(Invoke( - [](ResponseHeaderMapPtr& headers, bool) { ASSERT(headers->Status() != nullptr); })); + [](ResponseHeaderMapPtr& headers, bool) { ASSERT_NE(nullptr, headers->Status()); })); } MockResponseDecoder::~MockResponseDecoder() = default; diff --git a/test/mocks/http/stream_encoder.cc b/test/mocks/http/stream_encoder.cc index a76fffaace59..0c13a2ebe340 100644 --- a/test/mocks/http/stream_encoder.cc +++ b/test/mocks/http/stream_encoder.cc @@ -20,7 +20,7 @@ MockRequestEncoder::MockRequestEncoder() { .WillByDefault(Invoke([](const RequestHeaderMap& headers, bool) { // Check to see that method is not-null. Path can be null for CONNECT and authority can be // null at the codec level. - ASSERT(headers.Method() != nullptr); + ASSERT_NE(nullptr, headers.Method()); })); } MockRequestEncoder::~MockRequestEncoder() = default; @@ -29,7 +29,7 @@ MockResponseEncoder::MockResponseEncoder() { ON_CALL(*this, encodeHeaders(_, _)) .WillByDefault(Invoke([](const ResponseHeaderMap& headers, bool) { // Check for passing request headers as response headers in a test. - ASSERT(headers.Status() != nullptr); + ASSERT_NE(nullptr, headers.Status()); })); } MockResponseEncoder::~MockResponseEncoder() = default; diff --git a/test/mocks/network/mocks.h b/test/mocks/network/mocks.h index 3efea4912b35..b7537a4b9f74 100644 --- a/test/mocks/network/mocks.h +++ b/test/mocks/network/mocks.h @@ -355,8 +355,12 @@ class MockConnectionHandler : public ConnectionHandler { MOCK_METHOD(uint64_t, numConnections, (), (const)); MOCK_METHOD(void, incNumConnections, ()); MOCK_METHOD(void, decNumConnections, ()); - MOCK_METHOD(void, addListener, (ListenerConfig & config)); + MOCK_METHOD(void, addListener, + (absl::optional overridden_listener, ListenerConfig& config)); MOCK_METHOD(void, removeListeners, (uint64_t listener_tag)); + MOCK_METHOD(void, removeFilterChains, + (uint64_t listener_tag, const std::list& filter_chains, + std::function completion)); MOCK_METHOD(void, stopListeners, (uint64_t listener_tag)); MOCK_METHOD(void, stopListeners, ()); MOCK_METHOD(void, disableListeners, ()); diff --git a/test/mocks/server/mocks.cc b/test/mocks/server/mocks.cc index e9bea5ab1990..148fd24e9da5 100644 --- a/test/mocks/server/mocks.cc +++ b/test/mocks/server/mocks.cc @@ -118,9 +118,11 @@ MockWorkerFactory::MockWorkerFactory() = default; MockWorkerFactory::~MockWorkerFactory() = default; MockWorker::MockWorker() { - ON_CALL(*this, addListener(_, _)) + ON_CALL(*this, addListener(_, _, _)) .WillByDefault( - Invoke([this](Network::ListenerConfig& config, AddListenerCompletion completion) -> void { + Invoke([this](absl::optional overridden_listener, + Network::ListenerConfig& config, AddListenerCompletion completion) -> void { + UNREFERENCED_PARAMETER(overridden_listener); config.listenSocketFactory().getListenSocket(); EXPECT_EQ(nullptr, add_listener_completion_); add_listener_completion_ = completion; @@ -139,6 +141,13 @@ MockWorker::MockWorker() { completion(); } })); + + ON_CALL(*this, removeFilterChains(_, _, _)) + .WillByDefault(Invoke([this](uint64_t, const std::list&, + std::function completion) -> void { + EXPECT_EQ(nullptr, remove_filter_chains_completion_); + remove_filter_chains_completion_ = completion; + })); } MockWorker::~MockWorker() = default; diff --git a/test/mocks/server/mocks.h b/test/mocks/server/mocks.h index 7069d229c5f4..3cc405465e61 100644 --- a/test/mocks/server/mocks.h +++ b/test/mocks/server/mocks.h @@ -336,9 +336,16 @@ class MockWorker : public Worker { remove_listener_completion_ = nullptr; } + void callDrainFilterChainsComplete() { + EXPECT_NE(nullptr, remove_filter_chains_completion_); + remove_filter_chains_completion_(); + remove_filter_chains_completion_ = nullptr; + } + // Server::Worker MOCK_METHOD(void, addListener, - (Network::ListenerConfig & listener, AddListenerCompletion completion)); + (absl::optional overridden_listener, Network::ListenerConfig& listener, + AddListenerCompletion completion)); MOCK_METHOD(uint64_t, numConnections, (), (const)); MOCK_METHOD(void, removeListener, (Network::ListenerConfig & listener, std::function completion)); @@ -347,9 +354,13 @@ class MockWorker : public Worker { MOCK_METHOD(void, stop, ()); MOCK_METHOD(void, stopListener, (Network::ListenerConfig & listener, std::function completion)); + MOCK_METHOD(void, removeFilterChains, + (uint64_t listener_tag, const std::list& filter_chains, + std::function completion)); AddListenerCompletion add_listener_completion_; std::function remove_listener_completion_; + std::function remove_filter_chains_completion_; }; class MockOverloadManager : public OverloadManager { diff --git a/test/mocks/stats/mocks.cc b/test/mocks/stats/mocks.cc index 5440212979dd..adeed55eeb5e 100644 --- a/test/mocks/stats/mocks.cc +++ b/test/mocks/stats/mocks.cc @@ -30,6 +30,12 @@ MockGauge::MockGauge() : used_(false), value_(0), import_mode_(ImportMode::Accum } MockGauge::~MockGauge() = default; +MockTextReadout::MockTextReadout() { + ON_CALL(*this, used()).WillByDefault(ReturnPointee(&used_)); + ON_CALL(*this, value()).WillByDefault(ReturnPointee(&value_)); +} +MockTextReadout::~MockTextReadout() = default; + MockHistogram::MockHistogram() { ON_CALL(*this, unit()).WillByDefault(ReturnPointee(&unit_)); ON_CALL(*this, recordValue(_)).WillByDefault(Invoke([this](uint64_t value) { diff --git a/test/mocks/stats/mocks.h b/test/mocks/stats/mocks.h index 62b4131d3ebe..06a81e7c8cf5 100644 --- a/test/mocks/stats/mocks.h +++ b/test/mocks/stats/mocks.h @@ -245,6 +245,19 @@ class MockParentHistogram : public MockMetric { RefcountHelper refcount_helper_; }; +class MockTextReadout : public MockMetric { +public: + MockTextReadout(); + ~MockTextReadout() override; + + MOCK_METHOD1(set, void(std::string&& value)); + MOCK_CONST_METHOD0(used, bool()); + MOCK_CONST_METHOD0(value, std::string()); + + bool used_; + std::string value_; +}; + class MockMetricSnapshot : public MetricSnapshot { public: MockMetricSnapshot(); @@ -253,6 +266,7 @@ class MockMetricSnapshot : public MetricSnapshot { MOCK_METHOD(const std::vector&, counters, ()); MOCK_METHOD(const std::vector>&, gauges, ()); MOCK_METHOD(const std::vector>&, histograms, ()); + MOCK_METHOD(const std::vector&, textReadouts, ()); std::vector counters_; std::vector> gauges_; @@ -290,10 +304,13 @@ class MockStore : public SymbolTableProvider, public TestUtil::TestStore { MOCK_METHOD(Histogram&, histogram, (const std::string&, Histogram::Unit)); MOCK_METHOD(std::vector, histograms, (), (const)); MOCK_METHOD(Histogram&, histogramFromString, (const std::string& name, Histogram::Unit unit)); + MOCK_METHOD(TextReadout&, textReadout, (const std::string&)); + MOCK_METHOD(std::vector, text_readouts, (), (const)); MOCK_METHOD(CounterOptConstRef, findCounter, (StatName), (const)); MOCK_METHOD(GaugeOptConstRef, findGauge, (StatName), (const)); MOCK_METHOD(HistogramOptConstRef, findHistogram, (StatName), (const)); + MOCK_METHOD(TextReadoutOptConstRef, findTextReadout, (StatName), (const)); Counter& counterFromStatNameWithTags(const StatName& name, StatNameTagVectorOptConstRef) override { @@ -309,6 +326,11 @@ class MockStore : public SymbolTableProvider, public TestUtil::TestStore { Histogram::Unit unit) override { return histogram(symbol_table_->toString(name), unit); } + TextReadout& textReadoutFromStatNameWithTags(const StatName& name, + StatNameTagVectorOptConstRef) override { + // We always just respond with the mocked counter, so the tags don't matter. + return textReadout(symbol_table_->toString(name)); + } TestSymbolTable symbol_table_; testing::NiceMock counter_; diff --git a/test/proto/bookstore.proto b/test/proto/bookstore.proto index 42c180ae734d..d814cb8f2e8d 100644 --- a/test/proto/bookstore.proto +++ b/test/proto/bookstore.proto @@ -91,6 +91,11 @@ service Bookstore { get: "/index" }; } + rpc GetIndexStream(google.protobuf.Empty) returns (stream google.api.HttpBody) { + option (google.api.http) = { + get: "/indexStream" + }; + } rpc PostBody(EchoBodyRequest) returns (google.protobuf.Empty) { option (google.api.http) = { post: "/postBody" diff --git a/test/run_envoy_bazel_coverage.sh b/test/run_envoy_bazel_coverage.sh index 4c5f0458415c..cc4282784510 100755 --- a/test/run_envoy_bazel_coverage.sh +++ b/test/run_envoy_bazel_coverage.sh @@ -35,7 +35,7 @@ BAZEL_USE_LLVM_NATIVE_COVERAGE=1 GCOV=llvm-profdata bazel coverage ${BAZEL_BUILD COVERAGE_DIR="${SRCDIR}"/generated/coverage mkdir -p "${COVERAGE_DIR}" -COVERAGE_IGNORE_REGEX="(/external/|pb\.(validate\.)?(h|cc)|/chromium_url/|/test/|/tmp|/source/extensions/quic_listeners/quiche/)" +COVERAGE_IGNORE_REGEX="(/external/|pb\.(validate\.)?(h|cc)|/chromium_url/|/test/|/tmp|/tools/|/third_party/|/source/extensions/quic_listeners/quiche/)" COVERAGE_BINARY="bazel-bin/test/coverage/coverage_tests" COVERAGE_DATA="${COVERAGE_DIR}/coverage.dat" diff --git a/test/server/connection_handler_test.cc b/test/server/connection_handler_test.cc index 46ff6a6b3641..247810612f14 100644 --- a/test/server/connection_handler_test.cc +++ b/test/server/connection_handler_test.cc @@ -1,5 +1,6 @@ #include "envoy/config/core/v3/base.pb.h" #include "envoy/config/listener/v3/udp_listener_config.pb.h" +#include "envoy/network/filter.h" #include "envoy/server/active_udp_listener_config.h" #include "envoy/stats/scope.h" @@ -46,18 +47,21 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable> filter_chain_manager = nullptr) : parent_(parent), socket_(std::make_shared()), socket_factory_(std::move(socket_factory)), tag_(tag), bind_to_port_(bind_to_port), hand_off_restored_destination_connections_(hand_off_restored_destination_connections), name_(name), listener_filters_timeout_(listener_filters_timeout), continue_on_listener_filters_timeout_(continue_on_listener_filters_timeout), - connection_balancer_(std::make_unique()) { + connection_balancer_(std::make_unique()), + inline_filter_chain_manager_(filter_chain_manager) { envoy::config::listener::v3::UdpListenerConfig dummy; std::string listener_name("raw_udp_listener"); dummy.set_udp_listener_name(listener_name); @@ -68,7 +72,10 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable udp_listener_factory_; Network::ConnectionBalancerPtr connection_balancer_; const std::vector empty_access_logs_; + std::shared_ptr> inline_filter_chain_manager_; }; using TestListenerPtr = std::unique_ptr; - TestListener* - addListener(uint64_t tag, bool bind_to_port, bool hand_off_restored_destination_connections, - const std::string& name, Network::Listener* listener, - Network::ListenerCallbacks** listener_callbacks = nullptr, - Network::MockConnectionBalancer* connection_balancer = nullptr, - Network::BalancedConnectionHandler** balanced_connection_handler = nullptr, - Network::Address::SocketType socket_type = Network::Address::SocketType::Stream, - std::chrono::milliseconds listener_filters_timeout = std::chrono::milliseconds(15000), - bool continue_on_listener_filters_timeout = false) { - ASSERT(listener != nullptr); + TestListener* addListener( + uint64_t tag, bool bind_to_port, bool hand_off_restored_destination_connections, + const std::string& name, Network::Listener* listener, + Network::ListenerCallbacks** listener_callbacks = nullptr, + Network::MockConnectionBalancer* connection_balancer = nullptr, + Network::BalancedConnectionHandler** balanced_connection_handler = nullptr, + Network::Address::SocketType socket_type = Network::Address::SocketType::Stream, + std::chrono::milliseconds listener_filters_timeout = std::chrono::milliseconds(15000), + bool continue_on_listener_filters_timeout = false, + std::shared_ptr> overridden_filter_chain_manager = + nullptr) { listeners_.emplace_back(std::make_unique( *this, tag, bind_to_port, hand_off_restored_destination_connections, name, socket_type, - listener_filters_timeout, continue_on_listener_filters_timeout, socket_factory_)); + listener_filters_timeout, continue_on_listener_filters_timeout, socket_factory_, + overridden_filter_chain_manager)); EXPECT_CALL(*socket_factory_, socketType()).WillOnce(Return(socket_type)); + if (listener == nullptr) { + // Expecting listener config in place update. + // If so, dispatcher would not create new network listener. + return listeners_.back().get(); + } EXPECT_CALL(*socket_factory_, getListenSocket()).WillOnce(Return(listeners_.back()->socket_)); if (socket_type == Network::Address::SocketType::Stream) { EXPECT_CALL(dispatcher_, createListener_(_, _, _)) @@ -144,6 +159,7 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable(listener); })); } + if (connection_balancer != nullptr) { listeners_.back()->connection_balancer_.reset(connection_balancer); ASSERT(balanced_connection_handler != nullptr); @@ -185,7 +201,7 @@ TEST_F(ConnectionHandlerTest, RemoveListenerDuringRebalance) { addListener(1, true, false, "test_listener", listener, &listener_callbacks, connection_balancer, ¤t_handler); EXPECT_CALL(*socket_factory_, localAddress()).WillOnce(ReturnRef(local_address_)); - handler_->addListener(*test_listener); + handler_->addListener(absl::nullopt, *test_listener); // Fake a balancer posting a connection to us. Event::PostCb post_cb; @@ -204,6 +220,7 @@ TEST_F(ConnectionHandlerTest, RemoveListenerDuringRebalance) { // The original test continues without the previous line being run. To avoid the same assert // firing during teardown, run the posted callback now. post_cb(); + ASSERT(post_cb != nullptr); EXPECT_CALL(*listener, onDestroy()); #else // On release builds this should be fine. @@ -221,7 +238,7 @@ TEST_F(ConnectionHandlerTest, RemoveListener) { TestListener* test_listener = addListener(1, true, false, "test_listener", listener, &listener_callbacks); EXPECT_CALL(*socket_factory_, localAddress()).WillOnce(ReturnRef(local_address_)); - handler_->addListener(*test_listener); + handler_->addListener(absl::nullopt, *test_listener); Network::MockConnectionSocket* connection = new NiceMock(); listener_callbacks->onAccept(Network::ConnectionSocketPtr{connection}); @@ -251,7 +268,7 @@ TEST_F(ConnectionHandlerTest, DisableListener) { TestListener* test_listener = addListener(1, false, false, "test_listener", listener, &listener_callbacks); EXPECT_CALL(*socket_factory_, localAddress()).WillOnce(ReturnRef(local_address_)); - handler_->addListener(*test_listener); + handler_->addListener(absl::nullopt, *test_listener); EXPECT_CALL(*listener, disable()); EXPECT_CALL(*listener, onDestroy()); @@ -271,7 +288,7 @@ TEST_F(ConnectionHandlerTest, AddDisabledListener) { EXPECT_CALL(*listener, onDestroy()); handler_->disableListeners(); - handler_->addListener(*test_listener); + handler_->addListener(absl::nullopt, *test_listener); } TEST_F(ConnectionHandlerTest, DestroyCloseConnections) { @@ -282,7 +299,7 @@ TEST_F(ConnectionHandlerTest, DestroyCloseConnections) { TestListener* test_listener = addListener(1, true, false, "test_listener", listener, &listener_callbacks); EXPECT_CALL(*socket_factory_, localAddress()).WillOnce(ReturnRef(local_address_)); - handler_->addListener(*test_listener); + handler_->addListener(absl::nullopt, *test_listener); Network::MockConnectionSocket* connection = new NiceMock(); listener_callbacks->onAccept(Network::ConnectionSocketPtr{connection}); @@ -301,7 +318,7 @@ TEST_F(ConnectionHandlerTest, CloseDuringFilterChainCreate) { TestListener* test_listener = addListener(1, true, false, "test_listener", listener, &listener_callbacks); EXPECT_CALL(*socket_factory_, localAddress()).WillOnce(ReturnRef(local_address_)); - handler_->addListener(*test_listener); + handler_->addListener(absl::nullopt, *test_listener); EXPECT_CALL(manager_, findFilterChain(_)).WillOnce(Return(filter_chain_.get())); Network::MockConnection* connection = new NiceMock(); @@ -324,7 +341,7 @@ TEST_F(ConnectionHandlerTest, CloseConnectionOnEmptyFilterChain) { TestListener* test_listener = addListener(1, true, false, "test_listener", listener, &listener_callbacks); EXPECT_CALL(*socket_factory_, localAddress()).WillOnce(ReturnRef(local_address_)); - handler_->addListener(*test_listener); + handler_->addListener(absl::nullopt, *test_listener); EXPECT_CALL(manager_, findFilterChain(_)).WillOnce(Return(filter_chain_.get())); Network::MockConnection* connection = new NiceMock(); @@ -347,7 +364,7 @@ TEST_F(ConnectionHandlerTest, NormalRedirect) { Network::Address::InstanceConstSharedPtr normal_address( new Network::Address::Ipv4Instance("127.0.0.1", 10001)); EXPECT_CALL(*socket_factory_, localAddress()).WillRepeatedly(ReturnRef(normal_address)); - handler_->addListener(*test_listener1); + handler_->addListener(absl::nullopt, *test_listener1); Network::ListenerCallbacks* listener_callbacks2; auto listener2 = new NiceMock(); @@ -356,7 +373,7 @@ TEST_F(ConnectionHandlerTest, NormalRedirect) { Network::Address::InstanceConstSharedPtr alt_address( new Network::Address::Ipv4Instance("127.0.0.2", 20002)); EXPECT_CALL(*socket_factory_, localAddress()).WillRepeatedly(ReturnRef(alt_address)); - handler_->addListener(*test_listener2); + handler_->addListener(absl::nullopt, *test_listener2); auto* test_filter = new NiceMock(); EXPECT_CALL(*test_filter, destroy_()); @@ -409,7 +426,7 @@ TEST_F(ConnectionHandlerTest, FallbackToWildcardListener) { Network::Address::InstanceConstSharedPtr normal_address( new Network::Address::Ipv4Instance("127.0.0.1", 10001)); EXPECT_CALL(*socket_factory_, localAddress()).WillRepeatedly(ReturnRef(normal_address)); - handler_->addListener(*test_listener1); + handler_->addListener(absl::nullopt, *test_listener1); Network::ListenerCallbacks* listener_callbacks2; auto listener2 = new NiceMock(); @@ -417,7 +434,7 @@ TEST_F(ConnectionHandlerTest, FallbackToWildcardListener) { addListener(1, false, false, "test_listener2", listener2, &listener_callbacks2); Network::Address::InstanceConstSharedPtr any_address = Network::Utility::getIpv4AnyAddress(); EXPECT_CALL(*socket_factory_, localAddress()).WillRepeatedly(ReturnRef(any_address)); - handler_->addListener(*test_listener2); + handler_->addListener(absl::nullopt, *test_listener2); Network::MockListenerFilter* test_filter = new Network::MockListenerFilter(); EXPECT_CALL(*test_filter, destroy_()); @@ -468,7 +485,7 @@ TEST_F(ConnectionHandlerTest, WildcardListenerWithOriginalDst) { Network::Address::InstanceConstSharedPtr any_address = Network::Utility::getAddressWithPort( *Network::Utility::getIpv4AnyAddress(), normal_address->ip()->port()); EXPECT_CALL(*socket_factory_, localAddress()).WillRepeatedly(ReturnRef(any_address)); - handler_->addListener(*test_listener1); + handler_->addListener(absl::nullopt, *test_listener1); Network::MockListenerFilter* test_filter = new Network::MockListenerFilter(); Network::MockConnectionSocket* accepted_socket = new NiceMock(); @@ -508,7 +525,7 @@ TEST_F(ConnectionHandlerTest, WildcardListenerWithNoOriginalDst) { Network::Address::InstanceConstSharedPtr any_address = Network::Utility::getAddressWithPort( *Network::Utility::getIpv4AnyAddress(), normal_address->ip()->port()); EXPECT_CALL(*socket_factory_, localAddress()).WillRepeatedly(ReturnRef(any_address)); - handler_->addListener(*test_listener1); + handler_->addListener(absl::nullopt, *test_listener1); Network::MockListenerFilter* test_filter = new Network::MockListenerFilter(); EXPECT_CALL(*test_filter, destroy_()); @@ -538,7 +555,7 @@ TEST_F(ConnectionHandlerTest, TransportProtocolDefault) { TestListener* test_listener = addListener(1, true, false, "test_listener", listener, &listener_callbacks); EXPECT_CALL(*socket_factory_, localAddress()).WillRepeatedly(ReturnRef(local_address_)); - handler_->addListener(*test_listener); + handler_->addListener(absl::nullopt, *test_listener); Network::MockConnectionSocket* accepted_socket = new NiceMock(); EXPECT_CALL(*accepted_socket, detectedTransportProtocol()) @@ -556,7 +573,7 @@ TEST_F(ConnectionHandlerTest, TransportProtocolCustom) { TestListener* test_listener = addListener(1, true, false, "test_listener", listener, &listener_callbacks); EXPECT_CALL(*socket_factory_, localAddress()).WillRepeatedly(ReturnRef(local_address_)); - handler_->addListener(*test_listener); + handler_->addListener(absl::nullopt, *test_listener); Network::MockListenerFilter* test_filter = new Network::MockListenerFilter(); EXPECT_CALL(*test_filter, destroy_()); @@ -589,7 +606,7 @@ TEST_F(ConnectionHandlerTest, ListenerFilterTimeout) { TestListener* test_listener = addListener(1, true, false, "test_listener", listener, &listener_callbacks); EXPECT_CALL(*socket_factory_, localAddress()).WillRepeatedly(ReturnRef(local_address_)); - handler_->addListener(*test_listener); + handler_->addListener(absl::nullopt, *test_listener); Network::MockListenerFilter* test_filter = new Network::MockListenerFilter(); EXPECT_CALL(factory_, createListenerFilterChain(_)) @@ -634,7 +651,7 @@ TEST_F(ConnectionHandlerTest, ContinueOnListenerFilterTimeout) { addListener(1, true, false, "test_listener", listener, &listener_callbacks, nullptr, nullptr, Network::Address::SocketType::Stream, std::chrono::milliseconds(15000), true); EXPECT_CALL(*socket_factory_, localAddress()).WillRepeatedly(ReturnRef(local_address_)); - handler_->addListener(*test_listener); + handler_->addListener(absl::nullopt, *test_listener); Network::MockListenerFilter* test_filter = new NiceMock(); EXPECT_CALL(factory_, createListenerFilterChain(_)) @@ -679,7 +696,7 @@ TEST_F(ConnectionHandlerTest, ListenerFilterTimeoutResetOnSuccess) { TestListener* test_listener = addListener(1, true, false, "test_listener", listener, &listener_callbacks); EXPECT_CALL(*socket_factory_, localAddress()).WillRepeatedly(ReturnRef(local_address_)); - handler_->addListener(*test_listener); + handler_->addListener(absl::nullopt, *test_listener); Network::MockListenerFilter* test_filter = new Network::MockListenerFilter(); EXPECT_CALL(factory_, createListenerFilterChain(_)) @@ -718,7 +735,7 @@ TEST_F(ConnectionHandlerTest, ListenerFilterDisabledTimeout) { addListener(1, true, false, "test_listener", listener, &listener_callbacks, nullptr, nullptr, Network::Address::SocketType::Stream, std::chrono::milliseconds()); EXPECT_CALL(*socket_factory_, localAddress()).WillRepeatedly(ReturnRef(local_address_)); - handler_->addListener(*test_listener); + handler_->addListener(absl::nullopt, *test_listener); Network::MockListenerFilter* test_filter = new Network::MockListenerFilter(); EXPECT_CALL(factory_, createListenerFilterChain(_)) @@ -747,7 +764,7 @@ TEST_F(ConnectionHandlerTest, ListenerFilterReportError) { TestListener* test_listener = addListener(1, true, false, "test_listener", listener, &listener_callbacks); EXPECT_CALL(*socket_factory_, localAddress()).WillRepeatedly(ReturnRef(local_address_)); - handler_->addListener(*test_listener); + handler_->addListener(absl::nullopt, *test_listener); Network::MockListenerFilter* first_filter = new Network::MockListenerFilter(); Network::MockListenerFilter* last_filter = new Network::MockListenerFilter(); @@ -794,7 +811,7 @@ TEST_F(ConnectionHandlerTest, UdpListenerNoFilterThrowsException) { EXPECT_CALL(*listener, onDestroy()); try { - handler_->addListener(*test_listener); + handler_->addListener(absl::nullopt, *test_listener); FAIL(); } catch (const Network::CreateListenerException& e) { EXPECT_THAT( @@ -803,6 +820,79 @@ TEST_F(ConnectionHandlerTest, UdpListenerNoFilterThrowsException) { } } +TEST_F(ConnectionHandlerTest, TcpListenerInplaceUpdate) { + InSequence s; + uint64_t old_listener_tag = 1; + uint64_t new_listener_tag = 2; + Network::ListenerCallbacks* old_listener_callbacks; + auto old_listener = new NiceMock(); + TestListener* old_test_listener = addListener(old_listener_tag, true, false, "test_listener", + old_listener, &old_listener_callbacks); + EXPECT_CALL(*socket_factory_, localAddress()).WillOnce(ReturnRef(local_address_)); + handler_->addListener(absl::nullopt, *old_test_listener); + ASSERT_NE(old_test_listener, nullptr); + + Network::ListenerCallbacks* new_listener_callbacks = nullptr; + + auto overridden_filter_chain_manager = + std::make_shared>(); + TestListener* new_test_listener = + addListener(new_listener_tag, true, false, "test_listener", /* Network::Listener */ nullptr, + &new_listener_callbacks, nullptr, nullptr, Network::Address::SocketType::Stream, + std::chrono::milliseconds(15000), false, overridden_filter_chain_manager); + handler_->addListener(old_listener_tag, *new_test_listener); + ASSERT_EQ(new_listener_callbacks, nullptr) + << "new listener should be inplace added and callback should not change"; + + Network::MockConnectionSocket* connection = new NiceMock(); + EXPECT_CALL(manager_, findFilterChain(_)).Times(0); + EXPECT_CALL(*overridden_filter_chain_manager, findFilterChain(_)).WillOnce(Return(nullptr)); + old_listener_callbacks->onAccept(Network::ConnectionSocketPtr{connection}); + EXPECT_CALL(*old_listener, onDestroy()); +} + +TEST_F(ConnectionHandlerTest, TcpListenerRemoveFilterChain) { + InSequence s; + uint64_t listener_tag = 1; + Network::ListenerCallbacks* listener_callbacks; + auto listener = new NiceMock(); + TestListener* test_listener = + addListener(listener_tag, true, false, "test_listener", listener, &listener_callbacks); + EXPECT_CALL(*socket_factory_, localAddress()).WillOnce(ReturnRef(local_address_)); + handler_->addListener(absl::nullopt, *test_listener); + + Network::MockConnectionSocket* connection = new NiceMock(); + EXPECT_CALL(manager_, findFilterChain(_)).WillOnce(Return(filter_chain_.get())); + Network::MockConnection* server_connection = new NiceMock(); + EXPECT_CALL(dispatcher_, createServerConnection_()).WillOnce(Return(server_connection)); + EXPECT_CALL(factory_, createNetworkFilterChain(_, _)).WillOnce(Return(true)); + + listener_callbacks->onAccept(Network::ConnectionSocketPtr{connection}); + + EXPECT_EQ(1UL, handler_->numConnections()); + EXPECT_EQ(1UL, TestUtility::findCounter(stats_store_, "downstream_cx_total")->value()); + EXPECT_EQ(1UL, TestUtility::findGauge(stats_store_, "downstream_cx_active")->value()); + EXPECT_EQ(1UL, TestUtility::findCounter(stats_store_, "test.downstream_cx_total")->value()); + EXPECT_EQ(1UL, TestUtility::findGauge(stats_store_, "test.downstream_cx_active")->value()); + + const std::list filter_chains{filter_chain_.get()}; + + // The completion callback is scheduled + handler_->removeFilterChains(listener_tag, filter_chains, + []() { ENVOY_LOG(debug, "removed filter chains"); }); + // Trigger the deletion if any. + dispatcher_.clearDeferredDeleteList(); + EXPECT_EQ(0UL, handler_->numConnections()); + EXPECT_EQ(1UL, TestUtility::findCounter(stats_store_, "downstream_cx_total")->value()); + EXPECT_EQ(0UL, TestUtility::findGauge(stats_store_, "downstream_cx_active")->value()); + EXPECT_EQ(1UL, TestUtility::findCounter(stats_store_, "test.downstream_cx_total")->value()); + EXPECT_EQ(0UL, TestUtility::findGauge(stats_store_, "test.downstream_cx_active")->value()); + + EXPECT_CALL(dispatcher_, clearDeferredDeleteList()); + EXPECT_CALL(*listener, onDestroy()); + handler_.reset(); +} + // Listener Filter matchers works. TEST_F(ConnectionHandlerTest, ListenerFilterWorks) { Network::ListenerCallbacks* listener_callbacks; @@ -810,7 +900,7 @@ TEST_F(ConnectionHandlerTest, ListenerFilterWorks) { TestListener* test_listener = addListener(1, true, false, "test_listener", listener, &listener_callbacks); EXPECT_CALL(*socket_factory_, localAddress()).WillRepeatedly(ReturnRef(local_address_)); - handler_->addListener(*test_listener); + handler_->addListener(absl::nullopt, *test_listener); auto all_matcher = std::make_shared(); auto* disabled_listener_filter = new Network::MockListenerFilter(); diff --git a/test/server/filter_chain_benchmark_test.cc b/test/server/filter_chain_benchmark_test.cc index d099baf1e480..d3e7b58b1499 100644 --- a/test/server/filter_chain_benchmark_test.cc +++ b/test/server/filter_chain_benchmark_test.cc @@ -166,7 +166,7 @@ const char YamlSingleDstPortBottom[] = R"EOF( class FilterChainBenchmarkFixture : public benchmark::Fixture { public: - void SetUp(const ::benchmark::State& state) override { + void SetUp(::benchmark::State& state) override { int64_t input_size = state.range(0); std::vector port_chains; port_chains.reserve(input_size); diff --git a/test/server/http/BUILD b/test/server/http/BUILD index 406ba76e0c6e..51d73c734688 100644 --- a/test/server/http/BUILD +++ b/test/server/http/BUILD @@ -3,15 +3,30 @@ licenses(["notice"]) # Apache 2 load( "//bazel:envoy_build_system.bzl", "envoy_cc_test", + "envoy_cc_test_library", "envoy_package", ) envoy_package() +envoy_cc_test_library( + name = "admin_instance_lib", + srcs = ["admin_instance.cc"], + hdrs = ["admin_instance.h"], + deps = [ + "//source/server/http:admin_lib", + "//test/mocks/runtime:runtime_mocks", + "//test/mocks/server:server_mocks", + "//test/test_common:environment_lib", + "//test/test_common:network_utility_lib", + ], +) + envoy_cc_test( name = "admin_test", srcs = ["admin_test.cc"], deps = [ + ":admin_instance_lib", "//include/envoy/json:json_object_interface", "//include/envoy/runtime:runtime_interface", "//source/common/http:message_lib", @@ -46,6 +61,18 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "stats_handler_test", + srcs = ["stats_handler_test.cc"], + deps = [ + ":admin_instance_lib", + "//source/common/stats:thread_local_store_lib", + "//source/server/http:stats_handler_lib", + "//test/test_common:logging_lib", + "//test/test_common:utility_lib", + ], +) + envoy_cc_test( name = "config_tracker_impl_test", srcs = ["config_tracker_impl_test.cc"], diff --git a/test/server/http/admin_instance.cc b/test/server/http/admin_instance.cc new file mode 100644 index 000000000000..279e5b1d590c --- /dev/null +++ b/test/server/http/admin_instance.cc @@ -0,0 +1,50 @@ +#include "test/server/http/admin_instance.h" + +namespace Envoy { +namespace Server { + +AdminInstanceTest::AdminInstanceTest() + : address_out_path_(TestEnvironment::temporaryPath("admin.address")), + cpu_profile_path_(TestEnvironment::temporaryPath("envoy.prof")), + admin_(cpu_profile_path_, server_), request_headers_{{":path", "/"}}, + admin_filter_(admin_.createCallbackFunction()) { + admin_.startHttpListener("/dev/null", address_out_path_, + Network::Test::getCanonicalLoopbackAddress(GetParam()), nullptr, + listener_scope_.createScope("listener.admin.")); + EXPECT_EQ(std::chrono::milliseconds(100), admin_.drainTimeout()); + admin_.tracingStats().random_sampling_.inc(); + EXPECT_TRUE(admin_.setCurrentClientCertDetails().empty()); + admin_filter_.setDecoderFilterCallbacks(callbacks_); +} + +Http::Code AdminInstanceTest::runCallback(absl::string_view path_and_query, + Http::ResponseHeaderMap& response_headers, + Buffer::Instance& response, absl::string_view method, + absl::string_view body) { + if (!body.empty()) { + request_headers_.setReferenceContentType(Http::Headers::get().ContentTypeValues.FormUrlEncoded); + callbacks_.buffer_ = std::make_unique(body); + } + + request_headers_.setMethod(method); + admin_filter_.decodeHeaders(request_headers_, false); + + return admin_.runCallback(path_and_query, response_headers, response, admin_filter_); +} + +Http::Code AdminInstanceTest::getCallback(absl::string_view path_and_query, + Http::ResponseHeaderMap& response_headers, + Buffer::Instance& response) { + return runCallback(path_and_query, response_headers, response, + Http::Headers::get().MethodValues.Get); +} + +Http::Code AdminInstanceTest::postCallback(absl::string_view path_and_query, + Http::ResponseHeaderMap& response_headers, + Buffer::Instance& response) { + return runCallback(path_and_query, response_headers, response, + Http::Headers::get().MethodValues.Post); +} + +} // namespace Server +} // namespace Envoy diff --git a/test/server/http/admin_instance.h b/test/server/http/admin_instance.h new file mode 100644 index 000000000000..aaec8f7a98b8 --- /dev/null +++ b/test/server/http/admin_instance.h @@ -0,0 +1,43 @@ +#pragma once + +#include "server/http/admin.h" + +#include "test/mocks/http/mocks.h" +#include "test/mocks/runtime/mocks.h" +#include "test/mocks/server/mocks.h" +#include "test/test_common/environment.h" +#include "test/test_common/network_utility.h" + +#include "absl/strings/match.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace Envoy { +namespace Server { + +class AdminInstanceTest : public testing::TestWithParam { +public: + AdminInstanceTest(); + + Http::Code runCallback(absl::string_view path_and_query, + Http::ResponseHeaderMap& response_headers, Buffer::Instance& response, + absl::string_view method, absl::string_view body = absl::string_view()); + + Http::Code getCallback(absl::string_view path_and_query, + Http::ResponseHeaderMap& response_headers, Buffer::Instance& response); + + Http::Code postCallback(absl::string_view path_and_query, + Http::ResponseHeaderMap& response_headers, Buffer::Instance& response); + + std::string address_out_path_; + std::string cpu_profile_path_; + NiceMock server_; + Stats::IsolatedStoreImpl listener_scope_; + AdminImpl admin_; + Http::TestRequestHeaderMapImpl request_headers_; + Server::AdminFilter admin_filter_; + NiceMock callbacks_; +}; + +} // namespace Server +} // namespace Envoy diff --git a/test/server/http/admin_test.cc b/test/server/http/admin_test.cc index 3099de1ebf93..6c3dcf15e3f9 100644 --- a/test/server/http/admin_test.cc +++ b/test/server/http/admin_test.cc @@ -12,27 +12,17 @@ #include "envoy/extensions/transport_sockets/tls/v3/cert.pb.h" #include "envoy/json/json_object.h" #include "envoy/runtime/runtime.h" -#include "envoy/stats/stats.h" #include "common/http/message_impl.h" #include "common/json/json_loader.h" #include "common/profiler/profiler.h" #include "common/protobuf/protobuf.h" #include "common/protobuf/utility.h" -#include "common/stats/symbol_table_creator.h" -#include "common/stats/thread_local_store.h" - -#include "server/http/admin.h" -#include "server/http/admin_filter.h" #include "extensions/transport_sockets/tls/context_config_impl.h" -#include "test/mocks/http/mocks.h" -#include "test/mocks/runtime/mocks.h" -#include "test/mocks/server/mocks.h" -#include "test/test_common/environment.h" +#include "test/server/http/admin_instance.h" #include "test/test_common/logging.h" -#include "test/test_common/network_utility.h" #include "test/test_common/printers.h" #include "test/test_common/utility.h" @@ -41,10 +31,8 @@ #include "gtest/gtest.h" using testing::AllOf; -using testing::EndsWith; using testing::Ge; using testing::HasSubstr; -using testing::InSequence; using testing::Invoke; using testing::NiceMock; using testing::Property; @@ -52,545 +40,10 @@ using testing::Ref; using testing::Return; using testing::ReturnPointee; using testing::ReturnRef; -using testing::StartsWith; namespace Envoy { namespace Server { -class AdminStatsTest : public testing::TestWithParam { -public: - AdminStatsTest() - : symbol_table_(Stats::SymbolTableCreator::makeSymbolTable()), alloc_(*symbol_table_) { - store_ = std::make_unique(alloc_); - store_->addSink(sink_); - } - - static std::string - statsAsJsonHandler(std::map& all_stats, - const std::vector& all_histograms, - const bool used_only, const absl::optional regex = absl::nullopt) { - return AdminImpl::statsAsJson(all_stats, all_histograms, used_only, regex, - true /*pretty_print*/); - } - - Stats::SymbolTablePtr symbol_table_; - NiceMock main_thread_dispatcher_; - NiceMock tls_; - Stats::AllocatorImpl alloc_; - Stats::MockSink sink_; - std::unique_ptr store_; -}; - -INSTANTIATE_TEST_SUITE_P(IpVersions, AdminStatsTest, - testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), - TestUtility::ipTestParamsToString); - -TEST_P(AdminStatsTest, StatsAsJson) { - InSequence s; - store_->initializeThreading(main_thread_dispatcher_, tls_); - - Stats::Histogram& h1 = store_->histogramFromString("h1", Stats::Histogram::Unit::Unspecified); - Stats::Histogram& h2 = store_->histogramFromString("h2", Stats::Histogram::Unit::Unspecified); - - EXPECT_CALL(sink_, onHistogramComplete(Ref(h1), 200)); - h1.recordValue(200); - - EXPECT_CALL(sink_, onHistogramComplete(Ref(h2), 100)); - h2.recordValue(100); - - store_->mergeHistograms([]() -> void {}); - - // Again record a new value in h1 so that it has both interval and cumulative values. - // h2 should only have cumulative values. - EXPECT_CALL(sink_, onHistogramComplete(Ref(h1), 100)); - h1.recordValue(100); - - store_->mergeHistograms([]() -> void {}); - - std::vector histograms = store_->histograms(); - std::sort(histograms.begin(), histograms.end(), - [](const Stats::ParentHistogramSharedPtr& a, - const Stats::ParentHistogramSharedPtr& b) -> bool { return a->name() < b->name(); }); - std::map all_stats; - std::string actual_json = statsAsJsonHandler(all_stats, histograms, false); - - const std::string expected_json = R"EOF({ - "stats": [ - { - "histograms": { - "supported_quantiles": [ - 0.0, - 25.0, - 50.0, - 75.0, - 90.0, - 95.0, - 99.0, - 99.5, - 99.9, - 100.0 - ], - "computed_quantiles": [ - { - "name": "h1", - "values": [ - { - "interval": 100.0, - "cumulative": 100.0 - }, - { - "interval": 102.5, - "cumulative": 105.0 - }, - { - "interval": 105.0, - "cumulative": 110.0 - }, - { - "interval": 107.5, - "cumulative": 205.0 - }, - { - "interval": 109.0, - "cumulative": 208.0 - }, - { - "interval": 109.5, - "cumulative": 209.0 - }, - { - "interval": 109.9, - "cumulative": 209.8 - }, - { - "interval": 109.95, - "cumulative": 209.9 - }, - { - "interval": 109.99, - "cumulative": 209.98 - }, - { - "interval": 110.0, - "cumulative": 210.0 - } - ] - }, - { - "name": "h2", - "values": [ - { - "interval": null, - "cumulative": 100.0 - }, - { - "interval": null, - "cumulative": 102.5 - }, - { - "interval": null, - "cumulative": 105.0 - }, - { - "interval": null, - "cumulative": 107.5 - }, - { - "interval": null, - "cumulative": 109.0 - }, - { - "interval": null, - "cumulative": 109.5 - }, - { - "interval": null, - "cumulative": 109.9 - }, - { - "interval": null, - "cumulative": 109.95 - }, - { - "interval": null, - "cumulative": 109.99 - }, - { - "interval": null, - "cumulative": 110.0 - } - ] - } - ] - } - } - ] -})EOF"; - - EXPECT_THAT(expected_json, JsonStringEq(actual_json)); - store_->shutdownThreading(); -} - -TEST_P(AdminStatsTest, UsedOnlyStatsAsJson) { - InSequence s; - store_->initializeThreading(main_thread_dispatcher_, tls_); - - Stats::Histogram& h1 = store_->histogramFromString("h1", Stats::Histogram::Unit::Unspecified); - Stats::Histogram& h2 = store_->histogramFromString("h2", Stats::Histogram::Unit::Unspecified); - - EXPECT_EQ("h1", h1.name()); - EXPECT_EQ("h2", h2.name()); - - EXPECT_CALL(sink_, onHistogramComplete(Ref(h1), 200)); - h1.recordValue(200); - - store_->mergeHistograms([]() -> void {}); - - // Again record a new value in h1 so that it has both interval and cumulative values. - // h2 should only have cumulative values. - EXPECT_CALL(sink_, onHistogramComplete(Ref(h1), 100)); - h1.recordValue(100); - - store_->mergeHistograms([]() -> void {}); - - std::map all_stats; - std::string actual_json = statsAsJsonHandler(all_stats, store_->histograms(), true); - - // Expected JSON should not have h2 values as it is not used. - const std::string expected_json = R"EOF({ - "stats": [ - { - "histograms": { - "supported_quantiles": [ - 0.0, - 25.0, - 50.0, - 75.0, - 90.0, - 95.0, - 99.0, - 99.5, - 99.9, - 100.0 - ], - "computed_quantiles": [ - { - "name": "h1", - "values": [ - { - "interval": 100.0, - "cumulative": 100.0 - }, - { - "interval": 102.5, - "cumulative": 105.0 - }, - { - "interval": 105.0, - "cumulative": 110.0 - }, - { - "interval": 107.5, - "cumulative": 205.0 - }, - { - "interval": 109.0, - "cumulative": 208.0 - }, - { - "interval": 109.5, - "cumulative": 209.0 - }, - { - "interval": 109.9, - "cumulative": 209.8 - }, - { - "interval": 109.95, - "cumulative": 209.9 - }, - { - "interval": 109.99, - "cumulative": 209.98 - }, - { - "interval": 110.0, - "cumulative": 210.0 - } - ] - } - ] - } - } - ] -})EOF"; - - EXPECT_THAT(expected_json, JsonStringEq(actual_json)); - store_->shutdownThreading(); -} - -TEST_P(AdminStatsTest, StatsAsJsonFilterString) { - InSequence s; - store_->initializeThreading(main_thread_dispatcher_, tls_); - - Stats::Histogram& h1 = store_->histogramFromString("h1", Stats::Histogram::Unit::Unspecified); - Stats::Histogram& h2 = store_->histogramFromString("h2", Stats::Histogram::Unit::Unspecified); - - EXPECT_CALL(sink_, onHistogramComplete(Ref(h1), 200)); - h1.recordValue(200); - - EXPECT_CALL(sink_, onHistogramComplete(Ref(h2), 100)); - h2.recordValue(100); - - store_->mergeHistograms([]() -> void {}); - - // Again record a new value in h1 so that it has both interval and cumulative values. - // h2 should only have cumulative values. - EXPECT_CALL(sink_, onHistogramComplete(Ref(h1), 100)); - h1.recordValue(100); - - store_->mergeHistograms([]() -> void {}); - - std::map all_stats; - std::string actual_json = statsAsJsonHandler(all_stats, store_->histograms(), false, - absl::optional{std::regex("[a-z]1")}); - - // Because this is a filter case, we don't expect to see any stats except for those containing - // "h1" in their name. - const std::string expected_json = R"EOF({ - "stats": [ - { - "histograms": { - "supported_quantiles": [ - 0.0, - 25.0, - 50.0, - 75.0, - 90.0, - 95.0, - 99.0, - 99.5, - 99.9, - 100.0 - ], - "computed_quantiles": [ - { - "name": "h1", - "values": [ - { - "interval": 100.0, - "cumulative": 100.0 - }, - { - "interval": 102.5, - "cumulative": 105.0 - }, - { - "interval": 105.0, - "cumulative": 110.0 - }, - { - "interval": 107.5, - "cumulative": 205.0 - }, - { - "interval": 109.0, - "cumulative": 208.0 - }, - { - "interval": 109.5, - "cumulative": 209.0 - }, - { - "interval": 109.9, - "cumulative": 209.8 - }, - { - "interval": 109.95, - "cumulative": 209.9 - }, - { - "interval": 109.99, - "cumulative": 209.98 - }, - { - "interval": 110.0, - "cumulative": 210.0 - } - ] - } - ] - } - } - ] -})EOF"; - - EXPECT_THAT(expected_json, JsonStringEq(actual_json)); - store_->shutdownThreading(); -} - -TEST_P(AdminStatsTest, UsedOnlyStatsAsJsonFilterString) { - InSequence s; - store_->initializeThreading(main_thread_dispatcher_, tls_); - - Stats::Histogram& h1 = store_->histogramFromString( - "h1_matches", Stats::Histogram::Unit::Unspecified); // Will match, be used, and print - Stats::Histogram& h2 = store_->histogramFromString( - "h2_matches", Stats::Histogram::Unit::Unspecified); // Will match but not be used - Stats::Histogram& h3 = store_->histogramFromString( - "h3_not", Stats::Histogram::Unit::Unspecified); // Will be used but not match - - EXPECT_EQ("h1_matches", h1.name()); - EXPECT_EQ("h2_matches", h2.name()); - EXPECT_EQ("h3_not", h3.name()); - - EXPECT_CALL(sink_, onHistogramComplete(Ref(h1), 200)); - h1.recordValue(200); - EXPECT_CALL(sink_, onHistogramComplete(Ref(h3), 200)); - h3.recordValue(200); - - store_->mergeHistograms([]() -> void {}); - - // Again record a new value in h1 and h3 so that they have both interval and cumulative values. - // h2 should only have cumulative values. - EXPECT_CALL(sink_, onHistogramComplete(Ref(h1), 100)); - h1.recordValue(100); - EXPECT_CALL(sink_, onHistogramComplete(Ref(h3), 100)); - h3.recordValue(100); - - store_->mergeHistograms([]() -> void {}); - - std::map all_stats; - std::string actual_json = statsAsJsonHandler(all_stats, store_->histograms(), true, - absl::optional{std::regex("h[12]")}); - - // Expected JSON should not have h2 values as it is not used, and should not have h3 values as - // they are used but do not match. - const std::string expected_json = R"EOF({ - "stats": [ - { - "histograms": { - "supported_quantiles": [ - 0.0, - 25.0, - 50.0, - 75.0, - 90.0, - 95.0, - 99.0, - 99.5, - 99.9, - 100.0 - ], - "computed_quantiles": [ - { - "name": "h1_matches", - "values": [ - { - "interval": 100.0, - "cumulative": 100.0 - }, - { - "interval": 102.5, - "cumulative": 105.0 - }, - { - "interval": 105.0, - "cumulative": 110.0 - }, - { - "interval": 107.5, - "cumulative": 205.0 - }, - { - "interval": 109.0, - "cumulative": 208.0 - }, - { - "interval": 109.5, - "cumulative": 209.0 - }, - { - "interval": 109.9, - "cumulative": 209.8 - }, - { - "interval": 109.95, - "cumulative": 209.9 - }, - { - "interval": 109.99, - "cumulative": 209.98 - }, - { - "interval": 110.0, - "cumulative": 210.0 - } - ] - } - ] - } - } - ] -})EOF"; - - EXPECT_THAT(expected_json, JsonStringEq(actual_json)); - store_->shutdownThreading(); -} - -class AdminInstanceTest : public testing::TestWithParam { -public: - AdminInstanceTest() - : address_out_path_(TestEnvironment::temporaryPath("admin.address")), - cpu_profile_path_(TestEnvironment::temporaryPath("envoy.prof")), - admin_(cpu_profile_path_, server_), request_headers_{{":path", "/"}}, - admin_filter_(admin_.createCallbackFunction()) { - admin_.startHttpListener("/dev/null", address_out_path_, - Network::Test::getCanonicalLoopbackAddress(GetParam()), nullptr, - listener_scope_.createScope("listener.admin.")); - EXPECT_EQ(std::chrono::milliseconds(100), admin_.drainTimeout()); - admin_.tracingStats().random_sampling_.inc(); - EXPECT_TRUE(admin_.setCurrentClientCertDetails().empty()); - admin_filter_.setDecoderFilterCallbacks(callbacks_); - } - - Http::Code runCallback(absl::string_view path_and_query, - Http::ResponseHeaderMap& response_headers, Buffer::Instance& response, - absl::string_view method, absl::string_view body = absl::string_view()) { - if (!body.empty()) { - request_headers_.setReferenceContentType( - Http::Headers::get().ContentTypeValues.FormUrlEncoded); - callbacks_.buffer_ = std::make_unique(body); - } - - request_headers_.setMethod(method); - admin_filter_.decodeHeaders(request_headers_, false); - - return admin_.runCallback(path_and_query, response_headers, response, admin_filter_); - } - - Http::Code getCallback(absl::string_view path_and_query, - Http::ResponseHeaderMap& response_headers, Buffer::Instance& response) { - return runCallback(path_and_query, response_headers, response, - Http::Headers::get().MethodValues.Get); - } - - Http::Code postCallback(absl::string_view path_and_query, - Http::ResponseHeaderMap& response_headers, Buffer::Instance& response) { - return runCallback(path_and_query, response_headers, response, - Http::Headers::get().MethodValues.Post); - } - - std::string address_out_path_; - std::string cpu_profile_path_; - NiceMock server_; - Stats::IsolatedStoreImpl listener_scope_; - AdminImpl admin_; - Http::TestRequestHeaderMapImpl request_headers_; - Server::AdminFilter admin_filter_; - NiceMock callbacks_; -}; - INSTANTIATE_TEST_SUITE_P(IpVersions, AdminInstanceTest, testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), TestUtility::ipTestParamsToString); @@ -674,37 +127,6 @@ TEST_P(AdminInstanceTest, AdminBadProfiler) { EXPECT_FALSE(Profiler::Cpu::profilerEnabled()); } -TEST_P(AdminInstanceTest, StatsInvalidRegex) { - Http::ResponseHeaderMapImpl header_map; - Buffer::OwnedImpl data; - EXPECT_LOG_CONTAINS( - "error", "Invalid regex: ", - EXPECT_EQ(Http::Code::BadRequest, getCallback("/stats?filter=*.test", header_map, data))); - - // Note: depending on the library, the detailed error message might be one of: - // "One of *?+{ was not preceded by a valid regular expression." - // "regex_error" - // but we always precede by 'Invalid regex: "'. - EXPECT_THAT(data.toString(), StartsWith("Invalid regex: \"")); - EXPECT_THAT(data.toString(), EndsWith("\"\n")); -} - -TEST_P(AdminInstanceTest, PrometheusStatsInvalidRegex) { - Http::ResponseHeaderMapImpl header_map; - Buffer::OwnedImpl data; - EXPECT_LOG_CONTAINS( - "error", ": *.ptest", - EXPECT_EQ(Http::Code::BadRequest, - getCallback("/stats?format=prometheus&filter=*.ptest", header_map, data))); - - // Note: depending on the library, the detailed error message might be one of: - // "One of *?+{ was not preceded by a valid regular expression." - // "regex_error" - // but we always precede by 'Invalid regex: "'. - EXPECT_THAT(data.toString(), StartsWith("Invalid regex: \"")); - EXPECT_THAT(data.toString(), EndsWith("\"\n")); -} - TEST_P(AdminInstanceTest, WriteAddressToFile) { std::ifstream address_file(address_out_path_); std::string address_from_file; @@ -1194,13 +616,6 @@ TEST_P(AdminInstanceTest, ReopenLogs) { EXPECT_EQ(Http::Code::OK, postCallback("/reopen_logs", header_map, response)); } -TEST_P(AdminInstanceTest, TracingStatsDisabled) { - const std::string& name = admin_.tracingStats().service_forced_.name(); - for (const Stats::CounterSharedPtr& counter : server_.stats().counters()) { - EXPECT_NE(counter->name(), name) << "Unexpected tracing stat found in server stats: " << name; - } -} - TEST_P(AdminInstanceTest, ClustersJson) { Upstream::ClusterManager::ClusterInfoMap cluster_map; ON_CALL(server_.cluster_manager_, clusters()).WillByDefault(ReturnPointee(&cluster_map)); @@ -1510,29 +925,6 @@ TEST_P(AdminInstanceTest, GetReadyRequest) { HasSubstr("text/plain")); } -TEST_P(AdminInstanceTest, GetRequestJson) { - Http::ResponseHeaderMapImpl response_headers; - std::string body; - EXPECT_EQ(Http::Code::OK, admin_.request("/stats?format=json", "GET", response_headers, body)); - EXPECT_THAT(body, HasSubstr("{\"stats\":[")); - EXPECT_THAT(std::string(response_headers.ContentType()->value().getStringView()), - HasSubstr("application/json")); -} - -TEST_P(AdminInstanceTest, RecentLookups) { - Http::ResponseHeaderMapImpl response_headers; - std::string body; - - // Recent lookup tracking is disabled by default. - EXPECT_EQ(Http::Code::OK, admin_.request("/stats/recentlookups", "GET", response_headers, body)); - EXPECT_THAT(body, HasSubstr("Lookup tracking is not enabled")); - EXPECT_THAT(std::string(response_headers.ContentType()->value().getStringView()), - HasSubstr("text/plain")); - - // We can't test RecentLookups in admin unit tests as it doesn't work with a - // fake symbol table. However we cover this solidly in integration tests. -} - TEST_P(AdminInstanceTest, PostRequest) { Http::ResponseHeaderMapImpl response_headers; std::string body; @@ -1543,442 +935,5 @@ TEST_P(AdminInstanceTest, PostRequest) { HasSubstr("text/plain")); } -class HistogramWrapper { -public: - HistogramWrapper() : histogram_(hist_alloc()) {} - - ~HistogramWrapper() { hist_free(histogram_); } - - const histogram_t* getHistogram() { return histogram_; } - - void setHistogramValues(const std::vector& values) { - for (uint64_t value : values) { - hist_insert_intscale(histogram_, value, 0, 1); - } - } - - void setHistogramValuesWithCounts(const std::vector>& values) { - for (std::pair cv : values) { - hist_insert_intscale(histogram_, cv.first, 0, cv.second); - } - } - -private: - histogram_t* histogram_; -}; - -class PrometheusStatsFormatterTest : public testing::Test { -protected: - PrometheusStatsFormatterTest() - : symbol_table_(Stats::SymbolTableCreator::makeSymbolTable()), alloc_(*symbol_table_), - pool_(*symbol_table_) {} - - ~PrometheusStatsFormatterTest() override { clearStorage(); } - - void addCounter(const std::string& name, Stats::StatNameTagVector cluster_tags) { - Stats::StatNameManagedStorage storage(name, *symbol_table_); - Stats::StatName stat_name = storage.statName(); - counters_.push_back(alloc_.makeCounter(stat_name, stat_name, cluster_tags)); - } - - void addGauge(const std::string& name, Stats::StatNameTagVector cluster_tags) { - Stats::StatNameManagedStorage storage(name, *symbol_table_); - Stats::StatName stat_name = storage.statName(); - gauges_.push_back( - alloc_.makeGauge(stat_name, stat_name, cluster_tags, Stats::Gauge::ImportMode::Accumulate)); - } - - void addHistogram(const Stats::ParentHistogramSharedPtr histogram) { - histograms_.push_back(histogram); - } - - using MockHistogramSharedPtr = Stats::RefcountPtr>; - MockHistogramSharedPtr makeHistogram() { - return MockHistogramSharedPtr(new NiceMock()); - } - - Stats::StatName makeStat(absl::string_view name) { return pool_.add(name); } - - void clearStorage() { - pool_.clear(); - counters_.clear(); - gauges_.clear(); - histograms_.clear(); - EXPECT_EQ(0, symbol_table_->numSymbols()); - } - - Stats::SymbolTablePtr symbol_table_; - Stats::AllocatorImpl alloc_; - Stats::StatNamePool pool_; - std::vector counters_; - std::vector gauges_; - std::vector histograms_; -}; - -TEST_F(PrometheusStatsFormatterTest, MetricName) { - std::string raw = "vulture.eats-liver"; - std::string expected = "envoy_vulture_eats_liver"; - auto actual = PrometheusStatsFormatter::metricName(raw); - EXPECT_EQ(expected, actual); -} - -TEST_F(PrometheusStatsFormatterTest, SanitizeMetricName) { - std::string raw = "An.artist.plays-violin@019street"; - std::string expected = "envoy_An_artist_plays_violin_019street"; - auto actual = PrometheusStatsFormatter::metricName(raw); - EXPECT_EQ(expected, actual); -} - -TEST_F(PrometheusStatsFormatterTest, SanitizeMetricNameDigitFirst) { - std::string raw = "3.artists.play-violin@019street"; - std::string expected = "envoy_3_artists_play_violin_019street"; - auto actual = PrometheusStatsFormatter::metricName(raw); - EXPECT_EQ(expected, actual); -} - -TEST_F(PrometheusStatsFormatterTest, FormattedTags) { - std::vector tags; - Stats::Tag tag1 = {"a.tag-name", "a.tag-value"}; - Stats::Tag tag2 = {"another_tag_name", "another_tag-value"}; - tags.push_back(tag1); - tags.push_back(tag2); - std::string expected = "a_tag_name=\"a.tag-value\",another_tag_name=\"another_tag-value\""; - auto actual = PrometheusStatsFormatter::formattedTags(tags); - EXPECT_EQ(expected, actual); -} - -TEST_F(PrometheusStatsFormatterTest, MetricNameCollison) { - - // Create two counters and two gauges with each pair having the same name, - // but having different tag names and values. - //`statsAsPrometheus()` should return two implying it found two unique stat names - - addCounter("cluster.test_cluster_1.upstream_cx_total", - {{makeStat("a.tag-name"), makeStat("a.tag-value")}}); - addCounter("cluster.test_cluster_1.upstream_cx_total", - {{makeStat("another_tag_name"), makeStat("another_tag-value")}}); - addGauge("cluster.test_cluster_2.upstream_cx_total", - {{makeStat("another_tag_name_3"), makeStat("another_tag_3-value")}}); - addGauge("cluster.test_cluster_2.upstream_cx_total", - {{makeStat("another_tag_name_4"), makeStat("another_tag_4-value")}}); - - Buffer::OwnedImpl response; - auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response, - false, absl::nullopt); - EXPECT_EQ(2UL, size); -} - -TEST_F(PrometheusStatsFormatterTest, UniqueMetricName) { - - // Create two counters and two gauges, all with unique names. - // statsAsPrometheus() should return four implying it found - // four unique stat names. - - addCounter("cluster.test_cluster_1.upstream_cx_total", - {{makeStat("a.tag-name"), makeStat("a.tag-value")}}); - addCounter("cluster.test_cluster_2.upstream_cx_total", - {{makeStat("another_tag_name"), makeStat("another_tag-value")}}); - addGauge("cluster.test_cluster_3.upstream_cx_total", - {{makeStat("another_tag_name_3"), makeStat("another_tag_3-value")}}); - addGauge("cluster.test_cluster_4.upstream_cx_total", - {{makeStat("another_tag_name_4"), makeStat("another_tag_4-value")}}); - - Buffer::OwnedImpl response; - auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response, - false, absl::nullopt); - EXPECT_EQ(4UL, size); -} - -TEST_F(PrometheusStatsFormatterTest, HistogramWithNoValuesAndNoTags) { - HistogramWrapper h1_cumulative; - h1_cumulative.setHistogramValues(std::vector(0)); - Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram()); - - auto histogram = makeHistogram(); - histogram->name_ = "histogram1"; - histogram->used_ = true; - ON_CALL(*histogram, cumulativeStatistics()) - .WillByDefault(testing::ReturnRef(h1_cumulative_statistics)); - - addHistogram(histogram); - - Buffer::OwnedImpl response; - auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response, - false, absl::nullopt); - EXPECT_EQ(1UL, size); - - const std::string expected_output = R"EOF(# TYPE envoy_histogram1 histogram -envoy_histogram1_bucket{le="0.5"} 0 -envoy_histogram1_bucket{le="1"} 0 -envoy_histogram1_bucket{le="5"} 0 -envoy_histogram1_bucket{le="10"} 0 -envoy_histogram1_bucket{le="25"} 0 -envoy_histogram1_bucket{le="50"} 0 -envoy_histogram1_bucket{le="100"} 0 -envoy_histogram1_bucket{le="250"} 0 -envoy_histogram1_bucket{le="500"} 0 -envoy_histogram1_bucket{le="1000"} 0 -envoy_histogram1_bucket{le="2500"} 0 -envoy_histogram1_bucket{le="5000"} 0 -envoy_histogram1_bucket{le="10000"} 0 -envoy_histogram1_bucket{le="30000"} 0 -envoy_histogram1_bucket{le="60000"} 0 -envoy_histogram1_bucket{le="300000"} 0 -envoy_histogram1_bucket{le="600000"} 0 -envoy_histogram1_bucket{le="1800000"} 0 -envoy_histogram1_bucket{le="3600000"} 0 -envoy_histogram1_bucket{le="+Inf"} 0 -envoy_histogram1_sum{} 0 -envoy_histogram1_count{} 0 -)EOF"; - - EXPECT_EQ(expected_output, response.toString()); -} - -TEST_F(PrometheusStatsFormatterTest, HistogramWithHighCounts) { - HistogramWrapper h1_cumulative; - - // Force large counts to prove that the +Inf bucket doesn't overflow to scientific notation. - h1_cumulative.setHistogramValuesWithCounts(std::vector>({ - {1, 100000}, - {100, 1000000}, - {1000, 100000000}, - })); - - Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram()); - - auto histogram = makeHistogram(); - histogram->name_ = "histogram1"; - histogram->used_ = true; - ON_CALL(*histogram, cumulativeStatistics()) - .WillByDefault(testing::ReturnRef(h1_cumulative_statistics)); - - addHistogram(histogram); - - Buffer::OwnedImpl response; - auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response, - false, absl::nullopt); - EXPECT_EQ(1UL, size); - - const std::string expected_output = R"EOF(# TYPE envoy_histogram1 histogram -envoy_histogram1_bucket{le="0.5"} 0 -envoy_histogram1_bucket{le="1"} 0 -envoy_histogram1_bucket{le="5"} 100000 -envoy_histogram1_bucket{le="10"} 100000 -envoy_histogram1_bucket{le="25"} 100000 -envoy_histogram1_bucket{le="50"} 100000 -envoy_histogram1_bucket{le="100"} 100000 -envoy_histogram1_bucket{le="250"} 1100000 -envoy_histogram1_bucket{le="500"} 1100000 -envoy_histogram1_bucket{le="1000"} 1100000 -envoy_histogram1_bucket{le="2500"} 101100000 -envoy_histogram1_bucket{le="5000"} 101100000 -envoy_histogram1_bucket{le="10000"} 101100000 -envoy_histogram1_bucket{le="30000"} 101100000 -envoy_histogram1_bucket{le="60000"} 101100000 -envoy_histogram1_bucket{le="300000"} 101100000 -envoy_histogram1_bucket{le="600000"} 101100000 -envoy_histogram1_bucket{le="1800000"} 101100000 -envoy_histogram1_bucket{le="3600000"} 101100000 -envoy_histogram1_bucket{le="+Inf"} 101100000 -envoy_histogram1_sum{} 105105105000 -envoy_histogram1_count{} 101100000 -)EOF"; - - EXPECT_EQ(expected_output, response.toString()); -} - -TEST_F(PrometheusStatsFormatterTest, OutputWithAllMetricTypes) { - addCounter("cluster.test_1.upstream_cx_total", - {{makeStat("a.tag-name"), makeStat("a.tag-value")}}); - addCounter("cluster.test_2.upstream_cx_total", - {{makeStat("another_tag_name"), makeStat("another_tag-value")}}); - addGauge("cluster.test_3.upstream_cx_total", - {{makeStat("another_tag_name_3"), makeStat("another_tag_3-value")}}); - addGauge("cluster.test_4.upstream_cx_total", - {{makeStat("another_tag_name_4"), makeStat("another_tag_4-value")}}); - - const std::vector h1_values = {50, 20, 30, 70, 100, 5000, 200}; - HistogramWrapper h1_cumulative; - h1_cumulative.setHistogramValues(h1_values); - Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram()); - - auto histogram1 = makeHistogram(); - histogram1->name_ = "cluster.test_1.upstream_rq_time"; - histogram1->unit_ = Stats::Histogram::Unit::Milliseconds; - histogram1->used_ = true; - histogram1->setTags({Stats::Tag{"key1", "value1"}, Stats::Tag{"key2", "value2"}}); - addHistogram(histogram1); - EXPECT_CALL(*histogram1, cumulativeStatistics()) - .WillOnce(testing::ReturnRef(h1_cumulative_statistics)); - - Buffer::OwnedImpl response; - auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response, - false, absl::nullopt); - EXPECT_EQ(5UL, size); - - const std::string expected_output = R"EOF(# TYPE envoy_cluster_test_1_upstream_cx_total counter -envoy_cluster_test_1_upstream_cx_total{a_tag_name="a.tag-value"} 0 -# TYPE envoy_cluster_test_2_upstream_cx_total counter -envoy_cluster_test_2_upstream_cx_total{another_tag_name="another_tag-value"} 0 -# TYPE envoy_cluster_test_3_upstream_cx_total gauge -envoy_cluster_test_3_upstream_cx_total{another_tag_name_3="another_tag_3-value"} 0 -# TYPE envoy_cluster_test_4_upstream_cx_total gauge -envoy_cluster_test_4_upstream_cx_total{another_tag_name_4="another_tag_4-value"} 0 -# TYPE envoy_cluster_test_1_upstream_rq_time histogram -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="0.5"} 0 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="1"} 0 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="5"} 0 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="10"} 0 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="25"} 1 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="50"} 2 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="100"} 4 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="250"} 6 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="500"} 6 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="1000"} 6 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="2500"} 6 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="5000"} 6 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="10000"} 7 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="30000"} 7 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="60000"} 7 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="300000"} 7 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="600000"} 7 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="1800000"} 7 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="3600000"} 7 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="+Inf"} 7 -envoy_cluster_test_1_upstream_rq_time_sum{key1="value1",key2="value2"} 5532 -envoy_cluster_test_1_upstream_rq_time_count{key1="value1",key2="value2"} 7 -)EOF"; - - EXPECT_EQ(expected_output, response.toString()); -} - -TEST_F(PrometheusStatsFormatterTest, OutputWithUsedOnly) { - addCounter("cluster.test_1.upstream_cx_total", - {{makeStat("a.tag-name"), makeStat("a.tag-value")}}); - addCounter("cluster.test_2.upstream_cx_total", - {{makeStat("another_tag_name"), makeStat("another_tag-value")}}); - addGauge("cluster.test_3.upstream_cx_total", - {{makeStat("another_tag_name_3"), makeStat("another_tag_3-value")}}); - addGauge("cluster.test_4.upstream_cx_total", - {{makeStat("another_tag_name_4"), makeStat("another_tag_4-value")}}); - - const std::vector h1_values = {50, 20, 30, 70, 100, 5000, 200}; - HistogramWrapper h1_cumulative; - h1_cumulative.setHistogramValues(h1_values); - Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram()); - - auto histogram1 = makeHistogram(); - histogram1->name_ = "cluster.test_1.upstream_rq_time"; - histogram1->unit_ = Stats::Histogram::Unit::Milliseconds; - histogram1->used_ = true; - histogram1->setTags({Stats::Tag{"key1", "value1"}, Stats::Tag{"key2", "value2"}}); - addHistogram(histogram1); - EXPECT_CALL(*histogram1, cumulativeStatistics()) - .WillOnce(testing::ReturnRef(h1_cumulative_statistics)); - - Buffer::OwnedImpl response; - auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response, - true, absl::nullopt); - EXPECT_EQ(1UL, size); - - const std::string expected_output = R"EOF(# TYPE envoy_cluster_test_1_upstream_rq_time histogram -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="0.5"} 0 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="1"} 0 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="5"} 0 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="10"} 0 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="25"} 1 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="50"} 2 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="100"} 4 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="250"} 6 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="500"} 6 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="1000"} 6 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="2500"} 6 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="5000"} 6 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="10000"} 7 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="30000"} 7 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="60000"} 7 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="300000"} 7 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="600000"} 7 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="1800000"} 7 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="3600000"} 7 -envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="+Inf"} 7 -envoy_cluster_test_1_upstream_rq_time_sum{key1="value1",key2="value2"} 5532 -envoy_cluster_test_1_upstream_rq_time_count{key1="value1",key2="value2"} 7 -)EOF"; - - EXPECT_EQ(expected_output, response.toString()); -} - -TEST_F(PrometheusStatsFormatterTest, OutputWithUsedOnlyHistogram) { - const std::vector h1_values = {}; - HistogramWrapper h1_cumulative; - h1_cumulative.setHistogramValues(h1_values); - Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram()); - - auto histogram1 = makeHistogram(); - histogram1->name_ = "cluster.test_1.upstream_rq_time"; - histogram1->unit_ = Stats::Histogram::Unit::Milliseconds; - histogram1->used_ = false; - histogram1->setTags({Stats::Tag{"key1", "value1"}, Stats::Tag{"key2", "value2"}}); - addHistogram(histogram1); - - { - const bool used_only = true; - EXPECT_CALL(*histogram1, cumulativeStatistics()).Times(0); - - Buffer::OwnedImpl response; - auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, - response, used_only, absl::nullopt); - EXPECT_EQ(0UL, size); - } - - { - const bool used_only = false; - EXPECT_CALL(*histogram1, cumulativeStatistics()) - .WillOnce(testing::ReturnRef(h1_cumulative_statistics)); - - Buffer::OwnedImpl response; - auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, - response, used_only, absl::nullopt); - EXPECT_EQ(1UL, size); - } -} - -TEST_F(PrometheusStatsFormatterTest, OutputWithRegexp) { - addCounter("cluster.test_1.upstream_cx_total", - {{makeStat("a.tag-name"), makeStat("a.tag-value")}}); - addCounter("cluster.test_2.upstream_cx_total", - {{makeStat("another_tag_name"), makeStat("another_tag-value")}}); - addGauge("cluster.test_3.upstream_cx_total", - {{makeStat("another_tag_name_3"), makeStat("another_tag_3-value")}}); - addGauge("cluster.test_4.upstream_cx_total", - {{makeStat("another_tag_name_4"), makeStat("another_tag_4-value")}}); - - const std::vector h1_values = {50, 20, 30, 70, 100, 5000, 200}; - HistogramWrapper h1_cumulative; - h1_cumulative.setHistogramValues(h1_values); - Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram()); - - auto histogram1 = makeHistogram(); - histogram1->name_ = "cluster.test_1.upstream_rq_time"; - histogram1->unit_ = Stats::Histogram::Unit::Milliseconds; - histogram1->setTags({Stats::Tag{"key1", "value1"}, Stats::Tag{"key2", "value2"}}); - addHistogram(histogram1); - - Buffer::OwnedImpl response; - auto size = PrometheusStatsFormatter::statsAsPrometheus( - counters_, gauges_, histograms_, response, false, - absl::optional{std::regex("cluster.test_1.upstream_cx_total")}); - EXPECT_EQ(1UL, size); - - const std::string expected_output = - R"EOF(# TYPE envoy_cluster_test_1_upstream_cx_total counter -envoy_cluster_test_1_upstream_cx_total{a_tag_name="a.tag-value"} 0 -)EOF"; - - EXPECT_EQ(expected_output, response.toString()); -} - } // namespace Server } // namespace Envoy diff --git a/test/server/http/stats_handler_test.cc b/test/server/http/stats_handler_test.cc new file mode 100644 index 000000000000..8c02e3846ffa --- /dev/null +++ b/test/server/http/stats_handler_test.cc @@ -0,0 +1,1012 @@ +#include + +#include "common/stats/thread_local_store.h" + +#include "server/http/stats_handler.h" + +#include "test/server/http/admin_instance.h" +#include "test/test_common/logging.h" +#include "test/test_common/utility.h" + +using testing::EndsWith; +using testing::HasSubstr; +using testing::InSequence; +using testing::Ref; +using testing::StartsWith; + +namespace Envoy { +namespace Server { + +class AdminStatsTest : public testing::TestWithParam { +public: + AdminStatsTest() + : symbol_table_(Stats::SymbolTableCreator::makeSymbolTable()), alloc_(*symbol_table_) { + store_ = std::make_unique(alloc_); + store_->addSink(sink_); + } + + static std::string + statsAsJsonHandler(std::map& all_stats, + std::map& all_text_readouts, + const std::vector& all_histograms, + const bool used_only, const absl::optional regex = absl::nullopt) { + return StatsHandler::statsAsJson(all_stats, all_text_readouts, all_histograms, used_only, regex, + true /*pretty_print*/); + } + + Stats::SymbolTablePtr symbol_table_; + NiceMock main_thread_dispatcher_; + NiceMock tls_; + Stats::AllocatorImpl alloc_; + Stats::MockSink sink_; + std::unique_ptr store_; +}; + +INSTANTIATE_TEST_SUITE_P(IpVersions, AdminStatsTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), + TestUtility::ipTestParamsToString); + +TEST_P(AdminStatsTest, StatsAsJson) { + InSequence s; + store_->initializeThreading(main_thread_dispatcher_, tls_); + + Stats::Histogram& h1 = store_->histogramFromString("h1", Stats::Histogram::Unit::Unspecified); + Stats::Histogram& h2 = store_->histogramFromString("h2", Stats::Histogram::Unit::Unspecified); + + EXPECT_CALL(sink_, onHistogramComplete(Ref(h1), 200)); + h1.recordValue(200); + + EXPECT_CALL(sink_, onHistogramComplete(Ref(h2), 100)); + h2.recordValue(100); + + store_->mergeHistograms([]() -> void {}); + + // Again record a new value in h1 so that it has both interval and cumulative values. + // h2 should only have cumulative values. + EXPECT_CALL(sink_, onHistogramComplete(Ref(h1), 100)); + h1.recordValue(100); + + store_->mergeHistograms([]() -> void {}); + + std::vector histograms = store_->histograms(); + std::sort(histograms.begin(), histograms.end(), + [](const Stats::ParentHistogramSharedPtr& a, + const Stats::ParentHistogramSharedPtr& b) -> bool { return a->name() < b->name(); }); + std::map all_stats; + std::map all_text_readouts; + std::string actual_json = statsAsJsonHandler(all_stats, all_text_readouts, histograms, false); + + const std::string expected_json = R"EOF({ + "stats": [ + { + "histograms": { + "supported_quantiles": [ + 0.0, + 25.0, + 50.0, + 75.0, + 90.0, + 95.0, + 99.0, + 99.5, + 99.9, + 100.0 + ], + "computed_quantiles": [ + { + "name": "h1", + "values": [ + { + "interval": 100.0, + "cumulative": 100.0 + }, + { + "interval": 102.5, + "cumulative": 105.0 + }, + { + "interval": 105.0, + "cumulative": 110.0 + }, + { + "interval": 107.5, + "cumulative": 205.0 + }, + { + "interval": 109.0, + "cumulative": 208.0 + }, + { + "interval": 109.5, + "cumulative": 209.0 + }, + { + "interval": 109.9, + "cumulative": 209.8 + }, + { + "interval": 109.95, + "cumulative": 209.9 + }, + { + "interval": 109.99, + "cumulative": 209.98 + }, + { + "interval": 110.0, + "cumulative": 210.0 + } + ] + }, + { + "name": "h2", + "values": [ + { + "interval": null, + "cumulative": 100.0 + }, + { + "interval": null, + "cumulative": 102.5 + }, + { + "interval": null, + "cumulative": 105.0 + }, + { + "interval": null, + "cumulative": 107.5 + }, + { + "interval": null, + "cumulative": 109.0 + }, + { + "interval": null, + "cumulative": 109.5 + }, + { + "interval": null, + "cumulative": 109.9 + }, + { + "interval": null, + "cumulative": 109.95 + }, + { + "interval": null, + "cumulative": 109.99 + }, + { + "interval": null, + "cumulative": 110.0 + } + ] + } + ] + } + } + ] +})EOF"; + + EXPECT_THAT(expected_json, JsonStringEq(actual_json)); + store_->shutdownThreading(); +} + +TEST_P(AdminStatsTest, UsedOnlyStatsAsJson) { + InSequence s; + store_->initializeThreading(main_thread_dispatcher_, tls_); + + Stats::Histogram& h1 = store_->histogramFromString("h1", Stats::Histogram::Unit::Unspecified); + Stats::Histogram& h2 = store_->histogramFromString("h2", Stats::Histogram::Unit::Unspecified); + + EXPECT_EQ("h1", h1.name()); + EXPECT_EQ("h2", h2.name()); + + EXPECT_CALL(sink_, onHistogramComplete(Ref(h1), 200)); + h1.recordValue(200); + + store_->mergeHistograms([]() -> void {}); + + // Again record a new value in h1 so that it has both interval and cumulative values. + // h2 should only have cumulative values. + EXPECT_CALL(sink_, onHistogramComplete(Ref(h1), 100)); + h1.recordValue(100); + + store_->mergeHistograms([]() -> void {}); + + std::map all_stats; + std::map all_text_readouts; + std::string actual_json = + statsAsJsonHandler(all_stats, all_text_readouts, store_->histograms(), true); + + // Expected JSON should not have h2 values as it is not used. + const std::string expected_json = R"EOF({ + "stats": [ + { + "histograms": { + "supported_quantiles": [ + 0.0, + 25.0, + 50.0, + 75.0, + 90.0, + 95.0, + 99.0, + 99.5, + 99.9, + 100.0 + ], + "computed_quantiles": [ + { + "name": "h1", + "values": [ + { + "interval": 100.0, + "cumulative": 100.0 + }, + { + "interval": 102.5, + "cumulative": 105.0 + }, + { + "interval": 105.0, + "cumulative": 110.0 + }, + { + "interval": 107.5, + "cumulative": 205.0 + }, + { + "interval": 109.0, + "cumulative": 208.0 + }, + { + "interval": 109.5, + "cumulative": 209.0 + }, + { + "interval": 109.9, + "cumulative": 209.8 + }, + { + "interval": 109.95, + "cumulative": 209.9 + }, + { + "interval": 109.99, + "cumulative": 209.98 + }, + { + "interval": 110.0, + "cumulative": 210.0 + } + ] + } + ] + } + } + ] +})EOF"; + + EXPECT_THAT(expected_json, JsonStringEq(actual_json)); + store_->shutdownThreading(); +} + +TEST_P(AdminStatsTest, StatsAsJsonFilterString) { + InSequence s; + store_->initializeThreading(main_thread_dispatcher_, tls_); + + Stats::Histogram& h1 = store_->histogramFromString("h1", Stats::Histogram::Unit::Unspecified); + Stats::Histogram& h2 = store_->histogramFromString("h2", Stats::Histogram::Unit::Unspecified); + + EXPECT_CALL(sink_, onHistogramComplete(Ref(h1), 200)); + h1.recordValue(200); + + EXPECT_CALL(sink_, onHistogramComplete(Ref(h2), 100)); + h2.recordValue(100); + + store_->mergeHistograms([]() -> void {}); + + // Again record a new value in h1 so that it has both interval and cumulative values. + // h2 should only have cumulative values. + EXPECT_CALL(sink_, onHistogramComplete(Ref(h1), 100)); + h1.recordValue(100); + + store_->mergeHistograms([]() -> void {}); + + std::map all_stats; + std::map all_text_readouts; + std::string actual_json = + statsAsJsonHandler(all_stats, all_text_readouts, store_->histograms(), false, + absl::optional{std::regex("[a-z]1")}); + + // Because this is a filter case, we don't expect to see any stats except for those containing + // "h1" in their name. + const std::string expected_json = R"EOF({ + "stats": [ + { + "histograms": { + "supported_quantiles": [ + 0.0, + 25.0, + 50.0, + 75.0, + 90.0, + 95.0, + 99.0, + 99.5, + 99.9, + 100.0 + ], + "computed_quantiles": [ + { + "name": "h1", + "values": [ + { + "interval": 100.0, + "cumulative": 100.0 + }, + { + "interval": 102.5, + "cumulative": 105.0 + }, + { + "interval": 105.0, + "cumulative": 110.0 + }, + { + "interval": 107.5, + "cumulative": 205.0 + }, + { + "interval": 109.0, + "cumulative": 208.0 + }, + { + "interval": 109.5, + "cumulative": 209.0 + }, + { + "interval": 109.9, + "cumulative": 209.8 + }, + { + "interval": 109.95, + "cumulative": 209.9 + }, + { + "interval": 109.99, + "cumulative": 209.98 + }, + { + "interval": 110.0, + "cumulative": 210.0 + } + ] + } + ] + } + } + ] +})EOF"; + + EXPECT_THAT(expected_json, JsonStringEq(actual_json)); + store_->shutdownThreading(); +} + +TEST_P(AdminStatsTest, UsedOnlyStatsAsJsonFilterString) { + InSequence s; + store_->initializeThreading(main_thread_dispatcher_, tls_); + + Stats::Histogram& h1 = store_->histogramFromString( + "h1_matches", Stats::Histogram::Unit::Unspecified); // Will match, be used, and print + Stats::Histogram& h2 = store_->histogramFromString( + "h2_matches", Stats::Histogram::Unit::Unspecified); // Will match but not be used + Stats::Histogram& h3 = store_->histogramFromString( + "h3_not", Stats::Histogram::Unit::Unspecified); // Will be used but not match + + EXPECT_EQ("h1_matches", h1.name()); + EXPECT_EQ("h2_matches", h2.name()); + EXPECT_EQ("h3_not", h3.name()); + + EXPECT_CALL(sink_, onHistogramComplete(Ref(h1), 200)); + h1.recordValue(200); + EXPECT_CALL(sink_, onHistogramComplete(Ref(h3), 200)); + h3.recordValue(200); + + store_->mergeHistograms([]() -> void {}); + + // Again record a new value in h1 and h3 so that they have both interval and cumulative values. + // h2 should only have cumulative values. + EXPECT_CALL(sink_, onHistogramComplete(Ref(h1), 100)); + h1.recordValue(100); + EXPECT_CALL(sink_, onHistogramComplete(Ref(h3), 100)); + h3.recordValue(100); + + store_->mergeHistograms([]() -> void {}); + + std::map all_stats; + std::map all_text_readouts; + std::string actual_json = + statsAsJsonHandler(all_stats, all_text_readouts, store_->histograms(), true, + absl::optional{std::regex("h[12]")}); + + // Expected JSON should not have h2 values as it is not used, and should not have h3 values as + // they are used but do not match. + const std::string expected_json = R"EOF({ + "stats": [ + { + "histograms": { + "supported_quantiles": [ + 0.0, + 25.0, + 50.0, + 75.0, + 90.0, + 95.0, + 99.0, + 99.5, + 99.9, + 100.0 + ], + "computed_quantiles": [ + { + "name": "h1_matches", + "values": [ + { + "interval": 100.0, + "cumulative": 100.0 + }, + { + "interval": 102.5, + "cumulative": 105.0 + }, + { + "interval": 105.0, + "cumulative": 110.0 + }, + { + "interval": 107.5, + "cumulative": 205.0 + }, + { + "interval": 109.0, + "cumulative": 208.0 + }, + { + "interval": 109.5, + "cumulative": 209.0 + }, + { + "interval": 109.9, + "cumulative": 209.8 + }, + { + "interval": 109.95, + "cumulative": 209.9 + }, + { + "interval": 109.99, + "cumulative": 209.98 + }, + { + "interval": 110.0, + "cumulative": 210.0 + } + ] + } + ] + } + } + ] +})EOF"; + + EXPECT_THAT(expected_json, JsonStringEq(actual_json)); + store_->shutdownThreading(); +} + +INSTANTIATE_TEST_SUITE_P(IpVersions, AdminInstanceTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), + TestUtility::ipTestParamsToString); + +TEST_P(AdminInstanceTest, StatsInvalidRegex) { + Http::ResponseHeaderMapImpl header_map; + Buffer::OwnedImpl data; + EXPECT_LOG_CONTAINS( + "error", "Invalid regex: ", + EXPECT_EQ(Http::Code::BadRequest, getCallback("/stats?filter=*.test", header_map, data))); + + // Note: depending on the library, the detailed error message might be one of: + // "One of *?+{ was not preceded by a valid regular expression." + // "regex_error" + // but we always precede by 'Invalid regex: "'. + EXPECT_THAT(data.toString(), StartsWith("Invalid regex: \"")); + EXPECT_THAT(data.toString(), EndsWith("\"\n")); +} + +TEST_P(AdminInstanceTest, PrometheusStatsInvalidRegex) { + Http::ResponseHeaderMapImpl header_map; + Buffer::OwnedImpl data; + EXPECT_LOG_CONTAINS( + "error", ": *.ptest", + EXPECT_EQ(Http::Code::BadRequest, + getCallback("/stats?format=prometheus&filter=*.ptest", header_map, data))); + + // Note: depending on the library, the detailed error message might be one of: + // "One of *?+{ was not preceded by a valid regular expression." + // "regex_error" + // but we always precede by 'Invalid regex: "'. + EXPECT_THAT(data.toString(), StartsWith("Invalid regex: \"")); + EXPECT_THAT(data.toString(), EndsWith("\"\n")); +} + +TEST_P(AdminInstanceTest, TracingStatsDisabled) { + const std::string& name = admin_.tracingStats().service_forced_.name(); + for (const Stats::CounterSharedPtr& counter : server_.stats().counters()) { + EXPECT_NE(counter->name(), name) << "Unexpected tracing stat found in server stats: " << name; + } +} + +TEST_P(AdminInstanceTest, GetRequestJson) { + Http::ResponseHeaderMapImpl response_headers; + std::string body; + EXPECT_EQ(Http::Code::OK, admin_.request("/stats?format=json", "GET", response_headers, body)); + EXPECT_THAT(body, HasSubstr("{\"stats\":[")); + EXPECT_THAT(std::string(response_headers.ContentType()->value().getStringView()), + HasSubstr("application/json")); +} + +TEST_P(AdminInstanceTest, RecentLookups) { + Http::ResponseHeaderMapImpl response_headers; + std::string body; + + // Recent lookup tracking is disabled by default. + EXPECT_EQ(Http::Code::OK, admin_.request("/stats/recentlookups", "GET", response_headers, body)); + EXPECT_THAT(body, HasSubstr("Lookup tracking is not enabled")); + EXPECT_THAT(std::string(response_headers.ContentType()->value().getStringView()), + HasSubstr("text/plain")); + + // We can't test RecentLookups in admin unit tests as it doesn't work with a + // fake symbol table. However we cover this solidly in integration tests. +} + +class HistogramWrapper { +public: + HistogramWrapper() : histogram_(hist_alloc()) {} + + ~HistogramWrapper() { hist_free(histogram_); } + + const histogram_t* getHistogram() { return histogram_; } + + void setHistogramValues(const std::vector& values) { + for (uint64_t value : values) { + hist_insert_intscale(histogram_, value, 0, 1); + } + } + + void setHistogramValuesWithCounts(const std::vector>& values) { + for (std::pair cv : values) { + hist_insert_intscale(histogram_, cv.first, 0, cv.second); + } + } + +private: + histogram_t* histogram_; +}; + +class PrometheusStatsFormatterTest : public testing::Test { +protected: + PrometheusStatsFormatterTest() + : symbol_table_(Stats::SymbolTableCreator::makeSymbolTable()), alloc_(*symbol_table_), + pool_(*symbol_table_) {} + + ~PrometheusStatsFormatterTest() override { clearStorage(); } + + void addCounter(const std::string& name, Stats::StatNameTagVector cluster_tags) { + Stats::StatNameManagedStorage storage(name, *symbol_table_); + Stats::StatName stat_name = storage.statName(); + counters_.push_back(alloc_.makeCounter(stat_name, stat_name, cluster_tags)); + } + + void addGauge(const std::string& name, Stats::StatNameTagVector cluster_tags) { + Stats::StatNameManagedStorage storage(name, *symbol_table_); + Stats::StatName stat_name = storage.statName(); + gauges_.push_back( + alloc_.makeGauge(stat_name, stat_name, cluster_tags, Stats::Gauge::ImportMode::Accumulate)); + } + + void addHistogram(const Stats::ParentHistogramSharedPtr histogram) { + histograms_.push_back(histogram); + } + + using MockHistogramSharedPtr = Stats::RefcountPtr>; + MockHistogramSharedPtr makeHistogram() { + return MockHistogramSharedPtr(new NiceMock()); + } + + Stats::StatName makeStat(absl::string_view name) { return pool_.add(name); } + + void clearStorage() { + pool_.clear(); + counters_.clear(); + gauges_.clear(); + histograms_.clear(); + EXPECT_EQ(0, symbol_table_->numSymbols()); + } + + Stats::SymbolTablePtr symbol_table_; + Stats::AllocatorImpl alloc_; + Stats::StatNamePool pool_; + std::vector counters_; + std::vector gauges_; + std::vector histograms_; +}; + +TEST_F(PrometheusStatsFormatterTest, MetricName) { + std::string raw = "vulture.eats-liver"; + std::string expected = "envoy_vulture_eats_liver"; + auto actual = PrometheusStatsFormatter::metricName(raw); + EXPECT_EQ(expected, actual); +} + +TEST_F(PrometheusStatsFormatterTest, SanitizeMetricName) { + std::string raw = "An.artist.plays-violin@019street"; + std::string expected = "envoy_An_artist_plays_violin_019street"; + auto actual = PrometheusStatsFormatter::metricName(raw); + EXPECT_EQ(expected, actual); +} + +TEST_F(PrometheusStatsFormatterTest, SanitizeMetricNameDigitFirst) { + std::string raw = "3.artists.play-violin@019street"; + std::string expected = "envoy_3_artists_play_violin_019street"; + auto actual = PrometheusStatsFormatter::metricName(raw); + EXPECT_EQ(expected, actual); +} + +TEST_F(PrometheusStatsFormatterTest, FormattedTags) { + std::vector tags; + Stats::Tag tag1 = {"a.tag-name", "a.tag-value"}; + Stats::Tag tag2 = {"another_tag_name", "another_tag-value"}; + tags.push_back(tag1); + tags.push_back(tag2); + std::string expected = "a_tag_name=\"a.tag-value\",another_tag_name=\"another_tag-value\""; + auto actual = PrometheusStatsFormatter::formattedTags(tags); + EXPECT_EQ(expected, actual); +} + +TEST_F(PrometheusStatsFormatterTest, MetricNameCollison) { + + // Create two counters and two gauges with each pair having the same name, + // but having different tag names and values. + //`statsAsPrometheus()` should return two implying it found two unique stat names + + addCounter("cluster.test_cluster_1.upstream_cx_total", + {{makeStat("a.tag-name"), makeStat("a.tag-value")}}); + addCounter("cluster.test_cluster_1.upstream_cx_total", + {{makeStat("another_tag_name"), makeStat("another_tag-value")}}); + addGauge("cluster.test_cluster_2.upstream_cx_total", + {{makeStat("another_tag_name_3"), makeStat("another_tag_3-value")}}); + addGauge("cluster.test_cluster_2.upstream_cx_total", + {{makeStat("another_tag_name_4"), makeStat("another_tag_4-value")}}); + + Buffer::OwnedImpl response; + auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response, + false, absl::nullopt); + EXPECT_EQ(2UL, size); +} + +TEST_F(PrometheusStatsFormatterTest, UniqueMetricName) { + + // Create two counters and two gauges, all with unique names. + // statsAsPrometheus() should return four implying it found + // four unique stat names. + + addCounter("cluster.test_cluster_1.upstream_cx_total", + {{makeStat("a.tag-name"), makeStat("a.tag-value")}}); + addCounter("cluster.test_cluster_2.upstream_cx_total", + {{makeStat("another_tag_name"), makeStat("another_tag-value")}}); + addGauge("cluster.test_cluster_3.upstream_cx_total", + {{makeStat("another_tag_name_3"), makeStat("another_tag_3-value")}}); + addGauge("cluster.test_cluster_4.upstream_cx_total", + {{makeStat("another_tag_name_4"), makeStat("another_tag_4-value")}}); + + Buffer::OwnedImpl response; + auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response, + false, absl::nullopt); + EXPECT_EQ(4UL, size); +} + +TEST_F(PrometheusStatsFormatterTest, HistogramWithNoValuesAndNoTags) { + HistogramWrapper h1_cumulative; + h1_cumulative.setHistogramValues(std::vector(0)); + Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram()); + + auto histogram = makeHistogram(); + histogram->name_ = "histogram1"; + histogram->used_ = true; + ON_CALL(*histogram, cumulativeStatistics()) + .WillByDefault(testing::ReturnRef(h1_cumulative_statistics)); + + addHistogram(histogram); + + Buffer::OwnedImpl response; + auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response, + false, absl::nullopt); + EXPECT_EQ(1UL, size); + + const std::string expected_output = R"EOF(# TYPE envoy_histogram1 histogram +envoy_histogram1_bucket{le="0.5"} 0 +envoy_histogram1_bucket{le="1"} 0 +envoy_histogram1_bucket{le="5"} 0 +envoy_histogram1_bucket{le="10"} 0 +envoy_histogram1_bucket{le="25"} 0 +envoy_histogram1_bucket{le="50"} 0 +envoy_histogram1_bucket{le="100"} 0 +envoy_histogram1_bucket{le="250"} 0 +envoy_histogram1_bucket{le="500"} 0 +envoy_histogram1_bucket{le="1000"} 0 +envoy_histogram1_bucket{le="2500"} 0 +envoy_histogram1_bucket{le="5000"} 0 +envoy_histogram1_bucket{le="10000"} 0 +envoy_histogram1_bucket{le="30000"} 0 +envoy_histogram1_bucket{le="60000"} 0 +envoy_histogram1_bucket{le="300000"} 0 +envoy_histogram1_bucket{le="600000"} 0 +envoy_histogram1_bucket{le="1800000"} 0 +envoy_histogram1_bucket{le="3600000"} 0 +envoy_histogram1_bucket{le="+Inf"} 0 +envoy_histogram1_sum{} 0 +envoy_histogram1_count{} 0 +)EOF"; + + EXPECT_EQ(expected_output, response.toString()); +} + +TEST_F(PrometheusStatsFormatterTest, HistogramWithHighCounts) { + HistogramWrapper h1_cumulative; + + // Force large counts to prove that the +Inf bucket doesn't overflow to scientific notation. + h1_cumulative.setHistogramValuesWithCounts(std::vector>({ + {1, 100000}, + {100, 1000000}, + {1000, 100000000}, + })); + + Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram()); + + auto histogram = makeHistogram(); + histogram->name_ = "histogram1"; + histogram->used_ = true; + ON_CALL(*histogram, cumulativeStatistics()) + .WillByDefault(testing::ReturnRef(h1_cumulative_statistics)); + + addHistogram(histogram); + + Buffer::OwnedImpl response; + auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response, + false, absl::nullopt); + EXPECT_EQ(1UL, size); + + const std::string expected_output = R"EOF(# TYPE envoy_histogram1 histogram +envoy_histogram1_bucket{le="0.5"} 0 +envoy_histogram1_bucket{le="1"} 0 +envoy_histogram1_bucket{le="5"} 100000 +envoy_histogram1_bucket{le="10"} 100000 +envoy_histogram1_bucket{le="25"} 100000 +envoy_histogram1_bucket{le="50"} 100000 +envoy_histogram1_bucket{le="100"} 100000 +envoy_histogram1_bucket{le="250"} 1100000 +envoy_histogram1_bucket{le="500"} 1100000 +envoy_histogram1_bucket{le="1000"} 1100000 +envoy_histogram1_bucket{le="2500"} 101100000 +envoy_histogram1_bucket{le="5000"} 101100000 +envoy_histogram1_bucket{le="10000"} 101100000 +envoy_histogram1_bucket{le="30000"} 101100000 +envoy_histogram1_bucket{le="60000"} 101100000 +envoy_histogram1_bucket{le="300000"} 101100000 +envoy_histogram1_bucket{le="600000"} 101100000 +envoy_histogram1_bucket{le="1800000"} 101100000 +envoy_histogram1_bucket{le="3600000"} 101100000 +envoy_histogram1_bucket{le="+Inf"} 101100000 +envoy_histogram1_sum{} 105105105000 +envoy_histogram1_count{} 101100000 +)EOF"; + + EXPECT_EQ(expected_output, response.toString()); +} + +TEST_F(PrometheusStatsFormatterTest, OutputWithAllMetricTypes) { + addCounter("cluster.test_1.upstream_cx_total", + {{makeStat("a.tag-name"), makeStat("a.tag-value")}}); + addCounter("cluster.test_2.upstream_cx_total", + {{makeStat("another_tag_name"), makeStat("another_tag-value")}}); + addGauge("cluster.test_3.upstream_cx_total", + {{makeStat("another_tag_name_3"), makeStat("another_tag_3-value")}}); + addGauge("cluster.test_4.upstream_cx_total", + {{makeStat("another_tag_name_4"), makeStat("another_tag_4-value")}}); + + const std::vector h1_values = {50, 20, 30, 70, 100, 5000, 200}; + HistogramWrapper h1_cumulative; + h1_cumulative.setHistogramValues(h1_values); + Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram()); + + auto histogram1 = makeHistogram(); + histogram1->name_ = "cluster.test_1.upstream_rq_time"; + histogram1->unit_ = Stats::Histogram::Unit::Milliseconds; + histogram1->used_ = true; + histogram1->setTags({Stats::Tag{"key1", "value1"}, Stats::Tag{"key2", "value2"}}); + addHistogram(histogram1); + EXPECT_CALL(*histogram1, cumulativeStatistics()) + .WillOnce(testing::ReturnRef(h1_cumulative_statistics)); + + Buffer::OwnedImpl response; + auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response, + false, absl::nullopt); + EXPECT_EQ(5UL, size); + + const std::string expected_output = R"EOF(# TYPE envoy_cluster_test_1_upstream_cx_total counter +envoy_cluster_test_1_upstream_cx_total{a_tag_name="a.tag-value"} 0 +# TYPE envoy_cluster_test_2_upstream_cx_total counter +envoy_cluster_test_2_upstream_cx_total{another_tag_name="another_tag-value"} 0 +# TYPE envoy_cluster_test_3_upstream_cx_total gauge +envoy_cluster_test_3_upstream_cx_total{another_tag_name_3="another_tag_3-value"} 0 +# TYPE envoy_cluster_test_4_upstream_cx_total gauge +envoy_cluster_test_4_upstream_cx_total{another_tag_name_4="another_tag_4-value"} 0 +# TYPE envoy_cluster_test_1_upstream_rq_time histogram +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="0.5"} 0 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="1"} 0 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="5"} 0 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="10"} 0 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="25"} 1 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="50"} 2 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="100"} 4 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="250"} 6 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="500"} 6 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="1000"} 6 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="2500"} 6 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="5000"} 6 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="10000"} 7 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="30000"} 7 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="60000"} 7 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="300000"} 7 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="600000"} 7 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="1800000"} 7 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="3600000"} 7 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="+Inf"} 7 +envoy_cluster_test_1_upstream_rq_time_sum{key1="value1",key2="value2"} 5532 +envoy_cluster_test_1_upstream_rq_time_count{key1="value1",key2="value2"} 7 +)EOF"; + + EXPECT_EQ(expected_output, response.toString()); +} + +TEST_F(PrometheusStatsFormatterTest, OutputWithUsedOnly) { + addCounter("cluster.test_1.upstream_cx_total", + {{makeStat("a.tag-name"), makeStat("a.tag-value")}}); + addCounter("cluster.test_2.upstream_cx_total", + {{makeStat("another_tag_name"), makeStat("another_tag-value")}}); + addGauge("cluster.test_3.upstream_cx_total", + {{makeStat("another_tag_name_3"), makeStat("another_tag_3-value")}}); + addGauge("cluster.test_4.upstream_cx_total", + {{makeStat("another_tag_name_4"), makeStat("another_tag_4-value")}}); + + const std::vector h1_values = {50, 20, 30, 70, 100, 5000, 200}; + HistogramWrapper h1_cumulative; + h1_cumulative.setHistogramValues(h1_values); + Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram()); + + auto histogram1 = makeHistogram(); + histogram1->name_ = "cluster.test_1.upstream_rq_time"; + histogram1->unit_ = Stats::Histogram::Unit::Milliseconds; + histogram1->used_ = true; + histogram1->setTags({Stats::Tag{"key1", "value1"}, Stats::Tag{"key2", "value2"}}); + addHistogram(histogram1); + EXPECT_CALL(*histogram1, cumulativeStatistics()) + .WillOnce(testing::ReturnRef(h1_cumulative_statistics)); + + Buffer::OwnedImpl response; + auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response, + true, absl::nullopt); + EXPECT_EQ(1UL, size); + + const std::string expected_output = R"EOF(# TYPE envoy_cluster_test_1_upstream_rq_time histogram +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="0.5"} 0 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="1"} 0 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="5"} 0 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="10"} 0 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="25"} 1 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="50"} 2 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="100"} 4 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="250"} 6 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="500"} 6 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="1000"} 6 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="2500"} 6 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="5000"} 6 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="10000"} 7 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="30000"} 7 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="60000"} 7 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="300000"} 7 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="600000"} 7 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="1800000"} 7 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="3600000"} 7 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="+Inf"} 7 +envoy_cluster_test_1_upstream_rq_time_sum{key1="value1",key2="value2"} 5532 +envoy_cluster_test_1_upstream_rq_time_count{key1="value1",key2="value2"} 7 +)EOF"; + + EXPECT_EQ(expected_output, response.toString()); +} + +TEST_F(PrometheusStatsFormatterTest, OutputWithUsedOnlyHistogram) { + const std::vector h1_values = {}; + HistogramWrapper h1_cumulative; + h1_cumulative.setHistogramValues(h1_values); + Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram()); + + auto histogram1 = makeHistogram(); + histogram1->name_ = "cluster.test_1.upstream_rq_time"; + histogram1->unit_ = Stats::Histogram::Unit::Milliseconds; + histogram1->used_ = false; + histogram1->setTags({Stats::Tag{"key1", "value1"}, Stats::Tag{"key2", "value2"}}); + addHistogram(histogram1); + + { + const bool used_only = true; + EXPECT_CALL(*histogram1, cumulativeStatistics()).Times(0); + + Buffer::OwnedImpl response; + auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, + response, used_only, absl::nullopt); + EXPECT_EQ(0UL, size); + } + + { + const bool used_only = false; + EXPECT_CALL(*histogram1, cumulativeStatistics()) + .WillOnce(testing::ReturnRef(h1_cumulative_statistics)); + + Buffer::OwnedImpl response; + auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, + response, used_only, absl::nullopt); + EXPECT_EQ(1UL, size); + } +} + +TEST_F(PrometheusStatsFormatterTest, OutputWithRegexp) { + addCounter("cluster.test_1.upstream_cx_total", + {{makeStat("a.tag-name"), makeStat("a.tag-value")}}); + addCounter("cluster.test_2.upstream_cx_total", + {{makeStat("another_tag_name"), makeStat("another_tag-value")}}); + addGauge("cluster.test_3.upstream_cx_total", + {{makeStat("another_tag_name_3"), makeStat("another_tag_3-value")}}); + addGauge("cluster.test_4.upstream_cx_total", + {{makeStat("another_tag_name_4"), makeStat("another_tag_4-value")}}); + + const std::vector h1_values = {50, 20, 30, 70, 100, 5000, 200}; + HistogramWrapper h1_cumulative; + h1_cumulative.setHistogramValues(h1_values); + Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram()); + + auto histogram1 = makeHistogram(); + histogram1->name_ = "cluster.test_1.upstream_rq_time"; + histogram1->unit_ = Stats::Histogram::Unit::Milliseconds; + histogram1->setTags({Stats::Tag{"key1", "value1"}, Stats::Tag{"key2", "value2"}}); + addHistogram(histogram1); + + Buffer::OwnedImpl response; + auto size = PrometheusStatsFormatter::statsAsPrometheus( + counters_, gauges_, histograms_, response, false, + absl::optional{std::regex("cluster.test_1.upstream_cx_total")}); + EXPECT_EQ(1UL, size); + + const std::string expected_output = + R"EOF(# TYPE envoy_cluster_test_1_upstream_cx_total counter +envoy_cluster_test_1_upstream_cx_total{a_tag_name="a.tag-value"} 0 +)EOF"; + + EXPECT_EQ(expected_output, response.toString()); +} + +} // namespace Server +} // namespace Envoy diff --git a/test/server/listener_manager_impl_test.cc b/test/server/listener_manager_impl_test.cc index c2713bcc48b4..04072b8beca0 100644 --- a/test/server/listener_manager_impl_test.cc +++ b/test/server/listener_manager_impl_test.cc @@ -275,7 +275,7 @@ TEST_F(ListenerManagerImplWithRealFiltersTest, UdpAddress) { EXPECT_TRUE(Protobuf::TextFormat::ParseFromString(proto_text, &listener_proto)); EXPECT_CALL(server_.random_, uuid()); - EXPECT_CALL(*worker_, addListener(_, _)); + EXPECT_CALL(*worker_, addListener(_, _, _)); EXPECT_CALL(listener_factory_, createListenSocket(_, Network::Address::SocketType::Datagram, _, {{true, false}})); EXPECT_CALL(os_sys_calls_, setsockopt_(_, _, _, _, _)).Times(testing::AtLeast(1)); @@ -791,6 +791,115 @@ version_info: version1 } } +TEST_F(ListenerManagerImplTest, OverrideListener) { + time_system_.setSystemTime(std::chrono::milliseconds(1001001001001)); + + InSequence s; + + auto* lds_api = new MockLdsApi(); + EXPECT_CALL(listener_factory_, createLdsApi_(_)).WillOnce(Return(lds_api)); + envoy::config::core::v3::ConfigSource lds_config; + manager_->createLdsApi(lds_config); + + // Add foo listener. + const std::string listener_foo_yaml = R"EOF( +name: "foo" +address: + socket_address: + address: "127.0.0.1" + port_value: 1234 +filter_chains: {} + )EOF"; + + ListenerHandle* listener_foo = expectListenerCreate(false, true); + EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, {true})); + EXPECT_TRUE( + manager_->addOrUpdateListener(parseListenerFromV2Yaml(listener_foo_yaml), "version1", true)); + checkStats(1, 0, 0, 0, 1, 0); + + // Start workers and capture ListenerImpl. + Network::ListenerConfig* listener_config = nullptr; + EXPECT_CALL(*worker_, addListener(_, _, _)) + .WillOnce(Invoke([&listener_config](auto, Network::ListenerConfig& config, auto) -> void { + listener_config = &config; + })) + .RetiresOnSaturation(); + + EXPECT_CALL(*worker_, start(_)); + manager_->startWorkers(guard_dog_); + + EXPECT_EQ(0, server_.stats_store_.counter("listener_manager.listener_create_success").value()); + + // TODO(lambdai): No need to invoke `addListenerToWorkerForTest` explicitly when intelligent warm + // up procedure is added. + ListenerImpl* listener_impl = dynamic_cast(listener_config); + auto overridden_listener = absl::make_optional(1); + EXPECT_CALL(*worker_, addListener(_, _, _)) + .WillOnce(Invoke([](absl::optional, Network::ListenerConfig&, + auto completion) -> void { completion(true); })); + manager_->addListenerToWorkerForTest(*worker_, overridden_listener, *listener_impl, nullptr); + + EXPECT_EQ(1, server_.stats_store_.counter("listener_manager.listener_create_success").value()); + EXPECT_CALL(*listener_foo, onDestroy()); +} + +TEST_F(ListenerManagerImplTest, DrainFilterChains) { + time_system_.setSystemTime(std::chrono::milliseconds(1001001001001)); + + InSequence s; + + auto* lds_api = new MockLdsApi(); + EXPECT_CALL(listener_factory_, createLdsApi_(_)).WillOnce(Return(lds_api)); + envoy::config::core::v3::ConfigSource lds_config; + manager_->createLdsApi(lds_config); + + // Add foo listener. + const std::string listener_foo_yaml = R"EOF( +name: "foo" +address: + socket_address: + address: "127.0.0.1" + port_value: 1234 +filter_chains: +- filters: + - name: fake + config: {} + )EOF"; + + ListenerHandle* listener_foo = expectListenerCreate(false, true); + EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, {true})); + EXPECT_TRUE( + manager_->addOrUpdateListener(parseListenerFromV2Yaml(listener_foo_yaml), "version1", true)); + checkStats(1, 0, 0, 0, 1, 0); + + // Start workers and capture ListenerImpl. + Network::ListenerConfig* listener_config = nullptr; + EXPECT_CALL(*worker_, addListener(_, _, _)) + .WillOnce(Invoke([&listener_config](auto, Network::ListenerConfig& config, auto) -> void { + listener_config = &config; + })) + .RetiresOnSaturation(); + + EXPECT_CALL(*worker_, start(_)); + manager_->startWorkers(guard_dog_); + + ENVOY_LOG_MISC(debug, "lambdai: config ptr {}", static_cast(listener_config)); + + EXPECT_EQ(0, server_.stats_store_.counter("listener_manager.listener_create_success").value()); + + // TODO(lambdai): No need to invoke `drainFilterChains` explicitly when intelligent warm + // up procedure is added. + ListenerImpl* listener_impl = dynamic_cast(listener_config); + ASSERT(listener_impl != nullptr); + auto* timer = new Event::MockTimer(dynamic_cast(&server_.dispatcher())); + EXPECT_CALL(*timer, enableTimer(_, _)); + manager_->drainFilterChainsForTest(listener_impl); + EXPECT_CALL(*worker_, removeFilterChains(_, _, _)); + timer->invokeCallback(); + EXPECT_CALL(*listener_foo, onDestroy()); + worker_->callDrainFilterChainsComplete(); +} + TEST_F(ListenerManagerImplTest, AddOrUpdateListener) { time_system_.setSystemTime(std::chrono::milliseconds(1001001001001)); @@ -892,7 +1001,7 @@ version_info: version2 .value()); // Start workers. - EXPECT_CALL(*worker_, addListener(_, _)); + EXPECT_CALL(*worker_, addListener(_, _, _)); EXPECT_CALL(*worker_, start(_)); manager_->startWorkers(guard_dog_); // Validate that workers_started stat is still zero before workers set the status via @@ -918,7 +1027,7 @@ version_info: version2 // Update foo. Should go into warming, have an immediate warming callback, and start immediate // removal. ListenerHandle* listener_foo_update2 = expectListenerCreate(false, true); - EXPECT_CALL(*worker_, addListener(_, _)); + EXPECT_CALL(*worker_, addListener(_, _, _)); EXPECT_CALL(*worker_, stopListener(_, _)); EXPECT_CALL(*listener_foo_update1->drain_manager_, startDrainSequence(_)); EXPECT_TRUE( @@ -981,7 +1090,7 @@ filter_chains: {} ListenerHandle* listener_bar = expectListenerCreate(false, true); EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, {true})); - EXPECT_CALL(*worker_, addListener(_, _)); + EXPECT_CALL(*worker_, addListener(_, _, _)); EXPECT_TRUE( manager_->addOrUpdateListener(parseListenerFromV2Yaml(listener_bar_yaml), "version4", true)); EXPECT_EQ(2UL, manager_->listeners().size()); @@ -1084,7 +1193,7 @@ name: baz checkStats(3, 3, 0, 1, 2, 0); // Finish initialization for baz which should make it active. - EXPECT_CALL(*worker_, addListener(_, _)); + EXPECT_CALL(*worker_, addListener(_, _, _)); listener_baz_update1->target_.ready(); EXPECT_EQ(3UL, manager_->listeners().size()); worker_->callAddCompletion(true); @@ -1118,7 +1227,7 @@ name: foo ListenerHandle* listener_foo = expectListenerCreate(false, true); EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, {true})); - EXPECT_CALL(*worker_, addListener(_, _)); + EXPECT_CALL(*worker_, addListener(_, _, _)); EXPECT_TRUE(manager_->addOrUpdateListener(parseListenerFromV2Yaml(listener_foo_yaml), "", true)); worker_->callAddCompletion(true); checkStats(1, 0, 0, 0, 1, 0); @@ -1140,7 +1249,7 @@ name: foo // Add foo again. We should use the socket from draining. ListenerHandle* listener_foo2 = expectListenerCreate(false, true); - EXPECT_CALL(*worker_, addListener(_, _)); + EXPECT_CALL(*worker_, addListener(_, _, _)); EXPECT_TRUE(manager_->addOrUpdateListener(parseListenerFromV2Yaml(listener_foo_yaml), "", true)); worker_->callAddCompletion(true); checkStats(2, 0, 1, 0, 1, 1); @@ -1178,7 +1287,7 @@ name: foo ListenerHandle* listener_foo = expectListenerCreate(false, true); EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, {true})); - EXPECT_CALL(*worker_, addListener(_, _)); + EXPECT_CALL(*worker_, addListener(_, _, _)); EXPECT_TRUE(manager_->addOrUpdateListener(parseListenerFromV2Yaml(listener_foo_yaml), "", true)); worker_->callAddCompletion(true); checkStats(1, 0, 0, 0, 1, 0); @@ -1196,7 +1305,7 @@ name: foo // Add foo again. We should use the socket from draining. ListenerHandle* listener_foo2 = expectListenerCreate(false, true); EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, {true})); - EXPECT_CALL(*worker_, addListener(_, _)); + EXPECT_CALL(*worker_, addListener(_, _, _)); EXPECT_TRUE(manager_->addOrUpdateListener(parseListenerFromV2Yaml(listener_foo_yaml), "", true)); worker_->callAddCompletion(true); checkStats(2, 0, 1, 0, 1, 1); @@ -1398,7 +1507,7 @@ name: foo ListenerHandle* listener_foo = expectListenerCreate(false, true); EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, {true})); - EXPECT_CALL(*worker_, addListener(_, _)); + EXPECT_CALL(*worker_, addListener(_, _, _)); EXPECT_TRUE(manager_->addOrUpdateListener(parseListenerFromV2Yaml(listener_foo_yaml), "", true)); worker_->callAddCompletion(true); checkStats(1, 0, 0, 0, 1, 0); @@ -1469,7 +1578,7 @@ name: foo EXPECT_CALL(listener_foo->target_, initialize()); EXPECT_TRUE(manager_->addOrUpdateListener(parseListenerFromV2Yaml(listener_foo_yaml), "", true)); checkStats(2, 0, 1, 1, 0, 0); - EXPECT_CALL(*worker_, addListener(_, _)); + EXPECT_CALL(*worker_, addListener(_, _, _)); listener_foo->target_.ready(); worker_->callAddCompletion(true); EXPECT_EQ(1UL, manager_->listeners().size()); @@ -1536,7 +1645,7 @@ traffic_direction: INBOUND EXPECT_CALL(listener_foo->target_, initialize()); EXPECT_TRUE(manager_->addOrUpdateListener(parseListenerFromV2Yaml(listener_foo_yaml), "", true)); checkStats(1, 0, 0, 1, 0, 0); - EXPECT_CALL(*worker_, addListener(_, _)); + EXPECT_CALL(*worker_, addListener(_, _, _)); listener_foo->target_.ready(); worker_->callAddCompletion(true); EXPECT_EQ(1UL, manager_->listeners().size()); @@ -1559,7 +1668,7 @@ traffic_direction: OUTBOUND EXPECT_CALL(listener_foo_outbound->target_, initialize()); EXPECT_TRUE( manager_->addOrUpdateListener(parseListenerFromV2Yaml(listener_foo_outbound_yaml), "", true)); - EXPECT_CALL(*worker_, addListener(_, _)); + EXPECT_CALL(*worker_, addListener(_, _, _)); listener_foo_outbound->target_.ready(); worker_->callAddCompletion(true); EXPECT_EQ(2UL, manager_->listeners().size()); @@ -1584,7 +1693,7 @@ traffic_direction: OUTBOUND ListenerHandle* listener_bar_outbound = expectListenerCreate(false, true); EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, {true})); - EXPECT_CALL(*worker_, addListener(_, _)); + EXPECT_CALL(*worker_, addListener(_, _, _)); EXPECT_TRUE( manager_->addOrUpdateListener(parseListenerFromV2Yaml(listener_bar_outbound_yaml), "", true)); EXPECT_EQ(3UL, manager_->listeners().size()); @@ -1630,7 +1739,7 @@ name: foo EXPECT_CALL(listener_foo->target_, initialize()); EXPECT_TRUE(manager_->addOrUpdateListener(parseListenerFromV2Yaml(listener_foo_yaml), "", true)); checkStats(1, 0, 0, 1, 0, 0); - EXPECT_CALL(*worker_, addListener(_, _)); + EXPECT_CALL(*worker_, addListener(_, _, _)); listener_foo->target_.ready(); worker_->callAddCompletion(true); EXPECT_EQ(1UL, manager_->listeners().size()); @@ -1679,7 +1788,7 @@ traffic_direction: INBOUND EXPECT_CALL(listener_foo->target_, initialize()); EXPECT_TRUE(manager_->addOrUpdateListener(parseListenerFromV2Yaml(listener_foo_yaml), "", true)); checkStats(1, 0, 0, 1, 0, 0); - EXPECT_CALL(*worker_, addListener(_, _)); + EXPECT_CALL(*worker_, addListener(_, _, _)); listener_foo->target_.ready(); worker_->callAddCompletion(true); EXPECT_EQ(1UL, manager_->listeners().size()); @@ -1733,7 +1842,7 @@ name: foo ListenerHandle* listener_foo = expectListenerCreate(false, true); EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, {true})); - EXPECT_CALL(*worker_, addListener(_, _)); + EXPECT_CALL(*worker_, addListener(_, _, _)); EXPECT_TRUE(manager_->addOrUpdateListener(parseListenerFromV2Yaml(listener_foo_yaml), "", true)); EXPECT_CALL(*worker_, stopListener(_, _)); @@ -1812,7 +1921,7 @@ name: bar "error adding listener: 'bar' has duplicate address '0.0.0.0:1234' as existing listener"); // Move foo to active and then try to add again. This should still fail. - EXPECT_CALL(*worker_, addListener(_, _)); + EXPECT_CALL(*worker_, addListener(_, _, _)); listener_foo->target_.ready(); worker_->callAddCompletion(true); diff --git a/test/server/server_corpus/valid b/test/server/server_corpus/valid new file mode 100644 index 000000000000..7a7a63537434 --- /dev/null +++ b/test/server/server_corpus/valid @@ -0,0 +1,90 @@ +static_resources { + clusters { + name: "ser" + connect_timeout { + nanos: 813 + } + per_connection_buffer_limit_bytes { + } + lb_policy: RING_HASH + health_checks { + timeout { + nanos: 95 + } + interval { + seconds: 165537 + nanos: 95 + } + unhealthy_threshold { + } + healthy_threshold { + } + http_health_check { + host: "a\037\037\037\037\037\037\037\037\037\000\037\037\037\037\037\037\037\037\037\037\037\001\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\031\037\037\037\037\037\037\037\037\037\037\037\037" + path: "&" + } + healthy_edge_interval { + nanos: 54784 + } + interval_jitter_percent: 16 + } + http_protocol_options { + header_key_format { + proper_case_words { + } + } + } + load_assignment { + cluster_name: "." + endpoints { + locality { + sub_zone: "\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\001\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037" + } + lb_endpoints { + endpoint { + address { + pipe { + path: "." + } + } + } + health_status: DRAINING + } + } + endpoints { + locality { + region: "." + sub_zone: "\037\037\037\037\037\037\037\037\037\000\037\037\037\037\037\037\037\037\037\037\037\001\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\037\031\037\037\037\037\037\037\037\037\037\037\037\037" + } + lb_endpoints { + endpoint { + address { + pipe { + path: "." + } + } + } + health_status: DRAINING + } + } + endpoints { + priority: 16 + } + endpoints { + } + endpoints { + lb_endpoints { + endpoint { + address { + pipe { + path: "." + } + } + } + health_status: HEALTHY + } + } + } + } +} +enable_dispatcher_stats: true \ No newline at end of file diff --git a/test/server/worker_impl_test.cc b/test/server/worker_impl_test.cc index f998a01b1d3a..02afcc0a50fd 100644 --- a/test/server/worker_impl_test.cc +++ b/test/server/worker_impl_test.cc @@ -61,12 +61,13 @@ TEST_F(WorkerImplTest, BasicFlow) { // thread starts running. NiceMock listener; ON_CALL(listener, listenerTag()).WillByDefault(Return(1UL)); - EXPECT_CALL(*handler_, addListener(_)) - .WillOnce(Invoke([current_thread_id](Network::ListenerConfig& config) -> void { - EXPECT_EQ(config.listenerTag(), 1UL); - EXPECT_NE(current_thread_id, std::this_thread::get_id()); - })); - worker_.addListener(listener, [&ci](bool success) -> void { + EXPECT_CALL(*handler_, addListener(_, _)) + .WillOnce(Invoke( + [current_thread_id](absl::optional, Network::ListenerConfig& config) -> void { + EXPECT_EQ(config.listenerTag(), 1UL); + EXPECT_NE(current_thread_id, std::this_thread::get_id()); + })); + worker_.addListener(absl::nullopt, listener, [&ci](bool success) -> void { EXPECT_TRUE(success); ci.setReady(); }); @@ -79,12 +80,13 @@ TEST_F(WorkerImplTest, BasicFlow) { // After a worker is started adding/stopping/removing a listener happens on the worker thread. NiceMock listener2; ON_CALL(listener2, listenerTag()).WillByDefault(Return(2UL)); - EXPECT_CALL(*handler_, addListener(_)) - .WillOnce(Invoke([current_thread_id](Network::ListenerConfig& config) -> void { - EXPECT_EQ(config.listenerTag(), 2UL); - EXPECT_NE(current_thread_id, std::this_thread::get_id()); - })); - worker_.addListener(listener2, [&ci](bool success) -> void { + EXPECT_CALL(*handler_, addListener(_, _)) + .WillOnce(Invoke( + [current_thread_id](absl::optional, Network::ListenerConfig& config) -> void { + EXPECT_EQ(config.listenerTag(), 2UL); + EXPECT_NE(current_thread_id, std::this_thread::get_id()); + })); + worker_.addListener(absl::nullopt, listener2, [&ci](bool success) -> void { EXPECT_TRUE(success); ci.setReady(); }); @@ -118,12 +120,13 @@ TEST_F(WorkerImplTest, BasicFlow) { // Now test adding and removing a listener without stopping it first. NiceMock listener3; ON_CALL(listener3, listenerTag()).WillByDefault(Return(3UL)); - EXPECT_CALL(*handler_, addListener(_)) - .WillOnce(Invoke([current_thread_id](Network::ListenerConfig& config) -> void { - EXPECT_EQ(config.listenerTag(), 3UL); - EXPECT_NE(current_thread_id, std::this_thread::get_id()); - })); - worker_.addListener(listener3, [&ci](bool success) -> void { + EXPECT_CALL(*handler_, addListener(_, _)) + .WillOnce(Invoke( + [current_thread_id](absl::optional, Network::ListenerConfig& config) -> void { + EXPECT_EQ(config.listenerTag(), 3UL); + EXPECT_NE(current_thread_id, std::this_thread::get_id()); + })); + worker_.addListener(absl::nullopt, listener3, [&ci](bool success) -> void { EXPECT_TRUE(success); ci.setReady(); }); @@ -145,9 +148,9 @@ TEST_F(WorkerImplTest, ListenerException) { NiceMock listener; ON_CALL(listener, listenerTag()).WillByDefault(Return(1UL)); - EXPECT_CALL(*handler_, addListener(_)) + EXPECT_CALL(*handler_, addListener(_, _)) .WillOnce(Throw(Network::CreateListenerException("failed"))); - worker_.addListener(listener, [](bool success) -> void { EXPECT_FALSE(success); }); + worker_.addListener(absl::nullopt, listener, [](bool success) -> void { EXPECT_FALSE(success); }); worker_.start(guard_dog_); worker_.stop(); diff --git a/test/test_common/network_utility.cc b/test/test_common/network_utility.cc index 6d4d4fb09bad..1f40191733f1 100644 --- a/test/test_common/network_utility.cc +++ b/test/test_common/network_utility.cc @@ -159,6 +159,18 @@ bool supportsIpVersion(const Address::IpVersion version) { return true; } +std::string ipVersionToDnsFamily(Network::Address::IpVersion version) { + switch (version) { + case Network::Address::IpVersion::v4: + return "V4_ONLY"; + case Network::Address::IpVersion::v6: + return "V6_ONLY"; + } + + // This seems to be needed on the coverage build for some reason. + NOT_REACHED_GCOVR_EXCL_LINE; +} + std::pair bindFreeLoopbackPort(Address::IpVersion version, Address::SocketType type) { Address::InstanceConstSharedPtr addr = getCanonicalLoopbackAddress(version); diff --git a/test/test_common/network_utility.h b/test/test_common/network_utility.h index 4a7a7d05a4c4..965ec8b6dfad 100644 --- a/test/test_common/network_utility.h +++ b/test/test_common/network_utility.h @@ -100,6 +100,12 @@ Address::InstanceConstSharedPtr getAnyAddress(const Address::IpVersion version, */ bool supportsIpVersion(const Address::IpVersion version); +/** + * Returns the DNS family for the specified IP version. + * @param version the IP version of the DNS lookup family. + */ +std::string ipVersionToDnsFamily(Network::Address::IpVersion version); + /** * Bind a socket to a free port on a loopback address, and return the socket's fd and bound address. * Enables a test server to reliably "select" a port to listen on. Note that the socket option diff --git a/test/test_common/utility.cc b/test/test_common/utility.cc index 89e8eda51fb8..09e7ee1aff56 100644 --- a/test/test_common/utility.cc +++ b/test/test_common/utility.cc @@ -141,6 +141,11 @@ Stats::GaugeSharedPtr TestUtility::findGauge(Stats::Store& store, const std::str return findByName(store.gauges(), name); } +Stats::TextReadoutSharedPtr TestUtility::findTextReadout(Stats::Store& store, + const std::string& name) { + return findByName(store.textReadouts(), name); +} + void TestUtility::waitForCounterEq(Stats::Store& store, const std::string& name, uint64_t value, Event::TestTimeSystem& time_system) { while (findCounter(store, name) == nullptr || findCounter(store, name)->value() != value) { diff --git a/test/test_common/utility.h b/test/test_common/utility.h index 749bd4ea9b44..e66100184743 100644 --- a/test/test_common/utility.h +++ b/test/test_common/utility.h @@ -236,6 +236,14 @@ class TestUtility { static void waitForGaugeEq(Stats::Store& store, const std::string& name, uint64_t value, Event::TestTimeSystem& time_system); + /** + * Find a readout in a stats store. + * @param store supplies the stats store. + * @param name supplies the name to search for. + * @return Stats::TextReadoutSharedPtr the readout or nullptr if there is none. + */ + static Stats::TextReadoutSharedPtr findTextReadout(Stats::Store& store, const std::string& name); + /** * Convert a string list of IP addresses into a list of network addresses usable for DNS * response testing. diff --git a/tools/code_format/check_format.py b/tools/code_format/check_format.py index 87a0693e56a4..c9683f8cce4d 100755 --- a/tools/code_format/check_format.py +++ b/tools/code_format/check_format.py @@ -71,17 +71,16 @@ "upstream_cx_length_ms") # Files in these paths can use std::regex -STD_REGEX_WHITELIST = ("./source/common/common/utility.cc", "./source/common/common/regex.h", - "./source/common/common/regex.cc", - "./source/common/stats/tag_extractor_impl.h", - "./source/common/stats/tag_extractor_impl.cc", - "./source/common/access_log/access_log_formatter.cc", - "./source/extensions/filters/http/squash/squash_filter.h", - "./source/extensions/filters/http/squash/squash_filter.cc", - "./source/server/http/admin.h", "./source/server/http/admin.cc", - "./tools/clang_tools/api_booster/main.cc", - "./tools/clang_tools/api_booster/proto_cxx_utils.cc", - "./source/common/common/version.cc") +STD_REGEX_WHITELIST = ( + "./source/common/common/utility.cc", "./source/common/common/regex.h", + "./source/common/common/regex.cc", "./source/common/stats/tag_extractor_impl.h", + "./source/common/stats/tag_extractor_impl.cc", + "./source/common/access_log/access_log_formatter.cc", + "./source/extensions/filters/http/squash/squash_filter.h", + "./source/extensions/filters/http/squash/squash_filter.cc", "./source/server/http/utils.h", + "./source/server/http/utils.cc", "./source/server/http/stats_handler.h", + "./source/server/http/stats_handler.cc", "./tools/clang_tools/api_booster/main.cc", + "./tools/clang_tools/api_booster/proto_cxx_utils.cc", "./source/common/common/version.cc") # Only one C++ file should instantiate grpc_init GRPC_INIT_WHITELIST = ("./source/common/grpc/google_grpc_context.cc") @@ -664,8 +663,9 @@ def checkSourceLine(line, file_path, reportError): if isInSubdir(file_path, 'source') and file_path.endswith('.cc') and \ ('.counterFromString(' in line or '.gaugeFromString(' in line or \ - '.histogramFromString(' in line or '->counterFromString(' in line or \ - '->gaugeFromString(' in line or '->histogramFromString(' in line): + '.histogramFromString(' in line or '.textReadoutFromString(' in line or \ + '->counterFromString(' in line or '->gaugeFromString(' in line or \ + '->histogramFromString(' in line or '->textReadoutFromString(' in line): reportError("Don't lookup stats by name at runtime; use StatName saved during construction") if re.search("envoy::[a-z0-9_:]+::[A-Z][a-z]\w*_\w*_[A-Z]{2}", line): diff --git a/tools/deprecate_version/deprecate_version.py b/tools/deprecate_version/deprecate_version.py index e35edd2db009..aba1579734a0 100644 --- a/tools/deprecate_version/deprecate_version.py +++ b/tools/deprecate_version/deprecate_version.py @@ -47,26 +47,6 @@ class DeprecateVersionError(Exception): pass -# Figure out map from version to set of commits. -def GetHistory(): - """Obtain mapping from release version to docs/root/intro/deprecated.rst PRs. - - Returns: - A dictionary mapping from release version to a set of git commit objects. - """ - repo = Repo(os.getcwd()) - version = None - history = defaultdict(set) - for commit, lines in repo.blame('HEAD', 'docs/root/intro/deprecated.rst'): - for line in lines: - sr = re.match('## Version (.*) \(.*\)', line) - if sr: - version = sr.group(1) - continue - history[version].add(commit) - return history - - def GetConfirmation(): """Obtain stdin confirmation to create issues in GH.""" return input('Creates issues? [yN] ').strip().lower() in ('y', 'yes') diff --git a/tools/deprecate_version/requirements.txt b/tools/deprecate_version/requirements.txt index dc2a917a768e..8e654fb20fae 100644 --- a/tools/deprecate_version/requirements.txt +++ b/tools/deprecate_version/requirements.txt @@ -1,2 +1,2 @@ -GitPython==3.0.0 +GitPython==3.1.1 PyGithub==1.43.8 diff --git a/tools/protoxform/protoprint.py b/tools/protoxform/protoprint.py index 51bb9c573795..804b93b86600 100755 --- a/tools/protoxform/protoprint.py +++ b/tools/protoxform/protoprint.py @@ -315,6 +315,33 @@ def NormalizeFieldTypeName(type_context, field_fqn): remaining_field_fqn_splits = deque(field_fqn_splits[:-1]) normalized_splits = deque([field_fqn_splits[-1]]) + if list(remaining_field_fqn_splits)[:1] != type_context_splits[:1] and ( + len(remaining_field_fqn_splits) == 0 or + remaining_field_fqn_splits[0] in type_context_splits[1:]): + # Notice that in some cases it is error-prone to normalize a type name. + # E.g., it would be an error to replace ".external.Type" with "external.Type" + # in the context of "envoy.extensions.type.external.vX.Config". + # In such a context protoc resolves "external.Type" into + # "envoy.extensions.type.external.Type", which is exactly what the use of a + # fully-qualified name ".external.Type" was meant to prevent. + # + # A type SHOULD remain fully-qualified under the following conditions: + # 1. its root package is different from the root package of the context type + # 2. EITHER the type doesn't belong to any package at all + # OR its root package has a name that collides with one of the packages + # of the context type + # + # E.g., + # a) although ".some.Type" has a different root package than the context type + # "TopLevelType", it is still safe to normalize it into "some.Type" + # b) although ".google.protobuf.Any" has a different root package than the context type + # "envoy.api.v2.Cluster", it still safe to normalize it into "google.protobuf.Any" + # c) it is error-prone to normalize ".TopLevelType" in the context of "some.Type" + # into "TopLevelType" + # d) it is error-prone to normalize ".external.Type" in the context of + # "envoy.extensions.type.external.vX.Config" into "external.Type" + return field_fqn + def EquivalentInTypeContext(splits): type_context_splits_tmp = deque(type_context_splits) while type_context_splits_tmp: diff --git a/tools/protoxform/protoxform_test_helper.py b/tools/protoxform/protoxform_test_helper.py index dfa4c3ab7813..38c630cd6b5f 100755 --- a/tools/protoxform/protoxform_test_helper.py +++ b/tools/protoxform/protoxform_test_helper.py @@ -135,7 +135,6 @@ def Run(cmd, path, filename, version): path, filename = PathAndFilename(target) messages += Run(cmd, path, filename, 'active_or_frozen') messages += Run(cmd, path, filename, 'next_major_version_candidate') - messages += Run(cmd, path, filename, 'next_major_version_candidate') messages += Run(cmd, path, filename, 'next_major_version_candidate.envoy_internal') if len(messages) == 0: diff --git a/tools/spelling/check_spelling_pedantic.py b/tools/spelling/check_spelling_pedantic.py index b30a48bdcb3e..d7b4b7d4cb77 100755 --- a/tools/spelling/check_spelling_pedantic.py +++ b/tools/spelling/check_spelling_pedantic.py @@ -803,6 +803,13 @@ def execute(files, dictionary_file, fix): if not paths: paths = ['./api', './include', './source', './test', './tools'] + # Exclude ./third_party/ directory from spell checking, even when requested through arguments. + # Otherwise git pre-push hook checks it for merged commits. + paths = [ + path for path in paths + if not path.startswith('./third_party/') and not path.startswith('./third_party/') + ] + exts = ['.cc', '.h', '.proto'] if args.test_ignore_exts: exts = None diff --git a/tools/spelling/check_spelling_pedantic_test.py b/tools/spelling/check_spelling_pedantic_test.py index 91395805f110..494a78275736 100755 --- a/tools/spelling/check_spelling_pedantic_test.py +++ b/tools/spelling/check_spelling_pedantic_test.py @@ -57,14 +57,18 @@ def checkFileExpectingErrors(filename, expected_substrings): return expectError(filename, status, stdout, expected_substrings) -def checkFileExpectingOK(filename): - command, status, stdout = runCheckFormat("check", getInputFile(filename)) +def checkFilePathExpectingOK(filename): + command, status, stdout = runCheckFormat("check", filename) if status != 0: logging.error("Expected %s to have no errors; status=%d, output:\n" % (filename, status)) emitStdoutAsError(stdout) return status +def checkFileExpectingOK(filename): + return checkFilePathExpectingOK(getInputFile(filename)) + + def runChecks(): errors = 0 @@ -72,6 +76,9 @@ def runChecks(): errors += checkFileExpectingOK("skip_file") errors += checkFileExpectingOK("exclusions") + errors += checkFileExpectingOK("third_party/something/file.cc") + errors += checkFileExpectingOK("./third_party/something/file.cc") + errors += checkFileExpectingErrors("typos", ["spacific", "reelistic", "Awwful", "combeenations", "woork"]) errors += checkFileExpectingErrors( diff --git a/tools/testdata/protoxform/envoy/v2/BUILD b/tools/testdata/protoxform/envoy/v2/BUILD index f381f26cfa5f..08fcd5836915 100644 --- a/tools/testdata/protoxform/envoy/v2/BUILD +++ b/tools/testdata/protoxform/envoy/v2/BUILD @@ -4,12 +4,14 @@ proto_library( name = "fix_protos", srcs = [ "discovery_service.proto", + "fully_qualified_names.proto", "oneof.proto", "package_move.proto", "sample.proto", ], visibility = ["//visibility:public"], deps = [ + "//tools/testdata/protoxform/external:external_protos", "@com_github_cncf_udpa//udpa/annotations:pkg", "@envoy_api//envoy/annotations:pkg", "@envoy_api//envoy/api/v2:pkg", diff --git a/tools/testdata/protoxform/envoy/v2/fully_qualified_names.proto b/tools/testdata/protoxform/envoy/v2/fully_qualified_names.proto new file mode 100644 index 000000000000..2fff0bbed155 --- /dev/null +++ b/tools/testdata/protoxform/envoy/v2/fully_qualified_names.proto @@ -0,0 +1,31 @@ +syntax = "proto3"; + +package envoy.v2; + +import "envoy/api/v2/core/base.proto"; + +import "tools/testdata/protoxform/external/root_type.proto"; +import "tools/testdata/protoxform/external/package_type.proto"; + +import "google/protobuf/any.proto"; + +import "udpa/annotations/migrate.proto"; +import "udpa/annotations/status.proto"; + +option (udpa.annotations.file_migrate).move_to_package = "envoy.external.v3"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// Verifies normalization of fully-qualified type names. +message UsesFullyQualifiedTypeNames { + + envoy.api.v2.core.Locality another_envoy_type = 1; + .envoy.api.v2.core.Locality another_envoy_type_fqn = 2; + + google.protobuf.Any google_protobuf_any = 3; + .google.protobuf.Any google_protobuf_any_fqn = 4; + + external.PackageLevelType external_package_level_type = 5; + .external.PackageLevelType external_package_level_type_fqn = 6; + + .RootLevelType external_root_level_type_fqn = 7; +} diff --git a/tools/testdata/protoxform/envoy/v2/fully_qualified_names.proto.active_or_frozen.gold b/tools/testdata/protoxform/envoy/v2/fully_qualified_names.proto.active_or_frozen.gold new file mode 100644 index 000000000000..55d7af193dcf --- /dev/null +++ b/tools/testdata/protoxform/envoy/v2/fully_qualified_names.proto.active_or_frozen.gold @@ -0,0 +1,37 @@ +syntax = "proto3"; + +package envoy.v2; + +import "envoy/api/v2/core/base.proto"; + +import "google/protobuf/any.proto"; + +import "tools/testdata/protoxform/external/package_type.proto"; +import "tools/testdata/protoxform/external/root_type.proto"; + +import "udpa/annotations/migrate.proto"; +import "udpa/annotations/status.proto"; + +option java_package = "io.envoyproxy.envoy.v2"; +option java_outer_classname = "FullyQualifiedNamesProto"; +option java_multiple_files = true; +option (udpa.annotations.file_migrate).move_to_package = "envoy.external.v3"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// Verifies normalization of fully-qualified type names. +// [#next-free-field: 8] +message UsesFullyQualifiedTypeNames { + api.v2.core.Locality another_envoy_type = 1; + + api.v2.core.Locality another_envoy_type_fqn = 2; + + google.protobuf.Any google_protobuf_any = 3; + + google.protobuf.Any google_protobuf_any_fqn = 4; + + external.PackageLevelType external_package_level_type = 5; + + external.PackageLevelType external_package_level_type_fqn = 6; + + .RootLevelType external_root_level_type_fqn = 7; +} diff --git a/tools/testdata/protoxform/envoy/v2/fully_qualified_names.proto.next_major_version_candidate.envoy_internal.gold b/tools/testdata/protoxform/envoy/v2/fully_qualified_names.proto.next_major_version_candidate.envoy_internal.gold new file mode 100644 index 000000000000..fe6bb1585b87 --- /dev/null +++ b/tools/testdata/protoxform/envoy/v2/fully_qualified_names.proto.next_major_version_candidate.envoy_internal.gold @@ -0,0 +1,39 @@ +syntax = "proto3"; + +package envoy.external.v3; + +import "envoy/api/v2/core/base.proto"; + +import "google/protobuf/any.proto"; + +import "tools/testdata/protoxform/external/package_type.proto"; +import "tools/testdata/protoxform/external/root_type.proto"; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; + +option java_package = "io.envoyproxy.envoy.external.v3"; +option java_outer_classname = "FullyQualifiedNamesProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; + +// Verifies normalization of fully-qualified type names. +// [#next-free-field: 8] +message UsesFullyQualifiedTypeNames { + option (udpa.annotations.versioning).previous_message_type = + "envoy.v2.UsesFullyQualifiedTypeNames"; + + api.v2.core.Locality another_envoy_type = 1; + + api.v2.core.Locality another_envoy_type_fqn = 2; + + google.protobuf.Any google_protobuf_any = 3; + + google.protobuf.Any google_protobuf_any_fqn = 4; + + .external.PackageLevelType external_package_level_type = 5; + + .external.PackageLevelType external_package_level_type_fqn = 6; + + .RootLevelType external_root_level_type_fqn = 7; +} diff --git a/tools/testdata/protoxform/envoy/v2/fully_qualified_names.proto.next_major_version_candidate.gold b/tools/testdata/protoxform/envoy/v2/fully_qualified_names.proto.next_major_version_candidate.gold new file mode 100644 index 000000000000..fe6bb1585b87 --- /dev/null +++ b/tools/testdata/protoxform/envoy/v2/fully_qualified_names.proto.next_major_version_candidate.gold @@ -0,0 +1,39 @@ +syntax = "proto3"; + +package envoy.external.v3; + +import "envoy/api/v2/core/base.proto"; + +import "google/protobuf/any.proto"; + +import "tools/testdata/protoxform/external/package_type.proto"; +import "tools/testdata/protoxform/external/root_type.proto"; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; + +option java_package = "io.envoyproxy.envoy.external.v3"; +option java_outer_classname = "FullyQualifiedNamesProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; + +// Verifies normalization of fully-qualified type names. +// [#next-free-field: 8] +message UsesFullyQualifiedTypeNames { + option (udpa.annotations.versioning).previous_message_type = + "envoy.v2.UsesFullyQualifiedTypeNames"; + + api.v2.core.Locality another_envoy_type = 1; + + api.v2.core.Locality another_envoy_type_fqn = 2; + + google.protobuf.Any google_protobuf_any = 3; + + google.protobuf.Any google_protobuf_any_fqn = 4; + + .external.PackageLevelType external_package_level_type = 5; + + .external.PackageLevelType external_package_level_type_fqn = 6; + + .RootLevelType external_root_level_type_fqn = 7; +} diff --git a/tools/testdata/protoxform/external/BUILD b/tools/testdata/protoxform/external/BUILD new file mode 100644 index 000000000000..96986f3e19ae --- /dev/null +++ b/tools/testdata/protoxform/external/BUILD @@ -0,0 +1,11 @@ +licenses(["notice"]) # Apache 2 + +proto_library( + name = "external_protos", + srcs = [ + "package_type.proto", + "root_type.proto", + ], + visibility = ["//visibility:public"], + deps = [], +) diff --git a/tools/testdata/protoxform/external/package_type.proto b/tools/testdata/protoxform/external/package_type.proto new file mode 100644 index 000000000000..c0ef64646549 --- /dev/null +++ b/tools/testdata/protoxform/external/package_type.proto @@ -0,0 +1,10 @@ +syntax = "proto3"; + +package external; + +// Type that belongs to a non-envoy package. +// +// Part of a test suite that verifies normalization of +// fully-qualified type names. +message PackageLevelType { +} diff --git a/tools/testdata/protoxform/external/root_type.proto b/tools/testdata/protoxform/external/root_type.proto new file mode 100644 index 000000000000..2173b65f7508 --- /dev/null +++ b/tools/testdata/protoxform/external/root_type.proto @@ -0,0 +1,8 @@ +syntax = "proto3"; + +// Type that doesn't belong to any package. +// +// Part of a test suite that verifies normalization of +// fully-qualified type names. +message RootLevelType { +}