From 0e9119752d061c1d1439b8d37aa7e4ab2d6d69d1 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Tue, 10 May 2022 10:11:16 -0400 Subject: [PATCH] feat: add built in metrics measure and views --- google-cloud-bigtable-stats/pom.xml | 35 +- .../stats/BuiltinMeasureConstants.java | 92 ++++ .../stats/BuiltinMetricsRecorder.java | 170 +++++++ .../bigtable/stats/BuiltinViewConstants.java | 201 ++++++++ .../cloud/bigtable/stats/BuiltinViews.java | 48 ++ .../stats/MavenPlaceholderShaded.java | 26 -- .../cloud/bigtable/stats/StatsWrapper.java | 133 ++++++ .../stats/BuiltinMetricsRecorderTest.java | 433 ++++++++++++++++++ 8 files changed, 1110 insertions(+), 28 deletions(-) create mode 100644 google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMeasureConstants.java create mode 100644 google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorder.java create mode 100644 google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViewConstants.java create mode 100644 google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViews.java delete mode 100644 google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/MavenPlaceholderShaded.java create mode 100644 google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/StatsWrapper.java create mode 100644 google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorderTest.java diff --git a/google-cloud-bigtable-stats/pom.xml b/google-cloud-bigtable-stats/pom.xml index c965942922..2ccb745eba 100644 --- a/google-cloud-bigtable-stats/pom.xml +++ b/google-cloud-bigtable-stats/pom.xml @@ -29,6 +29,15 @@ + + com.google.api + gax + + + com.google.api + api-common + + io.opencensus opencensus-api @@ -38,8 +47,30 @@ opencensus-impl - io.opencensus - opencensus-exporter-stats-stackdriver + com.google.guava + guava + + + + + com.google.truth.extensions + truth-proto-extension + test + + + com.google.truth + truth + test + + + junit + junit + test + + + io.grpc + grpc-api + test diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMeasureConstants.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMeasureConstants.java new file mode 100644 index 0000000000..0f6a879ab3 --- /dev/null +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMeasureConstants.java @@ -0,0 +1,92 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.stats; + +import static io.opencensus.stats.Measure.MeasureLong; + +import com.google.api.core.InternalApi; +import io.opencensus.tags.TagKey; + +@InternalApi("For internal use only") +public class BuiltinMeasureConstants { + // TagKeys + public static final TagKey PROJECT_ID = TagKey.create("project_id"); + public static final TagKey INSTANCE_ID = TagKey.create("instance_id"); + public static final TagKey APP_PROFILE = TagKey.create("app_profile"); + public static final TagKey METHOD = TagKey.create("method"); + public static final TagKey STREAMING = TagKey.create("streaming"); + public static final TagKey STATUS = TagKey.create("status"); + public static final TagKey CLIENT_NAME = TagKey.create("client_name"); + public static final TagKey CLIENT_ID = TagKey.create("client_id"); + + // Monitored resource TagKeys + public static final TagKey TABLE = TagKey.create("table"); + public static final TagKey CLUSTER = TagKey.create("cluster"); + public static final TagKey ZONE = TagKey.create("zone"); + + // Units + private static final String COUNT = "1"; + private static final String MILLISECOND = "ms"; + + // Measurements + static final MeasureLong OPERATION_LATENCIES = + MeasureLong.create( + "bigtable.googleapis.com/internal/client/operation_latencies", + "Total time until final operation success or failure, including retries and backoff.", + MILLISECOND); + + static final MeasureLong ATTEMPT_LATENCIES = + MeasureLong.create( + "bigtable.googleapis.com/internal/client/attempt_latencies", + "Client observed latency per RPC attempt.", + MILLISECOND); + + static final MeasureLong RETRY_COUNT = + MeasureLong.create( + "bigtable.googleapis.com/internal/client/retry_count", + "The number of additional RPCs sent after the initial attempt.", + COUNT); + + static final MeasureLong FIRST_RESPONSE_LATENCIES = + MeasureLong.create( + "bigtable.googleapis.com/internal/client/first_response_latencies", + "Latency from operation start until the response headers were received. The publishing of the measurement will be delayed until the attempt response has been received.", + MILLISECOND); + + static final MeasureLong SERVER_LATENCIES = + MeasureLong.create( + "bigtable.googleapis.com/internal/client/server_latencies", + "The latency measured from the moment that the RPC entered the Google data center until the RPC was completed.", + MILLISECOND); + + static final MeasureLong CONNECTIVITY_ERROR_COUNT = + MeasureLong.create( + "bigtable.googleapis.com/internal/client/connectivity_error_count", + "Number of requests that failed to reach the Google datacenter. (Requests without google response headers).", + COUNT); + + static final MeasureLong APPLICATION_LATENCIES = + MeasureLong.create( + "bigtable.googleapis.com/internal/client/application_latencies", + "The latency of the client application consuming available response data.", + MILLISECOND); + + static final MeasureLong THROTTLING_LATENCIES = + MeasureLong.create( + "bigtable.googleapis.com/internal/client/throttling_latencies", + "The artificial latency introduced by the client to limit the number of outstanding requests. The publishing of the measurement will be delayed until the attempt trailers have been received.", + MILLISECOND); +} diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorder.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorder.java new file mode 100644 index 0000000000..51a0feb5cb --- /dev/null +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorder.java @@ -0,0 +1,170 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.stats; + +import com.google.api.core.InternalApi; +import com.google.api.gax.tracing.ApiTracerFactory.OperationType; +import com.google.api.gax.tracing.SpanName; +import io.opencensus.stats.MeasureMap; +import io.opencensus.stats.StatsRecorder; +import io.opencensus.tags.TagContext; +import io.opencensus.tags.TagContextBuilder; +import io.opencensus.tags.TagKey; +import io.opencensus.tags.TagValue; +import io.opencensus.tags.Tagger; +import io.opencensus.tags.Tags; +import java.util.Map; + +/** Add built-in metrics to the measure map * */ +@InternalApi("For internal use only") +public class BuiltinMetricsRecorder { + + private final OperationType operationType; + + private final Tagger tagger; + private final StatsRecorder statsRecorder; + private final TagContext parentContext; + private final SpanName spanName; + private final Map statsAttributes; + + private MeasureMap attemptLevelNoStreaming; + private MeasureMap attemptLevelWithStreaming; + private MeasureMap operationLevelNoStreaming; + private MeasureMap operationLevelWithStreaming; + + public BuiltinMetricsRecorder( + OperationType operationType, + SpanName spanName, + Map statsAttributes, + StatsWrapper builtinMetricsWrapper) { + this.operationType = operationType; + this.tagger = Tags.getTagger(); + this.statsRecorder = builtinMetricsWrapper.getStatsRecorder(); + this.spanName = spanName; + this.parentContext = tagger.getCurrentTagContext(); + this.statsAttributes = statsAttributes; + + this.attemptLevelNoStreaming = statsRecorder.newMeasureMap(); + this.attemptLevelWithStreaming = statsRecorder.newMeasureMap(); + this.operationLevelNoStreaming = statsRecorder.newMeasureMap(); + this.operationLevelWithStreaming = statsRecorder.newMeasureMap(); + } + + public void recordAttemptLevelWithoutStreaming( + String status, String tableId, String zone, String cluster) { + TagContextBuilder tagCtx = + newTagContextBuilder(tableId, zone, cluster) + .putLocal(BuiltinMeasureConstants.STATUS, TagValue.create(status)); + + attemptLevelNoStreaming.record(tagCtx.build()); + } + + public void recordAttemptLevelWithStreaming( + String status, String tableId, String zone, String cluster) { + TagContextBuilder tagCtx = + newTagContextBuilder(tableId, zone, cluster) + .putLocal(BuiltinMeasureConstants.STATUS, TagValue.create(status)); + + if (operationType == OperationType.ServerStreaming + && spanName.getMethodName().equals("ReadRows")) { + tagCtx.putLocal(BuiltinMeasureConstants.STREAMING, TagValue.create("true")); + } else { + tagCtx.putLocal(BuiltinMeasureConstants.STREAMING, TagValue.create("false")); + } + + attemptLevelWithStreaming.record(tagCtx.build()); + } + + public void recordOperationLevelWithoutStreaming( + String status, String tableId, String zone, String cluster) { + TagContextBuilder tagCtx = + newTagContextBuilder(tableId, zone, cluster) + .putLocal(BuiltinMeasureConstants.STATUS, TagValue.create(status)); + + operationLevelNoStreaming.record(tagCtx.build()); + } + + public void recordOperationLevelWithStreaming( + String status, String tableId, String zone, String cluster) { + TagContextBuilder tagCtx = + newTagContextBuilder(tableId, zone, cluster) + .putLocal(BuiltinMeasureConstants.STATUS, TagValue.create(status)); + + if (operationType == OperationType.ServerStreaming + && spanName.getMethodName().equals("ReadRows")) { + tagCtx.putLocal(BuiltinMeasureConstants.STREAMING, TagValue.create("true")); + } else { + tagCtx.putLocal(BuiltinMeasureConstants.STREAMING, TagValue.create("false")); + } + + operationLevelWithStreaming.record(tagCtx.build()); + } + + public void recordOperationLatencies(long operationLatency) { + operationLevelWithStreaming.put(BuiltinMeasureConstants.OPERATION_LATENCIES, operationLatency); + } + + public void recordAttemptLatency(long attemptLatency) { + attemptLevelWithStreaming.put(BuiltinMeasureConstants.ATTEMPT_LATENCIES, attemptLatency); + } + + public void recordRetryCount(int attemptCount) { + operationLevelNoStreaming.put(BuiltinMeasureConstants.RETRY_COUNT, attemptCount); + } + + public void recordApplicationLatency(long applicationLatency) { + operationLevelWithStreaming.put( + BuiltinMeasureConstants.APPLICATION_LATENCIES, applicationLatency); + } + + public void recordFirstResponseLatency(long firstResponseLatency) { + operationLevelNoStreaming.put( + BuiltinMeasureConstants.FIRST_RESPONSE_LATENCIES, firstResponseLatency); + } + + public void recordGfeLatencies(long serverLatency) { + attemptLevelWithStreaming.put(BuiltinMeasureConstants.SERVER_LATENCIES, serverLatency); + } + + public void recordGfeMissingHeaders(long connectivityErrors) { + attemptLevelNoStreaming.put( + BuiltinMeasureConstants.CONNECTIVITY_ERROR_COUNT, connectivityErrors); + } + + public void recordBatchRequestThrottled( + long throttledTimeMs, String tableId, String zone, String cluster) { + MeasureMap measures = + statsRecorder + .newMeasureMap() + .put(BuiltinMeasureConstants.THROTTLING_LATENCIES, throttledTimeMs); + measures.record(newTagContextBuilder(tableId, zone, cluster).build()); + } + + private TagContextBuilder newTagContextBuilder(String tableId, String zone, String cluster) { + TagContextBuilder tagContextBuilder = + tagger + .toBuilder(parentContext) + .putLocal(BuiltinMeasureConstants.CLIENT_NAME, TagValue.create("bigtable-java")) + .putLocal(BuiltinMeasureConstants.METHOD, TagValue.create(spanName.toString())) + .putLocal(BuiltinMeasureConstants.TABLE, TagValue.create(tableId)) + .putLocal(BuiltinMeasureConstants.ZONE, TagValue.create(zone)) + .putLocal(BuiltinMeasureConstants.CLUSTER, TagValue.create(cluster)); + for (Map.Entry entry : statsAttributes.entrySet()) { + tagContextBuilder.putLocal(TagKey.create(entry.getKey()), TagValue.create(entry.getValue())); + } + return tagContextBuilder; + } +} diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViewConstants.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViewConstants.java new file mode 100644 index 0000000000..c1ef763aeb --- /dev/null +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViewConstants.java @@ -0,0 +1,201 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.stats; + +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.APPLICATION_LATENCIES; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.APP_PROFILE; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.ATTEMPT_LATENCIES; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.CLIENT_NAME; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.CLUSTER; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.CONNECTIVITY_ERROR_COUNT; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.FIRST_RESPONSE_LATENCIES; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.INSTANCE_ID; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.METHOD; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.OPERATION_LATENCIES; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.PROJECT_ID; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.RETRY_COUNT; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.SERVER_LATENCIES; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.STATUS; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.STREAMING; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.TABLE; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.THROTTLING_LATENCIES; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.ZONE; +import static io.opencensus.stats.Aggregation.Distribution; + +import com.google.api.core.InternalApi; +import com.google.common.collect.ImmutableList; +import io.opencensus.stats.Aggregation; +import io.opencensus.stats.BucketBoundaries; +import io.opencensus.stats.View; + +@InternalApi("For internal use only") +public class BuiltinViewConstants { + private static final Aggregation AGGREGATION_WITH_MILLIS_HISTOGRAM = + Distribution.create( + BucketBoundaries.create( + ImmutableList.of( + 0.0, 0.01, 0.05, 0.1, 0.3, 0.6, 0.8, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 8.0, 10.0, + 13.0, 16.0, 20.0, 25.0, 30.0, 40.0, 50.0, 65.0, 80.0, 100.0, 130.0, 160.0, 200.0, + 250.0, 300.0, 400.0, 500.0, 650.0, 800.0, 1000.0, 2000.0, 5000.0, 10000.0, + 20000.0, 50000.0, 100000.0))); + + private static final Aggregation AGGREGATION_RETRY_COUNT = + Distribution.create( + BucketBoundaries.create( + ImmutableList.of( + 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 15.0, 20.0, 30.0, 40.0, 50.0, + 100.0))); + + private static final Aggregation AGGREGATION_ERROR_COUNT = + Distribution.create( + BucketBoundaries.create( + ImmutableList.of( + 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 15.0, 20.0, 30.0, 40.0, 50.0, + 100.0, 150.0, 200.0, 250.0, 300.0, 400.0, 500.0, 650.0, 800.0, 1000.0, 2000.0, + 5000.0, 10000.0, 20000.0, 50000.0, 75000.0, 100000.0))); + + // TODO: move project id, instance id to monitored resource + public static final View OPERATION_LATENCIES_VIEW = + View.create( + View.Name.create("bigtable.googleapis.com/internal/client/operation_latencies"), + "Total time until final operation success or failure, including retries and backoff.", + OPERATION_LATENCIES, + AGGREGATION_WITH_MILLIS_HISTOGRAM, + ImmutableList.of( + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE, + METHOD, + STREAMING, + STATUS, + CLIENT_NAME, + CLUSTER, + ZONE, + TABLE)); + + public static final View ATTEMPT_LATENCIES_VIEW = + View.create( + View.Name.create("bigtable.googleapis.com/internal/client/attempt_latencies"), + "Client observed latency per RPC attempt.", + ATTEMPT_LATENCIES, + AGGREGATION_WITH_MILLIS_HISTOGRAM, + ImmutableList.of( + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE, + METHOD, + STREAMING, + STATUS, + CLIENT_NAME, + CLUSTER, + ZONE, + TABLE)); + + public static final View RETRY_COUNT_VIEW = + View.create( + View.Name.create("bigtable.googleapis.com/internal/client/retry_count"), + "The number of additional RPCs sent after the initial attempt.", + RETRY_COUNT, + AGGREGATION_RETRY_COUNT, + ImmutableList.of( + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE, + METHOD, + STATUS, + CLIENT_NAME, + CLUSTER, + ZONE, + TABLE)); + + public static final View FIRST_RESPONSE_LATENCIES_VIEW = + View.create( + View.Name.create("bigtable.googleapis.com/internal/client/first_response_latencies"), + "Latency from operation start until the response headers were received. The publishing of the measurement will be delayed until the attempt response has been received.", + FIRST_RESPONSE_LATENCIES, + AGGREGATION_WITH_MILLIS_HISTOGRAM, + ImmutableList.of( + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE, + METHOD, + STATUS, + CLIENT_NAME, + CLUSTER, + ZONE, + TABLE)); + + public static final View SERVER_LATENCIES_VIEW = + View.create( + View.Name.create("bigtable.googleapis.com/internal/client/server_latencies"), + "The latency measured from the moment that the RPC entered the Google data center until the RPC was completed.", + SERVER_LATENCIES, + AGGREGATION_WITH_MILLIS_HISTOGRAM, + ImmutableList.of( + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE, + METHOD, + STATUS, + STREAMING, + CLIENT_NAME, + CLUSTER, + ZONE, + TABLE)); + + public static final View CONNECTIVITY_ERROR_COUNT_VIEW = + View.create( + View.Name.create("bigtable.googleapis.com/internal/client/connectivity_error_count"), + "Number of requests that failed to reach the Google datacenter. (Requests without google response headers).", + CONNECTIVITY_ERROR_COUNT, + AGGREGATION_ERROR_COUNT, + ImmutableList.of( + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE, + METHOD, + STATUS, + CLIENT_NAME, + CLUSTER, + ZONE, + TABLE)); + + public static final View APPLICATION_LATENCIES_VIEW = + View.create( + View.Name.create("bigtable.googleapis.com/internal/client/application_latencies"), + "The latency of the client application consuming available response data.", + APPLICATION_LATENCIES, + AGGREGATION_WITH_MILLIS_HISTOGRAM, + ImmutableList.of( + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE, + METHOD, + STREAMING, + CLIENT_NAME, + CLUSTER, + ZONE, + TABLE)); + + public static final View THROTTLING_LATENCIES_VIEW = + View.create( + View.Name.create("bigtable.googleapis.com/internal/client/throttling_latencies"), + "The artificial latency introduced by the client to limit the number of outstanding requests. The publishing of the measurement will be delayed until the attempt trailers have been received.", + THROTTLING_LATENCIES, + AGGREGATION_WITH_MILLIS_HISTOGRAM, + ImmutableList.of( + PROJECT_ID, INSTANCE_ID, APP_PROFILE, METHOD, CLIENT_NAME, CLUSTER, ZONE, TABLE)); +} diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViews.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViews.java new file mode 100644 index 0000000000..b37fa25715 --- /dev/null +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViews.java @@ -0,0 +1,48 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.stats; + +import com.google.api.core.InternalApi; +import com.google.common.collect.ImmutableSet; +import io.opencensus.stats.View; +import io.opencensus.stats.ViewManager; + +@InternalApi("For internal use only") +public class BuiltinViews { + private static StatsWrapper statsWrapper; + + public BuiltinViews(StatsWrapper wrapper) { + this.statsWrapper = wrapper; + } + + private static final ImmutableSet BIGTABLE_BUILTIN_VIEWS = + ImmutableSet.of( + BuiltinViewConstants.OPERATION_LATENCIES_VIEW, + BuiltinViewConstants.ATTEMPT_LATENCIES_VIEW, + BuiltinViewConstants.RETRY_COUNT_VIEW, + BuiltinViewConstants.FIRST_RESPONSE_LATENCIES_VIEW, + BuiltinViewConstants.SERVER_LATENCIES_VIEW, + BuiltinViewConstants.CONNECTIVITY_ERROR_COUNT_VIEW, + BuiltinViewConstants.APPLICATION_LATENCIES_VIEW, + BuiltinViewConstants.THROTTLING_LATENCIES_VIEW); + + public void registerBigtableBuiltinViews() { + ViewManager viewManager = statsWrapper.getViewManager(); + for (View view : BIGTABLE_BUILTIN_VIEWS) { + viewManager.registerView(view); + } + } +} diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/MavenPlaceholderShaded.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/MavenPlaceholderShaded.java deleted file mode 100644 index 44b0633b5b..0000000000 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/MavenPlaceholderShaded.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.cloud.bigtable.stats; - -public final class MavenPlaceholderShaded { - /** - * This class is here to force generation of source javadoc jars so that the maven release process - * doesn't complain. The shading plugin generated a shaded jar of bigtable-stats, but it doesn't - * generate javadoc or source files; this class is here as a hack and better methods should be - * employed. - */ - private MavenPlaceholderShaded() {} -} diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/StatsWrapper.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/StatsWrapper.java new file mode 100644 index 0000000000..160e16d8eb --- /dev/null +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/StatsWrapper.java @@ -0,0 +1,133 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.stats; + +import com.google.api.core.InternalApi; +import com.google.common.collect.ImmutableMap; +import io.opencensus.impl.stats.StatsComponentImpl; +import io.opencensus.stats.AggregationData; +import io.opencensus.stats.Stats; +import io.opencensus.stats.StatsComponent; +import io.opencensus.stats.StatsRecorder; +import io.opencensus.stats.View; +import io.opencensus.stats.ViewData; +import io.opencensus.stats.ViewManager; +import io.opencensus.tags.TagKey; +import io.opencensus.tags.TagValue; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** Wrapper class for accessing opencensus * */ +@InternalApi("For internal use only") +public class StatsWrapper { + + private final StatsComponent component; + private final boolean useLocalRecorder; + + public StatsWrapper(boolean useLocalRecorder) { + this.component = new StatsComponentImpl(); + this.useLocalRecorder = useLocalRecorder; + } + + StatsRecorder getStatsRecorder() { + if (!useLocalRecorder) { + return Stats.getStatsRecorder(); + } else { + return component.getStatsRecorder(); + } + } + + ViewManager getViewManager() { + if (!useLocalRecorder) { + return Stats.getViewManager(); + } else { + return component.getViewManager(); + } + } + + long getAggregationValueAsLong( + View view, + ImmutableMap tags, + String projectId, + String instanceId, + String appProfileId) { + ViewData viewData = getViewManager().getView(view.getName()); + Map, AggregationData> aggregationMap = + Objects.requireNonNull(viewData).getAggregationMap(); + + List tagValues = new ArrayList<>(); + + for (TagKey column : view.getColumns()) { + if (BuiltinMeasureConstants.PROJECT_ID == column) { + tagValues.add(TagValue.create(projectId)); + } else if (BuiltinMeasureConstants.INSTANCE_ID == column) { + tagValues.add(TagValue.create(instanceId)); + } else if (BuiltinMeasureConstants.APP_PROFILE == column) { + tagValues.add(TagValue.create(appProfileId)); + } else { + tagValues.add(TagValue.create(tags.get(column))); + } + } + + AggregationData aggregationData = aggregationMap.get(tagValues); + + return aggregationData.match( + new io.opencensus.common.Function() { + @Override + public Long apply(AggregationData.SumDataDouble arg) { + return (long) arg.getSum(); + } + }, + new io.opencensus.common.Function() { + @Override + public Long apply(AggregationData.SumDataLong arg) { + return arg.getSum(); + } + }, + new io.opencensus.common.Function() { + @Override + public Long apply(AggregationData.CountData arg) { + return arg.getCount(); + } + }, + new io.opencensus.common.Function() { + @Override + public Long apply(AggregationData.DistributionData arg) { + return (long) arg.getMean(); + } + }, + new io.opencensus.common.Function() { + @Override + public Long apply(AggregationData.LastValueDataDouble arg) { + return (long) arg.getLastValue(); + } + }, + new io.opencensus.common.Function() { + @Override + public Long apply(AggregationData.LastValueDataLong arg) { + return arg.getLastValue(); + } + }, + new io.opencensus.common.Function() { + @Override + public Long apply(AggregationData arg) { + throw new UnsupportedOperationException(); + } + }); + } +} diff --git a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorderTest.java b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorderTest.java new file mode 100644 index 0000000000..8f0c2688bf --- /dev/null +++ b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorderTest.java @@ -0,0 +1,433 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.stats; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.api.gax.tracing.ApiTracerFactory; +import com.google.api.gax.tracing.SpanName; +import com.google.common.collect.ImmutableMap; +import io.grpc.Status; +import org.junit.Before; +import org.junit.Test; + +public class BuiltinMetricsRecorderTest { + + private final String PROJECT_ID = "fake-project"; + private final String INSTANCE_ID = "fake-instance"; + private final String APP_PROFILE_ID = "fake-app-profile"; + + private final String TABLE_ID = "fake-table-id"; + private final String ZONE = "fake-zone"; + private final String CLUSTER = "fake-cluster"; + + private StatsWrapper wrapper; + + @Before + public void setup() { + this.wrapper = new StatsWrapper(true); + BuiltinViews views = new BuiltinViews(wrapper); + views.registerBigtableBuiltinViews(); + } + + @Test + public void testStreamingOperation() throws InterruptedException { + BuiltinMetricsRecorder tracer = + new BuiltinMetricsRecorder( + ApiTracerFactory.OperationType.ServerStreaming, + SpanName.of("Bigtable", "ReadRows"), + ImmutableMap.of( + BuiltinMeasureConstants.PROJECT_ID.getName(), PROJECT_ID, + BuiltinMeasureConstants.INSTANCE_ID.getName(), INSTANCE_ID, + BuiltinMeasureConstants.APP_PROFILE.getName(), APP_PROFILE_ID), + wrapper); + + long operationLatency = 1234; + int attemptCount = 2; + long attemptLatency = 56; + long serverLatency = 78; + long applicationLatency = 901; + long connectivityErrorCount = 15; + long throttlingLatency = 50; + long firstResponseLatency = 90; + + tracer.recordOperationLatencies(operationLatency); + tracer.recordRetryCount(attemptCount); + tracer.recordAttemptLatency(attemptLatency); + tracer.recordApplicationLatency(applicationLatency); + tracer.recordGfeLatencies(serverLatency); + tracer.recordGfeMissingHeaders(connectivityErrorCount); + tracer.recordFirstResponseLatency(firstResponseLatency); + tracer.recordBatchRequestThrottled(throttlingLatency, TABLE_ID, ZONE, CLUSTER); + + tracer.recordAttemptLevelWithoutStreaming( + Status.UNAVAILABLE.toString(), TABLE_ID, ZONE, CLUSTER); + tracer.recordAttemptLevelWithStreaming(Status.ABORTED.toString(), TABLE_ID, ZONE, CLUSTER); + tracer.recordOperationLevelWithoutStreaming("OK", TABLE_ID, ZONE, CLUSTER); + tracer.recordOperationLevelWithStreaming("OK", TABLE_ID, ZONE, CLUSTER); + + Thread.sleep(100); + + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.OPERATION_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, "Bigtable.ReadRows", + BuiltinMeasureConstants.STATUS, "OK", + BuiltinMeasureConstants.TABLE, TABLE_ID, + BuiltinMeasureConstants.ZONE, ZONE, + BuiltinMeasureConstants.CLUSTER, CLUSTER, + BuiltinMeasureConstants.CLIENT_NAME, "bigtable-java", + BuiltinMeasureConstants.STREAMING, "true"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(operationLatency); + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.ATTEMPT_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.ReadRows", + BuiltinMeasureConstants.STATUS, + Status.ABORTED.toString(), + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER, + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java", + BuiltinMeasureConstants.STREAMING, + "true"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(attemptLatency); + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.RETRY_COUNT_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.ReadRows", + BuiltinMeasureConstants.STATUS, + "OK", + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER, + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(attemptCount); + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.SERVER_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.ReadRows", + BuiltinMeasureConstants.STATUS, + Status.ABORTED.toString(), + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java", + BuiltinMeasureConstants.STREAMING, + "true", + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(serverLatency); + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.APPLICATION_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.ReadRows", + BuiltinMeasureConstants.STATUS, + "OK", + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER, + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java", + BuiltinMeasureConstants.STREAMING, + "true"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(applicationLatency); + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.CONNECTIVITY_ERROR_COUNT_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.ReadRows", + BuiltinMeasureConstants.STATUS, + Status.UNAVAILABLE.toString(), + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java", + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(connectivityErrorCount); + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.THROTTLING_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, "Bigtable.ReadRows", + BuiltinMeasureConstants.TABLE, TABLE_ID, + BuiltinMeasureConstants.ZONE, ZONE, + BuiltinMeasureConstants.CLUSTER, CLUSTER, + BuiltinMeasureConstants.CLIENT_NAME, "bigtable-java"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(throttlingLatency); + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.FIRST_RESPONSE_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.ReadRows", + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER, + BuiltinMeasureConstants.STATUS, + "OK", + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(firstResponseLatency); + } + + @Test + public void testUnaryOperations() throws InterruptedException { + BuiltinMetricsRecorder tracer = + new BuiltinMetricsRecorder( + ApiTracerFactory.OperationType.ServerStreaming, + SpanName.of("Bigtable", "MutateRow"), + ImmutableMap.of( + BuiltinMeasureConstants.PROJECT_ID.getName(), PROJECT_ID, + BuiltinMeasureConstants.INSTANCE_ID.getName(), INSTANCE_ID, + BuiltinMeasureConstants.APP_PROFILE.getName(), APP_PROFILE_ID), + wrapper); + + long operationLatency = 1234; + int attemptCount = 2; + long attemptLatency = 56; + long serverLatency = 78; + long applicationLatency = 901; + long connectivityErrorCount = 15; + long throttlingLatency = 50; + long firstResponseLatency = 90; + + tracer.recordOperationLatencies(operationLatency); + tracer.recordRetryCount(attemptCount); + tracer.recordAttemptLatency(attemptLatency); + tracer.recordApplicationLatency(applicationLatency); + tracer.recordGfeLatencies(serverLatency); + tracer.recordGfeMissingHeaders(connectivityErrorCount); + tracer.recordFirstResponseLatency(firstResponseLatency); + tracer.recordBatchRequestThrottled(throttlingLatency, TABLE_ID, ZONE, CLUSTER); + + tracer.recordOperationLevelWithStreaming("OK", TABLE_ID, ZONE, CLUSTER); + tracer.recordOperationLevelWithoutStreaming("OK", TABLE_ID, ZONE, CLUSTER); + tracer.recordAttemptLevelWithoutStreaming( + Status.UNAVAILABLE.toString(), TABLE_ID, ZONE, CLUSTER); + tracer.recordAttemptLevelWithStreaming(Status.ABORTED.toString(), TABLE_ID, ZONE, CLUSTER); + + Thread.sleep(100); + + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.OPERATION_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, "Bigtable.MutateRow", + BuiltinMeasureConstants.STATUS, "OK", + BuiltinMeasureConstants.TABLE, TABLE_ID, + BuiltinMeasureConstants.ZONE, ZONE, + BuiltinMeasureConstants.CLUSTER, CLUSTER, + BuiltinMeasureConstants.CLIENT_NAME, "bigtable-java", + BuiltinMeasureConstants.STREAMING, "false"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(operationLatency); + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.ATTEMPT_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.MutateRow", + BuiltinMeasureConstants.STATUS, + Status.ABORTED.toString(), + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER, + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java", + BuiltinMeasureConstants.STREAMING, + "false"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(attemptLatency); + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.RETRY_COUNT_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.MutateRow", + BuiltinMeasureConstants.STATUS, + "OK", + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER, + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(attemptCount); + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.SERVER_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.MutateRow", + BuiltinMeasureConstants.STATUS, + Status.ABORTED.toString(), + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java", + BuiltinMeasureConstants.STREAMING, + "false", + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(serverLatency); + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.APPLICATION_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.MutateRow", + BuiltinMeasureConstants.STATUS, + "OK", + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER, + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java", + BuiltinMeasureConstants.STREAMING, + "false"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(applicationLatency); + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.CONNECTIVITY_ERROR_COUNT_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.MutateRow", + BuiltinMeasureConstants.STATUS, + Status.UNAVAILABLE.toString(), + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java", + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(connectivityErrorCount); + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.THROTTLING_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, "Bigtable.MutateRow", + BuiltinMeasureConstants.TABLE, TABLE_ID, + BuiltinMeasureConstants.ZONE, ZONE, + BuiltinMeasureConstants.CLUSTER, CLUSTER, + BuiltinMeasureConstants.CLIENT_NAME, "bigtable-java"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(throttlingLatency); + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.FIRST_RESPONSE_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.MutateRow", + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER, + BuiltinMeasureConstants.STATUS, + "OK", + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(firstResponseLatency); + } +}