diff --git a/google-cloud-bigtable-bom/pom.xml b/google-cloud-bigtable-bom/pom.xml index 6868f2c93c..cc7eac753d 100644 --- a/google-cloud-bigtable-bom/pom.xml +++ b/google-cloud-bigtable-bom/pom.xml @@ -94,6 +94,11 @@ proto-google-cloud-bigtable-v2 2.8.1-SNAPSHOT + + com.google.cloud + google-cloud-bigtable-stats + 2.8.1-SNAPSHOT + diff --git a/google-cloud-bigtable-stats/pom.xml b/google-cloud-bigtable-stats/pom.xml index 6f9d0a49e3..a62e898fe3 100644 --- a/google-cloud-bigtable-stats/pom.xml +++ b/google-cloud-bigtable-stats/pom.xml @@ -29,17 +29,38 @@ + + com.google.api + gax + + + com.google.api + api-common + + io.opencensus opencensus-api + + com.google.guava + guava + + io.opencensus opencensus-impl + test - io.opencensus - opencensus-exporter-stats-stackdriver + com.google.truth + truth + test + + + junit + junit + test @@ -48,6 +69,7 @@ org.apache.maven.plugins maven-shade-plugin + 3.2.4 package @@ -78,11 +100,6 @@ org.apache.maven.plugins maven-dependency-plugin 3.3.0 - - - * - - diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMeasureConstants.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMeasureConstants.java new file mode 100644 index 0000000000..06ca674ffc --- /dev/null +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMeasureConstants.java @@ -0,0 +1,92 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.stats; + +import static io.opencensus.stats.Measure.MeasureLong; + +import io.opencensus.tags.TagKey; + +/** Built-in metrics that will be readable under bigtable.googleapis.com/client namespace */ +class BuiltinMeasureConstants { + // Monitored resource TagKeys + static final TagKey PROJECT_ID = TagKey.create("project_id"); + static final TagKey INSTANCE_ID = TagKey.create("instance_id"); + static final TagKey CLUSTER = TagKey.create("cluster"); + static final TagKey TABLE = TagKey.create("table"); + static final TagKey ZONE = TagKey.create("zone"); + // Placeholder TagKey to be used in Stackdriver exporter + static final TagKey CLIENT_ID = TagKey.create("client_id"); + + // Metrics TagKeys + static final TagKey APP_PROFILE = TagKey.create("app_profile"); + static final TagKey METHOD = TagKey.create("method"); + static final TagKey STREAMING = TagKey.create("streaming"); + static final TagKey STATUS = TagKey.create("status"); + static final TagKey CLIENT_NAME = TagKey.create("client_name"); + + // Units + private static final String COUNT = "1"; + private static final String MILLISECOND = "ms"; + + // Measurements + static final MeasureLong OPERATION_LATENCIES = + MeasureLong.create( + "bigtable.googleapis.com/internal/client/operation_latencies", + "Total time until final operation success or failure, including retries and backoff.", + MILLISECOND); + + static final MeasureLong ATTEMPT_LATENCIES = + MeasureLong.create( + "bigtable.googleapis.com/internal/client/attempt_latencies", + "Client observed latency per RPC attempt.", + MILLISECOND); + + static final MeasureLong RETRY_COUNT = + MeasureLong.create( + "bigtable.googleapis.com/internal/client/retry_count", + "The number of additional RPCs sent after the initial attempt.", + COUNT); + + static final MeasureLong FIRST_RESPONSE_LATENCIES = + MeasureLong.create( + "bigtable.googleapis.com/internal/client/first_response_latencies", + "Latency from operation start until the response headers were received. The publishing of the measurement will be delayed until the attempt response has been received.", + MILLISECOND); + + static final MeasureLong SERVER_LATENCIES = + MeasureLong.create( + "bigtable.googleapis.com/internal/client/server_latencies", + "The latency measured from the moment that the RPC entered the Google data center until the RPC was completed.", + MILLISECOND); + + static final MeasureLong CONNECTIVITY_ERROR_COUNT = + MeasureLong.create( + "bigtable.googleapis.com/internal/client/connectivity_error_count", + "Number of requests that failed to reach the Google datacenter. (Requests without google response headers).", + COUNT); + + static final MeasureLong APPLICATION_LATENCIES = + MeasureLong.create( + "bigtable.googleapis.com/internal/client/application_latencies", + "The latency of the client application consuming available response data.", + MILLISECOND); + + static final MeasureLong THROTTLING_LATENCIES = + MeasureLong.create( + "bigtable.googleapis.com/internal/client/throttling_latencies", + "The artificial latency introduced by the client to limit the number of outstanding requests. The publishing of the measurement will be delayed until the attempt trailers have been received.", + MILLISECOND); +} diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViewConstants.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViewConstants.java new file mode 100644 index 0000000000..beceeeab83 --- /dev/null +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViewConstants.java @@ -0,0 +1,194 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.stats; + +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.APPLICATION_LATENCIES; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.APP_PROFILE; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.ATTEMPT_LATENCIES; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.CLIENT_NAME; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.CLUSTER; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.CONNECTIVITY_ERROR_COUNT; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.FIRST_RESPONSE_LATENCIES; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.INSTANCE_ID; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.METHOD; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.OPERATION_LATENCIES; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.PROJECT_ID; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.RETRY_COUNT; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.SERVER_LATENCIES; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.STATUS; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.STREAMING; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.TABLE; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.THROTTLING_LATENCIES; +import static com.google.cloud.bigtable.stats.BuiltinMeasureConstants.ZONE; +import static io.opencensus.stats.Aggregation.Distribution; +import static io.opencensus.stats.Aggregation.Sum; + +import com.google.common.collect.ImmutableList; +import io.opencensus.stats.Aggregation; +import io.opencensus.stats.BucketBoundaries; +import io.opencensus.stats.View; + +/** Create built-in metrics views under bigtable.googleapis.com/internal/client namespace */ +class BuiltinViewConstants { + private static final Aggregation AGGREGATION_WITH_MILLIS_HISTOGRAM = + Distribution.create( + BucketBoundaries.create( + ImmutableList.of( + 0.0, 0.01, 0.05, 0.1, 0.3, 0.6, 0.8, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 8.0, 10.0, + 13.0, 16.0, 20.0, 25.0, 30.0, 40.0, 50.0, 65.0, 80.0, 100.0, 130.0, 160.0, 200.0, + 250.0, 300.0, 400.0, 500.0, 650.0, 800.0, 1000.0, 2000.0, 5000.0, 10000.0, + 20000.0, 50000.0, 100000.0))); + + private static final Aggregation AGGREGATION_RETRY_COUNT = + Distribution.create( + BucketBoundaries.create( + ImmutableList.of( + 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 15.0, 20.0, 30.0, 40.0, 50.0, + 100.0))); + + private static final Aggregation AGGREGATION_ERROR_COUNT = Sum.create(); + + static final View OPERATION_LATENCIES_VIEW = + View.create( + View.Name.create("bigtable.googleapis.com/internal/client/operation_latencies"), + "Total time until final operation success or failure, including retries and backoff.", + OPERATION_LATENCIES, + AGGREGATION_WITH_MILLIS_HISTOGRAM, + ImmutableList.of( + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE, + METHOD, + STREAMING, + STATUS, + CLIENT_NAME, + CLUSTER, + ZONE, + TABLE)); + + static final View ATTEMPT_LATENCIES_VIEW = + View.create( + View.Name.create("bigtable.googleapis.com/internal/client/attempt_latencies"), + "Client observed latency per RPC attempt.", + ATTEMPT_LATENCIES, + AGGREGATION_WITH_MILLIS_HISTOGRAM, + ImmutableList.of( + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE, + METHOD, + STREAMING, + STATUS, + CLIENT_NAME, + CLUSTER, + ZONE, + TABLE)); + + static final View RETRY_COUNT_VIEW = + View.create( + View.Name.create("bigtable.googleapis.com/internal/client/retry_count"), + "The number of additional RPCs sent after the initial attempt.", + RETRY_COUNT, + AGGREGATION_RETRY_COUNT, + ImmutableList.of( + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE, + METHOD, + STATUS, + CLIENT_NAME, + CLUSTER, + ZONE, + TABLE)); + + static final View FIRST_RESPONSE_LATENCIES_VIEW = + View.create( + View.Name.create("bigtable.googleapis.com/internal/client/first_response_latencies"), + "Latency from operation start until the response headers were received. The publishing of the measurement will be delayed until the attempt response has been received.", + FIRST_RESPONSE_LATENCIES, + AGGREGATION_WITH_MILLIS_HISTOGRAM, + ImmutableList.of( + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE, + METHOD, + STATUS, + CLIENT_NAME, + CLUSTER, + ZONE, + TABLE)); + + static final View SERVER_LATENCIES_VIEW = + View.create( + View.Name.create("bigtable.googleapis.com/internal/client/server_latencies"), + "The latency measured from the moment that the RPC entered the Google data center until the RPC was completed.", + SERVER_LATENCIES, + AGGREGATION_WITH_MILLIS_HISTOGRAM, + ImmutableList.of( + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE, + METHOD, + STATUS, + STREAMING, + CLIENT_NAME, + CLUSTER, + ZONE, + TABLE)); + + static final View CONNECTIVITY_ERROR_COUNT_VIEW = + View.create( + View.Name.create("bigtable.googleapis.com/internal/client/connectivity_error_count"), + "Number of requests that failed to reach the Google datacenter. (Requests without google response headers).", + CONNECTIVITY_ERROR_COUNT, + AGGREGATION_ERROR_COUNT, + ImmutableList.of( + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE, + METHOD, + STATUS, + CLIENT_NAME, + CLUSTER, + ZONE, + TABLE)); + + static final View APPLICATION_LATENCIES_VIEW = + View.create( + View.Name.create("bigtable.googleapis.com/internal/client/application_latencies"), + "The latency of the client application consuming available response data.", + APPLICATION_LATENCIES, + AGGREGATION_WITH_MILLIS_HISTOGRAM, + ImmutableList.of( + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE, + METHOD, + STREAMING, + CLIENT_NAME, + CLUSTER, + ZONE, + TABLE)); + + static final View THROTTLING_LATENCIES_VIEW = + View.create( + View.Name.create("bigtable.googleapis.com/internal/client/throttling_latencies"), + "The artificial latency introduced by the client to limit the number of outstanding requests. The publishing of the measurement will be delayed until the attempt trailers have been received.", + THROTTLING_LATENCIES, + AGGREGATION_WITH_MILLIS_HISTOGRAM, + ImmutableList.of( + PROJECT_ID, INSTANCE_ID, APP_PROFILE, METHOD, CLIENT_NAME, CLUSTER, ZONE, TABLE)); +} diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViews.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViews.java new file mode 100644 index 0000000000..b5cf180b5b --- /dev/null +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViews.java @@ -0,0 +1,53 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.stats; + +import com.google.api.core.InternalApi; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableSet; +import io.opencensus.stats.Stats; +import io.opencensus.stats.View; +import io.opencensus.stats.ViewManager; + +/** For registering built-in metric views */ +@InternalApi("For internal use only") +public class BuiltinViews { + @VisibleForTesting + static final ImmutableSet BIGTABLE_BUILTIN_VIEWS = + ImmutableSet.of( + BuiltinViewConstants.OPERATION_LATENCIES_VIEW, + BuiltinViewConstants.ATTEMPT_LATENCIES_VIEW, + BuiltinViewConstants.RETRY_COUNT_VIEW, + BuiltinViewConstants.FIRST_RESPONSE_LATENCIES_VIEW, + BuiltinViewConstants.SERVER_LATENCIES_VIEW, + BuiltinViewConstants.CONNECTIVITY_ERROR_COUNT_VIEW, + BuiltinViewConstants.APPLICATION_LATENCIES_VIEW, + BuiltinViewConstants.THROTTLING_LATENCIES_VIEW); + + @VisibleForTesting + void registerPrivateViews(ViewManager viewManager) { + for (View view : BIGTABLE_BUILTIN_VIEWS) { + viewManager.registerView(view); + } + } + + public void registerBigtableBuiltinViews() { + ViewManager viewManager = Stats.getViewManager(); + for (View view : BIGTABLE_BUILTIN_VIEWS) { + viewManager.registerView(view); + } + } +} diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/MavenPlaceholderShaded.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/MavenPlaceholderShaded.java deleted file mode 100644 index 44b0633b5b..0000000000 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/MavenPlaceholderShaded.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.cloud.bigtable.stats; - -public final class MavenPlaceholderShaded { - /** - * This class is here to force generation of source javadoc jars so that the maven release process - * doesn't complain. The shading plugin generated a shaded jar of bigtable-stats, but it doesn't - * generate javadoc or source files; this class is here as a hack and better methods should be - * employed. - */ - private MavenPlaceholderShaded() {} -} diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/StatsRecorderWrapper.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/StatsRecorderWrapper.java new file mode 100644 index 0000000000..ff3568c5f4 --- /dev/null +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/StatsRecorderWrapper.java @@ -0,0 +1,118 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.stats; + +import com.google.api.core.InternalApi; +import com.google.api.gax.tracing.ApiTracerFactory.OperationType; +import com.google.api.gax.tracing.SpanName; +import io.opencensus.stats.MeasureMap; +import io.opencensus.stats.StatsRecorder; +import io.opencensus.tags.TagContext; +import io.opencensus.tags.TagContextBuilder; +import io.opencensus.tags.TagKey; +import io.opencensus.tags.TagValue; +import io.opencensus.tags.Tagger; +import io.opencensus.tags.Tags; +import java.util.Map; + +/** A wrapper to record built-in metrics */ +@InternalApi("For internal use only") +public class StatsRecorderWrapper { + + private final OperationType operationType; + + private final Tagger tagger; + private final StatsRecorder statsRecorder; + private final TagContext parentContext; + private final SpanName spanName; + private final Map statsAttributes; + + private MeasureMap measureMap; + + public StatsRecorderWrapper( + OperationType operationType, + SpanName spanName, + Map statsAttributes, + StatsRecorder statsRecorder) { + this.operationType = operationType; + this.tagger = Tags.getTagger(); + this.statsRecorder = statsRecorder; + this.spanName = spanName; + this.parentContext = tagger.getCurrentTagContext(); + this.statsAttributes = statsAttributes; + + this.measureMap = statsRecorder.newMeasureMap(); + } + + public void record(String status, String tableId, String zone, String cluster) { + TagContextBuilder tagCtx = + newTagContextBuilder(tableId, zone, cluster) + .putLocal(BuiltinMeasureConstants.STATUS, TagValue.create(status)); + + boolean isStreaming = operationType == OperationType.ServerStreaming; + tagCtx.putLocal( + BuiltinMeasureConstants.STREAMING, TagValue.create(Boolean.toString(isStreaming))); + + measureMap.record(tagCtx.build()); + } + + public void putOperationLatencies(long operationLatency) { + measureMap.put(BuiltinMeasureConstants.OPERATION_LATENCIES, operationLatency); + } + + public void putAttemptLatencies(long attemptLatency) { + measureMap.put(BuiltinMeasureConstants.ATTEMPT_LATENCIES, attemptLatency); + } + + public void putRetryCount(int attemptCount) { + measureMap.put(BuiltinMeasureConstants.RETRY_COUNT, attemptCount); + } + + public void putApplicationLatencies(long applicationLatency) { + measureMap.put(BuiltinMeasureConstants.APPLICATION_LATENCIES, applicationLatency); + } + + public void putFirstResponseLatencies(long firstResponseLatency) { + measureMap.put(BuiltinMeasureConstants.FIRST_RESPONSE_LATENCIES, firstResponseLatency); + } + + public void putGfeLatencies(long serverLatency) { + measureMap.put(BuiltinMeasureConstants.SERVER_LATENCIES, serverLatency); + } + + public void putGfeMissingHeaders(long connectivityErrors) { + measureMap.put(BuiltinMeasureConstants.CONNECTIVITY_ERROR_COUNT, connectivityErrors); + } + + public void putBatchRequestThrottled(long throttledTimeMs) { + measureMap.put(BuiltinMeasureConstants.THROTTLING_LATENCIES, throttledTimeMs); + } + + private TagContextBuilder newTagContextBuilder(String tableId, String zone, String cluster) { + TagContextBuilder tagContextBuilder = + tagger + .toBuilder(parentContext) + .putLocal(BuiltinMeasureConstants.CLIENT_NAME, TagValue.create("bigtable-java")) + .putLocal(BuiltinMeasureConstants.METHOD, TagValue.create(spanName.toString())) + .putLocal(BuiltinMeasureConstants.TABLE, TagValue.create(tableId)) + .putLocal(BuiltinMeasureConstants.ZONE, TagValue.create(zone)) + .putLocal(BuiltinMeasureConstants.CLUSTER, TagValue.create(cluster)); + for (Map.Entry entry : statsAttributes.entrySet()) { + tagContextBuilder.putLocal(TagKey.create(entry.getKey()), TagValue.create(entry.getValue())); + } + return tagContextBuilder; + } +} diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/StatsWrapper.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/StatsWrapper.java new file mode 100644 index 0000000000..6b7bd6df9b --- /dev/null +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/StatsWrapper.java @@ -0,0 +1,37 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.stats; + +import static com.google.api.gax.tracing.ApiTracerFactory.OperationType; + +import com.google.api.core.InternalApi; +import com.google.api.gax.tracing.SpanName; +import io.opencensus.stats.Stats; +import java.util.Map; + +/** + * Wrapper class for accessing opencensus. We use a shaded version of opencensus to avoid polluting + * the global opencensus namespace. And this provides a facade that will not be relocated. + */ +@InternalApi("For internal use only") +public class StatsWrapper { + + public static StatsRecorderWrapper createRecorder( + OperationType operationType, SpanName spanName, Map statsAttributes) { + return new StatsRecorderWrapper( + operationType, spanName, statsAttributes, Stats.getStatsRecorder()); + } +} diff --git a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BuiltinViewConstantsTest.java b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BuiltinViewConstantsTest.java new file mode 100644 index 0000000000..a7d20f6da1 --- /dev/null +++ b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BuiltinViewConstantsTest.java @@ -0,0 +1,39 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.stats; + +import static com.google.common.truth.Truth.assertWithMessage; + +import io.opencensus.stats.View; +import org.junit.Test; + +public class BuiltinViewConstantsTest { + @Test + public void testBasicTagsExistForAllViews() { + for (View v : BuiltinViews.BIGTABLE_BUILTIN_VIEWS) { + assertWithMessage(v.getName() + " should have all basic tags") + .that(v.getColumns()) + .containsAtLeast( + BuiltinMeasureConstants.PROJECT_ID, + BuiltinMeasureConstants.INSTANCE_ID, + BuiltinMeasureConstants.APP_PROFILE, + BuiltinMeasureConstants.METHOD, + BuiltinMeasureConstants.ZONE, + BuiltinMeasureConstants.CLUSTER, + BuiltinMeasureConstants.TABLE); + } + } +} diff --git a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/StatsRecorderWrapperTest.java b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/StatsRecorderWrapperTest.java new file mode 100644 index 0000000000..ed67472623 --- /dev/null +++ b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/StatsRecorderWrapperTest.java @@ -0,0 +1,500 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.stats; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.api.gax.tracing.ApiTracerFactory; +import com.google.api.gax.tracing.SpanName; +import com.google.common.collect.ImmutableMap; +import io.opencensus.impl.stats.StatsComponentImpl; +import io.opencensus.stats.AggregationData; +import io.opencensus.stats.StatsComponent; +import io.opencensus.stats.View; +import io.opencensus.stats.ViewData; +import io.opencensus.stats.ViewManager; +import io.opencensus.tags.TagKey; +import io.opencensus.tags.TagValue; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import org.junit.Before; +import org.junit.Test; + +public class StatsRecorderWrapperTest { + + private final String PROJECT_ID = "fake-project"; + private final String INSTANCE_ID = "fake-instance"; + private final String APP_PROFILE_ID = "fake-app-profile"; + + private final String TABLE_ID = "fake-table-id"; + private final String ZONE = "fake-zone"; + private final String CLUSTER = "fake-cluster"; + + private final StatsComponent statsComponent = new StatsComponentImpl(); + + @Before + public void setup() { + BuiltinViews views = new BuiltinViews(); + views.registerPrivateViews(statsComponent.getViewManager()); + } + + @Test + public void testStreamingOperation() throws InterruptedException { + StatsRecorderWrapper recorderWrapper = + new StatsRecorderWrapper( + ApiTracerFactory.OperationType.ServerStreaming, + SpanName.of("Bigtable", "ReadRows"), + ImmutableMap.of( + BuiltinMeasureConstants.PROJECT_ID.getName(), + PROJECT_ID, + BuiltinMeasureConstants.INSTANCE_ID.getName(), + INSTANCE_ID, + BuiltinMeasureConstants.APP_PROFILE.getName(), + APP_PROFILE_ID), + statsComponent.getStatsRecorder()); + + long operationLatency = 1234; + int attemptCount = 2; + long attemptLatency = 56; + long serverLatency = 78; + long applicationLatency = 901; + long connectivityErrorCount = 15; + long throttlingLatency = 50; + long firstResponseLatency = 90; + + recorderWrapper.putOperationLatencies(operationLatency); + recorderWrapper.putRetryCount(attemptCount); + recorderWrapper.putAttemptLatencies(attemptLatency); + recorderWrapper.putApplicationLatencies(applicationLatency); + recorderWrapper.putGfeLatencies(serverLatency); + recorderWrapper.putGfeMissingHeaders(connectivityErrorCount); + recorderWrapper.putFirstResponseLatencies(firstResponseLatency); + recorderWrapper.putBatchRequestThrottled(throttlingLatency); + + recorderWrapper.record("OK", TABLE_ID, ZONE, CLUSTER); + + Thread.sleep(100); + + assertThat( + getAggregationValueAsLong( + BuiltinViewConstants.OPERATION_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, "Bigtable.ReadRows", + BuiltinMeasureConstants.STATUS, "OK", + BuiltinMeasureConstants.TABLE, TABLE_ID, + BuiltinMeasureConstants.ZONE, ZONE, + BuiltinMeasureConstants.CLUSTER, CLUSTER, + BuiltinMeasureConstants.CLIENT_NAME, "bigtable-java", + BuiltinMeasureConstants.STREAMING, "true"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID, + statsComponent.getViewManager())) + .isEqualTo(operationLatency); + assertThat( + getAggregationValueAsLong( + BuiltinViewConstants.ATTEMPT_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.ReadRows", + BuiltinMeasureConstants.STATUS, + "OK", + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER, + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java", + BuiltinMeasureConstants.STREAMING, + "true"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID, + statsComponent.getViewManager())) + .isEqualTo(attemptLatency); + assertThat( + getAggregationValueAsLong( + BuiltinViewConstants.RETRY_COUNT_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.ReadRows", + BuiltinMeasureConstants.STATUS, + "OK", + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER, + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID, + statsComponent.getViewManager())) + .isEqualTo(attemptCount); + assertThat( + getAggregationValueAsLong( + BuiltinViewConstants.SERVER_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.ReadRows", + BuiltinMeasureConstants.STATUS, + "OK", + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java", + BuiltinMeasureConstants.STREAMING, + "true", + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID, + statsComponent.getViewManager())) + .isEqualTo(serverLatency); + assertThat( + getAggregationValueAsLong( + BuiltinViewConstants.APPLICATION_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.ReadRows", + BuiltinMeasureConstants.STATUS, + "OK", + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER, + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java", + BuiltinMeasureConstants.STREAMING, + "true"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID, + statsComponent.getViewManager())) + .isEqualTo(applicationLatency); + assertThat( + getAggregationValueAsLong( + BuiltinViewConstants.CONNECTIVITY_ERROR_COUNT_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.ReadRows", + BuiltinMeasureConstants.STATUS, + "OK", + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java", + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID, + statsComponent.getViewManager())) + .isEqualTo(connectivityErrorCount); + assertThat( + getAggregationValueAsLong( + BuiltinViewConstants.THROTTLING_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, "Bigtable.ReadRows", + BuiltinMeasureConstants.TABLE, TABLE_ID, + BuiltinMeasureConstants.ZONE, ZONE, + BuiltinMeasureConstants.CLUSTER, CLUSTER, + BuiltinMeasureConstants.CLIENT_NAME, "bigtable-java"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID, + statsComponent.getViewManager())) + .isEqualTo(throttlingLatency); + assertThat( + getAggregationValueAsLong( + BuiltinViewConstants.FIRST_RESPONSE_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.ReadRows", + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER, + BuiltinMeasureConstants.STATUS, + "OK", + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID, + statsComponent.getViewManager())) + .isEqualTo(firstResponseLatency); + } + + @Test + public void testUnaryOperations() throws InterruptedException { + StatsRecorderWrapper recorderWrapper = + new StatsRecorderWrapper( + ApiTracerFactory.OperationType.Unary, + SpanName.of("Bigtable", "MutateRow"), + ImmutableMap.of( + BuiltinMeasureConstants.PROJECT_ID.getName(), PROJECT_ID, + BuiltinMeasureConstants.INSTANCE_ID.getName(), INSTANCE_ID, + BuiltinMeasureConstants.APP_PROFILE.getName(), APP_PROFILE_ID), + statsComponent.getStatsRecorder()); + + long operationLatency = 1234; + int attemptCount = 2; + long attemptLatency = 56; + long serverLatency = 78; + long applicationLatency = 901; + long connectivityErrorCount = 15; + long throttlingLatency = 50; + long firstResponseLatency = 90; + + recorderWrapper.putOperationLatencies(operationLatency); + recorderWrapper.putRetryCount(attemptCount); + recorderWrapper.putAttemptLatencies(attemptLatency); + recorderWrapper.putApplicationLatencies(applicationLatency); + recorderWrapper.putGfeLatencies(serverLatency); + recorderWrapper.putGfeMissingHeaders(connectivityErrorCount); + recorderWrapper.putFirstResponseLatencies(firstResponseLatency); + recorderWrapper.putBatchRequestThrottled(throttlingLatency); + + recorderWrapper.record("UNAVAILABLE", TABLE_ID, ZONE, CLUSTER); + + Thread.sleep(100); + + assertThat( + getAggregationValueAsLong( + BuiltinViewConstants.OPERATION_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.MutateRow", + BuiltinMeasureConstants.STATUS, + "UNAVAILABLE", + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER, + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java", + BuiltinMeasureConstants.STREAMING, + "false"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID, + statsComponent.getViewManager())) + .isEqualTo(operationLatency); + assertThat( + getAggregationValueAsLong( + BuiltinViewConstants.ATTEMPT_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.MutateRow", + BuiltinMeasureConstants.STATUS, + "UNAVAILABLE", + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER, + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java", + BuiltinMeasureConstants.STREAMING, + "false"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID, + statsComponent.getViewManager())) + .isEqualTo(attemptLatency); + assertThat( + getAggregationValueAsLong( + BuiltinViewConstants.RETRY_COUNT_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.MutateRow", + BuiltinMeasureConstants.STATUS, + "UNAVAILABLE", + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER, + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID, + statsComponent.getViewManager())) + .isEqualTo(attemptCount); + assertThat( + getAggregationValueAsLong( + BuiltinViewConstants.SERVER_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.MutateRow", + BuiltinMeasureConstants.STATUS, + "UNAVAILABLE", + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java", + BuiltinMeasureConstants.STREAMING, + "false", + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID, + statsComponent.getViewManager())) + .isEqualTo(serverLatency); + assertThat( + getAggregationValueAsLong( + BuiltinViewConstants.APPLICATION_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.MutateRow", + BuiltinMeasureConstants.STATUS, + "UNAVAILABLE", + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER, + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java", + BuiltinMeasureConstants.STREAMING, + "false"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID, + statsComponent.getViewManager())) + .isEqualTo(applicationLatency); + assertThat( + getAggregationValueAsLong( + BuiltinViewConstants.CONNECTIVITY_ERROR_COUNT_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.MutateRow", + BuiltinMeasureConstants.STATUS, + "UNAVAILABLE", + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java", + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID, + statsComponent.getViewManager())) + .isEqualTo(connectivityErrorCount); + assertThat( + getAggregationValueAsLong( + BuiltinViewConstants.THROTTLING_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, "Bigtable.MutateRow", + BuiltinMeasureConstants.TABLE, TABLE_ID, + BuiltinMeasureConstants.ZONE, ZONE, + BuiltinMeasureConstants.CLUSTER, CLUSTER, + BuiltinMeasureConstants.CLIENT_NAME, "bigtable-java"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID, + statsComponent.getViewManager())) + .isEqualTo(throttlingLatency); + assertThat( + getAggregationValueAsLong( + BuiltinViewConstants.FIRST_RESPONSE_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.MutateRow", + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER, + BuiltinMeasureConstants.STATUS, + "UNAVAILABLE", + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID, + statsComponent.getViewManager())) + .isEqualTo(firstResponseLatency); + } + + long getAggregationValueAsLong( + View view, + ImmutableMap tags, + String projectId, + String instanceId, + String appProfileId, + ViewManager viewManager) { + ViewData viewData = viewManager.getView(view.getName()); + Map, AggregationData> aggregationMap = + Objects.requireNonNull(viewData).getAggregationMap(); + + List tagValues = new ArrayList<>(); + + for (TagKey column : view.getColumns()) { + if (BuiltinMeasureConstants.PROJECT_ID == column) { + tagValues.add(TagValue.create(projectId)); + } else if (BuiltinMeasureConstants.INSTANCE_ID == column) { + tagValues.add(TagValue.create(instanceId)); + } else if (BuiltinMeasureConstants.APP_PROFILE == column) { + tagValues.add(TagValue.create(appProfileId)); + } else { + tagValues.add(TagValue.create(tags.get(column))); + } + } + + AggregationData aggregationData = aggregationMap.get(tagValues); + + return aggregationData.match( + arg -> (long) arg.getSum(), + AggregationData.SumDataLong::getSum, + arg -> arg.getCount(), + arg -> (long) arg.getMean(), + arg -> (long) arg.getLastValue(), + AggregationData.LastValueDataLong::getLastValue, + arg -> { + throw new UnsupportedOperationException(); + }); + } +} diff --git a/pom.xml b/pom.xml index a22e9eee2e..74f02cdeb4 100644 --- a/pom.xml +++ b/pom.xml @@ -340,6 +340,7 @@ grpc-google-cloud-bigtable-v2 proto-google-cloud-bigtable-admin-v2 proto-google-cloud-bigtable-v2 + google-cloud-bigtable-stats google-cloud-bigtable-emulator-core google-cloud-bigtable-emulator google-cloud-bigtable-bom