From 7a588c8eb50d9fa887ee1cd7bb2abff00ae73719 Mon Sep 17 00:00:00 2001 From: Sydney Munro Date: Tue, 1 Aug 2023 14:11:16 -0700 Subject: [PATCH 01/12] feat: Initial CLI for SSB integration --- pom.xml | 1 + ssb/pom.xml | 71 +++++++++++++++++++ .../cloud/StorageSharedBenchmarkingCli.java | 68 ++++++++++++++++++ 3 files changed, 140 insertions(+) create mode 100644 ssb/pom.xml create mode 100644 ssb/src/main/java/com/google/cloud/StorageSharedBenchmarkingCli.java diff --git a/pom.xml b/pom.xml index af3f0ffde2..bad97df71c 100644 --- a/pom.xml +++ b/pom.xml @@ -206,6 +206,7 @@ proto-google-cloud-storage-v2 gapic-google-cloud-storage-v2 google-cloud-storage-bom + ssb diff --git a/ssb/pom.xml b/ssb/pom.xml new file mode 100644 index 0000000000..1c7ed6c86e --- /dev/null +++ b/ssb/pom.xml @@ -0,0 +1,71 @@ + + + 4.0.0 + + com.google.cloud + google-cloud-storage-parent + 2.24.1-SNAPSHOT + + + ssb + + + 11 + 11 + UTF-8 + + + + com.google.cloud + google-cloud-storage + 2.22.6 + + + info.picocli + picocli + 4.7.0 + + + + + + + + org.apache.maven.plugins + maven-jar-plugin + 3.3.0 + + + + true + dependency-jars/ + com.google.cloud.StorageSharedBenchmarkingCli + + + + + + org.apache.maven.plugins + maven-dependency-plugin + 3.6.0 + + + copy-dependencies + package + + copy-dependencies + + + + ${project.build.directory}/dependency-jars/ + + + + + + + + + \ No newline at end of file diff --git a/ssb/src/main/java/com/google/cloud/StorageSharedBenchmarkingCli.java b/ssb/src/main/java/com/google/cloud/StorageSharedBenchmarkingCli.java new file mode 100644 index 0000000000..13515c128c --- /dev/null +++ b/ssb/src/main/java/com/google/cloud/StorageSharedBenchmarkingCli.java @@ -0,0 +1,68 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud; + +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageOptions; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; + +@Command(name = "ssbcli") +public class StorageSharedBenchmarkingCli implements Runnable { + // TODO: check what input validation is needed for option values. + @Option(names = "-project", description = "GCP Project Identifier") + String project; + + @Option(names = "-bucket", description = "Name of the bucket to use") + String bucket; + + @Option(names = "-samples", defaultValue = "8000", description = "Number of samples to report") + int samples; + + @Option( + names = "-workers", + defaultValue = "16", + description = "Number of workers to run in parallel for the workload") + int workers; + + @Option(names = "-api", defaultValue = "Mixed", description = "API to use") + String api; + + @Option( + names = "-object_size", + defaultValue = "1048576", + description = "Object size in bytes to use for the workload") + int objectSize; + + @Option( + names = "-output_type", + defaultValue = "cloud-monitoring", + description = "Output results format") + String outputType; + + public static void main(String[] args) { + CommandLine cmd = new CommandLine(StorageSharedBenchmarkingCli.class).setUsageHelpWidth(100); + System.exit(cmd.execute(args)); + } + + @Override + public void run() { + Storage storageClient = StorageOptions.newBuilder().setProjectId(project).build().getService(); + + } +} From b28a1557a2c2632b089c13fcee9d57423c3fb5d5 Mon Sep 17 00:00:00 2001 From: Sydney Munro Date: Fri, 4 Aug 2023 12:55:47 -0700 Subject: [PATCH 02/12] Make project and bucket required fields --- .../google/cloud/StorageSharedBenchmarkingCli.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ssb/src/main/java/com/google/cloud/StorageSharedBenchmarkingCli.java b/ssb/src/main/java/com/google/cloud/StorageSharedBenchmarkingCli.java index 13515c128c..635fa8d9e9 100644 --- a/ssb/src/main/java/com/google/cloud/StorageSharedBenchmarkingCli.java +++ b/ssb/src/main/java/com/google/cloud/StorageSharedBenchmarkingCli.java @@ -16,19 +16,20 @@ package com.google.cloud; +import com.google.cloud.storage.Bucket; import com.google.cloud.storage.Storage; import com.google.cloud.storage.StorageOptions; import picocli.CommandLine; import picocli.CommandLine.Command; import picocli.CommandLine.Option; -@Command(name = "ssbcli") +@Command(name = "ssb") public class StorageSharedBenchmarkingCli implements Runnable { // TODO: check what input validation is needed for option values. - @Option(names = "-project", description = "GCP Project Identifier") + @Option(names = "-project", description = "GCP Project Identifier", required = true) String project; - @Option(names = "-bucket", description = "Name of the bucket to use") + @Option(names = "-bucket", description = "Name of the bucket to use", required = true) String bucket; @Option(names = "-samples", defaultValue = "8000", description = "Number of samples to report") @@ -56,13 +57,13 @@ public class StorageSharedBenchmarkingCli implements Runnable { String outputType; public static void main(String[] args) { - CommandLine cmd = new CommandLine(StorageSharedBenchmarkingCli.class).setUsageHelpWidth(100); + CommandLine cmd = new CommandLine(StorageSharedBenchmarkingCli.class); System.exit(cmd.execute(args)); } @Override public void run() { Storage storageClient = StorageOptions.newBuilder().setProjectId(project).build().getService(); - + Bucket bucketToUse = storageClient.get(bucket); } } From 5f0c161133a2ca4af765b161d45086927fdcd494 Mon Sep 17 00:00:00 2001 From: Sydney Munro Date: Fri, 11 Aug 2023 16:06:24 -0700 Subject: [PATCH 03/12] test: Add performance tests for workload1 --- ssb/pom.xml | 35 ++- .../google/cloud/CloudMonitoringResult.java | 266 ++++++++++++++++++ .../cloud/StorageSharedBenchmarkingCli.java | 61 +++- .../main/java/com/google/cloud/Workload1.java | 76 +++++ 4 files changed, 428 insertions(+), 10 deletions(-) create mode 100644 ssb/src/main/java/com/google/cloud/CloudMonitoringResult.java create mode 100644 ssb/src/main/java/com/google/cloud/Workload1.java diff --git a/ssb/pom.xml b/ssb/pom.xml index 1c7ed6c86e..e6e59d3281 100644 --- a/ssb/pom.xml +++ b/ssb/pom.xml @@ -17,15 +17,26 @@ UTF-8 + + info.picocli + picocli + 4.7.0 + com.google.cloud google-cloud-storage - 2.22.6 + 2.24.1-SNAPSHOT - info.picocli - picocli - 4.7.0 + com.google.cloud + google-cloud-storage + 2.24.1-SNAPSHOT + tests + + + junit + junit + compile @@ -65,6 +76,22 @@ + + org.apache.maven.plugins + maven-deploy-plugin + 3.1.1 + + true + + + + org.sonatype.plugins + nexus-staging-maven-plugin + 1.6.13 + + true + + diff --git a/ssb/src/main/java/com/google/cloud/CloudMonitoringResult.java b/ssb/src/main/java/com/google/cloud/CloudMonitoringResult.java new file mode 100644 index 0000000000..6cadb884df --- /dev/null +++ b/ssb/src/main/java/com/google/cloud/CloudMonitoringResult.java @@ -0,0 +1,266 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud; + +import com.google.common.base.MoreObjects; +import java.util.Objects; + +final class CloudMonitoringResult { + private final String library; + private final String api; + private final String op; + + private final int workers; + private final int object_size; + private final int app_buffer_size; + private final int chunksize; + private final boolean crc32c_enabled; + private final boolean md5_enabled; + private final int cpu_time_us; + private final String bucket_name; + private final String status; + private final String transfer_size; + private final String transfer_offset; + private final String failure_msg; + private final double throughput; + + CloudMonitoringResult( + String library, + String api, + String op, + int workers, + int objectSize, + int appBufferSize, + int chunksize, + boolean crc32cEnabled, + boolean md5Enabled, + int cpuTimeUs, + String bucketName, + String status, + String transferSize, + String transferOffset, + String failureMsg, + double throughput) { + this.library = library; + this.api = api; + this.op = op; + this.workers = workers; + this.object_size = objectSize; + this.app_buffer_size = appBufferSize; + this.chunksize = chunksize; + this.crc32c_enabled = crc32cEnabled; + this.md5_enabled = md5Enabled; + this.cpu_time_us = cpuTimeUs; + this.bucket_name = bucketName; + this.status = status; + this.transfer_size = transferSize; + this.transfer_offset = transferOffset; + this.failure_msg = failureMsg; + this.throughput = throughput; + } + + public static Builder newBuilder() { + return new Builder(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof CloudMonitoringResult)) { + return false; + } + CloudMonitoringResult result = (CloudMonitoringResult) o; + return workers == result.workers + && object_size == result.object_size + && app_buffer_size == result.app_buffer_size + && chunksize == result.chunksize + && crc32c_enabled == result.crc32c_enabled + && md5_enabled == result.md5_enabled + && cpu_time_us == result.cpu_time_us + && Objects.equals(library, result.library) + && Objects.equals(api, result.api) + && Objects.equals(op, result.op) + && Objects.equals(bucket_name, result.bucket_name) + && Objects.equals(status, result.status) + && Objects.equals(transfer_size, result.transfer_size) + && Objects.equals(transfer_offset, result.transfer_offset) + && Objects.equals(failure_msg, result.failure_msg); + } + + @Override + public int hashCode() { + return Objects.hash( + library, + api, + op, + workers, + object_size, + app_buffer_size, + chunksize, + crc32c_enabled, + md5_enabled, + cpu_time_us, + bucket_name, + status, + transfer_size, + transfer_offset, + failure_msg); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("library", library) + .add("api", api) + .add("op", op) + .add("workers", workers) + .add("object_size", object_size) + .add("app_buffer_size", app_buffer_size) + .add("chunksize", chunksize) + .add("crc32c_enabled", crc32c_enabled) + .add("md5_enabled", md5_enabled) + .add("cpu_time_us", cpu_time_us) + .add("bucket_name", bucket_name) + .add("status", status) + .add("transfer_size", transfer_size) + .add("transfer_offset", transfer_offset) + .add("failure_msg", failure_msg) + .toString(); + } + + public static class Builder { + + private String library; + private String api; + private String op; + private int workers; + private int objectSize; + private int appBufferSize; + private int chunksize; + private boolean crc32cEnabled; + private boolean md5Enabled; + private int cpuTimeUs; + private String bucketName; + private String status; + private String transferSize; + private String transferOffset; + private String failureMsg; + private double throughput; + + public Builder setLibrary(String library) { + this.library = library; + return this; + } + + public Builder setApi(String api) { + this.api = api; + return this; + } + + public Builder setOp(String op) { + this.op = op; + return this; + } + + public Builder setWorkers(int workers) { + this.workers = workers; + return this; + } + + public Builder setObjectSize(int objectSize) { + this.objectSize = objectSize; + return this; + } + + public Builder setAppBufferSize(int appBufferSize) { + this.appBufferSize = appBufferSize; + return this; + } + + public Builder setChunksize(int chunksize) { + this.chunksize = chunksize; + return this; + } + + public Builder setCrc32cEnabled(boolean crc32cEnabled) { + this.crc32cEnabled = crc32cEnabled; + return this; + } + + public Builder setMd5Enabled(boolean md5Enabled) { + this.md5Enabled = md5Enabled; + return this; + } + + public Builder setCpuTimeUs(int cpuTimeUs) { + this.cpuTimeUs = cpuTimeUs; + return this; + } + + public Builder setBucketName(String bucketName) { + this.bucketName = bucketName; + return this; + } + + public Builder setStatus(String status) { + this.status = status; + return this; + } + + public Builder setTransferSize(String transferSize) { + this.transferSize = transferSize; + return this; + } + + public Builder setTransferOffset(String transferOffset) { + this.transferOffset = transferOffset; + return this; + } + + public Builder setFailureMsg(String failureMsg) { + this.failureMsg = failureMsg; + return this; + } + + public Builder setThroughput(double throughput) { + this.throughput = throughput; + return this; + } + + public CloudMonitoringResult build() { + return new CloudMonitoringResult( + library, + api, + op, + workers, + objectSize, + appBufferSize, + chunksize, + crc32cEnabled, + md5Enabled, + cpuTimeUs, + bucketName, + status, + transferSize, + transferOffset, + failureMsg, + throughput); + } + } +} diff --git a/ssb/src/main/java/com/google/cloud/StorageSharedBenchmarkingCli.java b/ssb/src/main/java/com/google/cloud/StorageSharedBenchmarkingCli.java index 635fa8d9e9..a44daff72e 100644 --- a/ssb/src/main/java/com/google/cloud/StorageSharedBenchmarkingCli.java +++ b/ssb/src/main/java/com/google/cloud/StorageSharedBenchmarkingCli.java @@ -16,15 +16,32 @@ package com.google.cloud; -import com.google.cloud.storage.Bucket; +import com.google.api.core.ApiFuture; +import com.google.api.core.ApiFutures; +import com.google.api.core.ListenableFutureToApiFuture; +import com.google.api.gax.retrying.RetrySettings; +import com.google.api.gax.rpc.ApiExceptions; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.DataGenerator; import com.google.cloud.storage.Storage; import com.google.cloud.storage.StorageOptions; +import com.google.cloud.storage.TmpFile; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Executors; import picocli.CommandLine; import picocli.CommandLine.Command; import picocli.CommandLine.Option; @Command(name = "ssb") public class StorageSharedBenchmarkingCli implements Runnable { + public static long SSB_SIZE_THRESHOLD_BYTES = 1048576; // TODO: check what input validation is needed for option values. @Option(names = "-project", description = "GCP Project Identifier", required = true) String project; @@ -39,10 +56,10 @@ public class StorageSharedBenchmarkingCli implements Runnable { names = "-workers", defaultValue = "16", description = "Number of workers to run in parallel for the workload") - int workers; + static int workers; - @Option(names = "-api", defaultValue = "Mixed", description = "API to use") - String api; + @Option(names = "-api", defaultValue = "JSON", description = "API to use") + static String api; @Option( names = "-object_size", @@ -56,6 +73,12 @@ public class StorageSharedBenchmarkingCli implements Runnable { description = "Output results format") String outputType; + @Option( + names = "-test_type", + description = "Specify which workload the cli should run", + required = true) + String testType; + public static void main(String[] args) { CommandLine cmd = new CommandLine(StorageSharedBenchmarkingCli.class); System.exit(cmd.execute(args)); @@ -63,7 +86,33 @@ public static void main(String[] args) { @Override public void run() { - Storage storageClient = StorageOptions.newBuilder().setProjectId(project).build().getService(); - Bucket bucketToUse = storageClient.get(bucket); + // TODO: Make this a switch once we add more workloads + runWorkload1(); + } + + private void runWorkload1() { + RetrySettings retrySettings = StorageOptions.getDefaultRetrySettings().toBuilder().build(); + + StorageOptions alwaysRetryStorageOptions = + StorageOptions.newBuilder().setProjectId(project).setRetrySettings(retrySettings).build(); + Storage storageClient = alwaysRetryStorageOptions.getService(); + Path tempDir = Paths.get(System.getProperty("java.io.tmpdir")); + try { + ListeningExecutorService executorService = + MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(workers)); + List> workloadRuns = new ArrayList<>(); + for (int i = 0; i < samples; i++) { + TmpFile file = DataGenerator.base64Characters().tempFile(tempDir, objectSize); + BlobInfo blob = BlobInfo.newBuilder(bucket, file.toString()).build(); + workloadRuns.add(convert(executorService.submit(new Workload1(file, blob, storageClient)))); + } + ApiExceptions.callAndTranslateApiException(ApiFutures.allAsList(workloadRuns)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static ApiFuture convert(ListenableFuture lf) { + return new ListenableFutureToApiFuture<>(lf); } } diff --git a/ssb/src/main/java/com/google/cloud/Workload1.java b/ssb/src/main/java/com/google/cloud/Workload1.java new file mode 100644 index 0000000000..5b0e6dd50e --- /dev/null +++ b/ssb/src/main/java/com/google/cloud/Workload1.java @@ -0,0 +1,76 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud; + +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.TmpFile; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.concurrent.Callable; + +final class Workload1 implements Callable { + private final TmpFile file; + private final BlobInfo blob; + private final Storage storage; + + Workload1(TmpFile file, BlobInfo blob, Storage storage) { + this.file = file; + this.blob = blob; + this.storage = storage; + } + + @Override + public String call() throws Exception { + Clock clock = Clock.systemDefaultZone(); + + // Get the start time + Instant startTime = clock.instant(); + Blob created = storage.createFrom(blob, file.getPath()); + Instant endTime = clock.instant(); + Duration elapsedTime = Duration.between(startTime, endTime); + double throughput = + created.getSize() >= StorageSharedBenchmarkingCli.SSB_SIZE_THRESHOLD_BYTES + ? created.getSize() / 1024 / 1024 / (elapsedTime.toNanos()) + : created.getSize() / 1024 / 1024 / (elapsedTime.toNanos()); + System.out.println(generateCloudMonitoringResult("WRITE", throughput, created).toString()); + return "OK"; + } + + private CloudMonitoringResult generateCloudMonitoringResult( + String op, double throughput, Blob created) { + CloudMonitoringResult result = + CloudMonitoringResult.newBuilder() + .setLibrary("java") + .setApi(StorageSharedBenchmarkingCli.api) + .setOp(op) + .setWorkers(StorageSharedBenchmarkingCli.workers) + .setObjectSize(created.getSize().intValue()) + .setChunksize(created.getSize().intValue()) + .setCrc32cEnabled(false) + .setMd5Enabled(false) + .setCpuTimeUs(-1) + .setBucketName(created.getBucket()) + .setStatus("OK") + .setTransferSize(created.getSize().toString()) + .setThroughput(throughput) + .build(); + return result; + } +} From 64f1aae70b9d2ead76a78fa6a1d9b3e47b8ae172 Mon Sep 17 00:00:00 2001 From: Sydney Munro Date: Mon, 14 Aug 2023 10:44:29 -0700 Subject: [PATCH 04/12] Use a range for object size --- .../cloud/StorageSharedBenchmarkingCli.java | 56 +++++++++++++++---- 1 file changed, 45 insertions(+), 11 deletions(-) diff --git a/ssb/src/main/java/com/google/cloud/StorageSharedBenchmarkingCli.java b/ssb/src/main/java/com/google/cloud/StorageSharedBenchmarkingCli.java index a44daff72e..b10fa3672d 100644 --- a/ssb/src/main/java/com/google/cloud/StorageSharedBenchmarkingCli.java +++ b/ssb/src/main/java/com/google/cloud/StorageSharedBenchmarkingCli.java @@ -34,7 +34,9 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; +import java.util.Random; import java.util.concurrent.Executors; +import java.util.regex.Pattern; import picocli.CommandLine; import picocli.CommandLine.Command; import picocli.CommandLine.Option; @@ -63,9 +65,9 @@ public class StorageSharedBenchmarkingCli implements Runnable { @Option( names = "-object_size", - defaultValue = "1048576", + defaultValue = "1048576...1048576", description = "Object size in bytes to use for the workload") - int objectSize; + String objectSize; @Option( names = "-output_type", @@ -97,22 +99,54 @@ private void runWorkload1() { StorageOptions.newBuilder().setProjectId(project).setRetrySettings(retrySettings).build(); Storage storageClient = alwaysRetryStorageOptions.getService(); Path tempDir = Paths.get(System.getProperty("java.io.tmpdir")); - try { - ListeningExecutorService executorService = - MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(workers)); - List> workloadRuns = new ArrayList<>(); - for (int i = 0; i < samples; i++) { - TmpFile file = DataGenerator.base64Characters().tempFile(tempDir, objectSize); + ListeningExecutorService executorService = + MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(workers)); + List> workloadRuns = new ArrayList<>(); + Range objectSizeRange = Range.of(objectSize); + for (int i = 0; i < samples; i++) { + try (TmpFile file = + DataGenerator.base64Characters() + .tempFile(tempDir, getRandomInt(objectSizeRange.min, objectSizeRange.max))) { BlobInfo blob = BlobInfo.newBuilder(bucket, file.toString()).build(); workloadRuns.add(convert(executorService.submit(new Workload1(file, blob, storageClient)))); + } catch (IOException e) { + throw new RuntimeException(e); } - ApiExceptions.callAndTranslateApiException(ApiFutures.allAsList(workloadRuns)); - } catch (IOException e) { - throw new RuntimeException(e); } + ApiExceptions.callAndTranslateApiException(ApiFutures.allAsList(workloadRuns)); + } + + public static int getRandomInt(int min, int max) { + Random random = new Random(); + return random.nextInt((max - min) + 1) + min; } private static ApiFuture convert(ListenableFuture lf) { return new ListenableFutureToApiFuture<>(lf); } + + private static final class Range { + private final int min; + private final int max; + + private Range(int min, int max) { + this.min = min; + this.max = max; + } + + public static Range of(int min, int max) { + return new Range(min, max); + } + // Takes an object size range of format min...max and creates a range object + public static Range of(String range) { + Pattern p = Pattern.compile("..."); + String[] splitRangeVals = p.split(range); + if (splitRangeVals.length == 2) { + String min = splitRangeVals[0]; + String max = splitRangeVals[1]; + return of(Integer.parseInt(min), Integer.parseInt(max)); + } + throw new IllegalStateException("Expected a size range of format min..max, but got " + range); + } + } } From 8fa9367237b35d513fadd03370ec382ac7fa8a11 Mon Sep 17 00:00:00 2001 From: Sydney Munro Date: Mon, 14 Aug 2023 14:26:46 -0700 Subject: [PATCH 05/12] Add Reads for Workload 1 --- .../google/cloud/CloudMonitoringResult.java | 47 ++++++++++--------- .../cloud/StorageSharedBenchmarkingCli.java | 21 +++++---- .../cloud/StorageSharedBenchmarkingUtils.java | 36 ++++++++++++++ .../main/java/com/google/cloud/Workload1.java | 44 +++++++++++++---- 4 files changed, 108 insertions(+), 40 deletions(-) create mode 100644 ssb/src/main/java/com/google/cloud/StorageSharedBenchmarkingUtils.java diff --git a/ssb/src/main/java/com/google/cloud/CloudMonitoringResult.java b/ssb/src/main/java/com/google/cloud/CloudMonitoringResult.java index 6cadb884df..61f0d5133b 100644 --- a/ssb/src/main/java/com/google/cloud/CloudMonitoringResult.java +++ b/ssb/src/main/java/com/google/cloud/CloudMonitoringResult.java @@ -77,6 +77,28 @@ public static Builder newBuilder() { return new Builder(); } + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("library", library) + .add("api", api) + .add("op", op) + .add("workers", workers) + .add("object_size", object_size) + .add("app_buffer_size", app_buffer_size) + .add("chunksize", chunksize) + .add("crc32c_enabled", crc32c_enabled) + .add("md5_enabled", md5_enabled) + .add("cpu_time_us", cpu_time_us) + .add("bucket_name", bucket_name) + .add("status", status) + .add("transfer_size", transfer_size) + .add("transfer_offset", transfer_offset) + .add("failure_msg", failure_msg) + .add("throughput", throughput) + .toString(); + } + @Override public boolean equals(Object o) { if (this == o) { @@ -93,6 +115,7 @@ public boolean equals(Object o) { && crc32c_enabled == result.crc32c_enabled && md5_enabled == result.md5_enabled && cpu_time_us == result.cpu_time_us + && Double.compare(result.throughput, throughput) == 0 && Objects.equals(library, result.library) && Objects.equals(api, result.api) && Objects.equals(op, result.op) @@ -120,28 +143,8 @@ public int hashCode() { status, transfer_size, transfer_offset, - failure_msg); - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("library", library) - .add("api", api) - .add("op", op) - .add("workers", workers) - .add("object_size", object_size) - .add("app_buffer_size", app_buffer_size) - .add("chunksize", chunksize) - .add("crc32c_enabled", crc32c_enabled) - .add("md5_enabled", md5_enabled) - .add("cpu_time_us", cpu_time_us) - .add("bucket_name", bucket_name) - .add("status", status) - .add("transfer_size", transfer_size) - .add("transfer_offset", transfer_offset) - .add("failure_msg", failure_msg) - .toString(); + failure_msg, + throughput); } public static class Builder { diff --git a/ssb/src/main/java/com/google/cloud/StorageSharedBenchmarkingCli.java b/ssb/src/main/java/com/google/cloud/StorageSharedBenchmarkingCli.java index b10fa3672d..62c77cc273 100644 --- a/ssb/src/main/java/com/google/cloud/StorageSharedBenchmarkingCli.java +++ b/ssb/src/main/java/com/google/cloud/StorageSharedBenchmarkingCli.java @@ -42,8 +42,7 @@ import picocli.CommandLine.Option; @Command(name = "ssb") -public class StorageSharedBenchmarkingCli implements Runnable { - public static long SSB_SIZE_THRESHOLD_BYTES = 1048576; +public final class StorageSharedBenchmarkingCli implements Runnable { // TODO: check what input validation is needed for option values. @Option(names = "-project", description = "GCP Project Identifier", required = true) String project; @@ -58,10 +57,10 @@ public class StorageSharedBenchmarkingCli implements Runnable { names = "-workers", defaultValue = "16", description = "Number of workers to run in parallel for the workload") - static int workers; + int workers; @Option(names = "-api", defaultValue = "JSON", description = "API to use") - static String api; + String api; @Option( names = "-object_size", @@ -104,11 +103,14 @@ private void runWorkload1() { List> workloadRuns = new ArrayList<>(); Range objectSizeRange = Range.of(objectSize); for (int i = 0; i < samples; i++) { - try (TmpFile file = - DataGenerator.base64Characters() - .tempFile(tempDir, getRandomInt(objectSizeRange.min, objectSizeRange.max))) { + try { + TmpFile file = + DataGenerator.base64Characters() + .tempFile(tempDir, getRandomInt(objectSizeRange.min, objectSizeRange.max)); BlobInfo blob = BlobInfo.newBuilder(bucket, file.toString()).build(); - workloadRuns.add(convert(executorService.submit(new Workload1(file, blob, storageClient)))); + workloadRuns.add( + convert( + executorService.submit(new Workload1(file, blob, storageClient, workers, api)))); } catch (IOException e) { throw new RuntimeException(e); } @@ -117,6 +119,7 @@ private void runWorkload1() { } public static int getRandomInt(int min, int max) { + if (min == max) return min; Random random = new Random(); return random.nextInt((max - min) + 1) + min; } @@ -139,7 +142,7 @@ public static Range of(int min, int max) { } // Takes an object size range of format min...max and creates a range object public static Range of(String range) { - Pattern p = Pattern.compile("..."); + Pattern p = Pattern.compile("\\.\\.\\."); String[] splitRangeVals = p.split(range); if (splitRangeVals.length == 2) { String min = splitRangeVals[0]; diff --git a/ssb/src/main/java/com/google/cloud/StorageSharedBenchmarkingUtils.java b/ssb/src/main/java/com/google/cloud/StorageSharedBenchmarkingUtils.java new file mode 100644 index 0000000000..1eee4cb5ad --- /dev/null +++ b/ssb/src/main/java/com/google/cloud/StorageSharedBenchmarkingUtils.java @@ -0,0 +1,36 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud; + +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.Storage; +import java.time.Duration; + +class StorageSharedBenchmarkingUtils { + public static long SSB_SIZE_THRESHOLD_BYTES = 1048576; + public static int DEFAULT_NUMBER_OF_READS = 3; + + public static void cleanupObject(Storage storage, Blob created) { + storage.delete(created.getBlobId()); + } + + public static double calculateThroughput(long size, Duration elapsedTime) { + return size >= StorageSharedBenchmarkingUtils.SSB_SIZE_THRESHOLD_BYTES + ? size / 1024 / 1024 / (elapsedTime.toNanos()) + : size / 1024 / (elapsedTime.toNanos()); + } +} diff --git a/ssb/src/main/java/com/google/cloud/Workload1.java b/ssb/src/main/java/com/google/cloud/Workload1.java index 5b0e6dd50e..4f40de9561 100644 --- a/ssb/src/main/java/com/google/cloud/Workload1.java +++ b/ssb/src/main/java/com/google/cloud/Workload1.java @@ -16,10 +16,16 @@ package com.google.cloud; +import static com.google.cloud.StorageSharedBenchmarkingUtils.DEFAULT_NUMBER_OF_READS; +import static com.google.cloud.StorageSharedBenchmarkingUtils.calculateThroughput; +import static com.google.cloud.StorageSharedBenchmarkingUtils.cleanupObject; + import com.google.cloud.storage.Blob; import com.google.cloud.storage.BlobInfo; import com.google.cloud.storage.Storage; import com.google.cloud.storage.TmpFile; +import java.nio.file.Path; +import java.nio.file.Paths; import java.time.Clock; import java.time.Duration; import java.time.Instant; @@ -29,11 +35,15 @@ final class Workload1 implements Callable { private final TmpFile file; private final BlobInfo blob; private final Storage storage; + private final int workers; + private final String api; - Workload1(TmpFile file, BlobInfo blob, Storage storage) { + Workload1(TmpFile file, BlobInfo blob, Storage storage, int workers, String api) { this.file = file; this.blob = blob; this.storage = storage; + this.workers = workers; + this.api = api; } @Override @@ -44,12 +54,28 @@ public String call() throws Exception { Instant startTime = clock.instant(); Blob created = storage.createFrom(blob, file.getPath()); Instant endTime = clock.instant(); - Duration elapsedTime = Duration.between(startTime, endTime); - double throughput = - created.getSize() >= StorageSharedBenchmarkingCli.SSB_SIZE_THRESHOLD_BYTES - ? created.getSize() / 1024 / 1024 / (elapsedTime.toNanos()) - : created.getSize() / 1024 / 1024 / (elapsedTime.toNanos()); - System.out.println(generateCloudMonitoringResult("WRITE", throughput, created).toString()); + Duration elapsedTimeUpload = Duration.between(startTime, endTime); + System.out.println( + generateCloudMonitoringResult( + "WRITE", + calculateThroughput(created.getSize().longValue(), elapsedTimeUpload), + created) + .toString()); + Path tempDir = Paths.get(System.getProperty("java.io.tmpdir")); + for (int i = 0; i <= DEFAULT_NUMBER_OF_READS; i++) { + TmpFile dest = TmpFile.of(tempDir, "prefix", "bin"); + startTime = clock.instant(); + storage.downloadTo(created.getBlobId(), dest.getPath()); + endTime = clock.instant(); + Duration elapsedTimeDownload = Duration.between(startTime, endTime); + System.out.println( + generateCloudMonitoringResult( + "READ[" + i + "]", + calculateThroughput(created.getSize().longValue(), elapsedTimeDownload), + created) + .toString()); + } + cleanupObject(storage, created); return "OK"; } @@ -58,9 +84,9 @@ private CloudMonitoringResult generateCloudMonitoringResult( CloudMonitoringResult result = CloudMonitoringResult.newBuilder() .setLibrary("java") - .setApi(StorageSharedBenchmarkingCli.api) + .setApi(api) .setOp(op) - .setWorkers(StorageSharedBenchmarkingCli.workers) + .setWorkers(workers) .setObjectSize(created.getSize().intValue()) .setChunksize(created.getSize().intValue()) .setCrc32cEnabled(false) From 2934505f925d22bd777590aee68adcab7bfdf215 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Tue, 15 Aug 2023 16:17:53 +0000 Subject: [PATCH 06/12] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 66fddeaa59..fecf1389e6 100644 --- a/README.md +++ b/README.md @@ -50,20 +50,20 @@ If you are using Maven without the BOM, add this to your dependencies: If you are using Gradle 5.x or later, add this to your dependencies: ```Groovy -implementation platform('com.google.cloud:libraries-bom:26.18.0') +implementation platform('com.google.cloud:libraries-bom:26.22.0') implementation 'com.google.cloud:google-cloud-storage' ``` If you are using Gradle without BOM, add this to your dependencies: ```Groovy -implementation 'com.google.cloud:google-cloud-storage:2.24.0' +implementation 'com.google.cloud:google-cloud-storage:2.26.1' ``` If you are using SBT, add this to your dependencies: ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-storage" % "2.24.0" +libraryDependencies += "com.google.cloud" % "google-cloud-storage" % "2.26.1" ``` @@ -428,7 +428,7 @@ Java is a registered trademark of Oracle and/or its affiliates. [kokoro-badge-link-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-storage/java11.html [stability-image]: https://img.shields.io/badge/stability-stable-green [maven-version-image]: https://img.shields.io/maven-central/v/com.google.cloud/google-cloud-storage.svg -[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-storage/2.24.0 +[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-storage/2.26.1 [authentication]: https://github.com/googleapis/google-cloud-java#authentication [auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes [predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles From 9d530330ee602b0f7d25f30a22c98ef91f03e61a Mon Sep 17 00:00:00 2001 From: Sydney Munro Date: Tue, 15 Aug 2023 09:59:58 -0700 Subject: [PATCH 07/12] Fixing up dependencies --- ssb/pom.xml | 43 +++++++++++++++++++++++++++++++++---------- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/ssb/pom.xml b/ssb/pom.xml index e6e59d3281..e5dea1c631 100644 --- a/ssb/pom.xml +++ b/ssb/pom.xml @@ -4,18 +4,30 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - com.google.cloud - google-cloud-storage-parent - 2.24.1-SNAPSHOT + com.google.cloud.samples + shared-configuration + 1.2.0 + ssb - 11 - 11 + 1.8 + 1.8 UTF-8 + + + + com.google.cloud + libraries-bom + 26.22.0 + pom + import + + + info.picocli @@ -25,18 +37,29 @@ com.google.cloud google-cloud-storage - 2.24.1-SNAPSHOT + 2.26.2-SNAPSHOT com.google.cloud google-cloud-storage - 2.24.1-SNAPSHOT + 2.26.2-SNAPSHOT tests - junit - junit - compile + com.google.api + gax + + + com.google.api + api-common + + + com.google.guava + guava + + + com.google.cloud + google-cloud-core From 0627c2dd4bef9c51071dc2eef29c095984d15ffe Mon Sep 17 00:00:00 2001 From: Sydney Munro Date: Tue, 15 Aug 2023 14:09:56 -0700 Subject: [PATCH 08/12] Address some of the PR comments --- ssb/pom.xml | 1 + .../google/cloud/CloudMonitoringResult.java | 11 +++++---- .../cloud/StorageSharedBenchmarkingCli.java | 23 ++++++++++--------- .../cloud/StorageSharedBenchmarkingUtils.java | 13 ++++++----- .../main/java/com/google/cloud/Workload1.java | 23 ++++++++----------- 5 files changed, 36 insertions(+), 35 deletions(-) diff --git a/ssb/pom.xml b/ssb/pom.xml index e5dea1c631..d6d2cde589 100644 --- a/ssb/pom.xml +++ b/ssb/pom.xml @@ -17,6 +17,7 @@ 1.8 UTF-8 + diff --git a/ssb/src/main/java/com/google/cloud/CloudMonitoringResult.java b/ssb/src/main/java/com/google/cloud/CloudMonitoringResult.java index 61f0d5133b..e16c564b06 100644 --- a/ssb/src/main/java/com/google/cloud/CloudMonitoringResult.java +++ b/ssb/src/main/java/com/google/cloud/CloudMonitoringResult.java @@ -7,11 +7,12 @@ * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * */ package com.google.cloud; diff --git a/ssb/src/main/java/com/google/cloud/StorageSharedBenchmarkingCli.java b/ssb/src/main/java/com/google/cloud/StorageSharedBenchmarkingCli.java index 62c77cc273..b7d19634e5 100644 --- a/ssb/src/main/java/com/google/cloud/StorageSharedBenchmarkingCli.java +++ b/ssb/src/main/java/com/google/cloud/StorageSharedBenchmarkingCli.java @@ -7,11 +7,12 @@ * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * */ package com.google.cloud; @@ -64,8 +65,8 @@ public final class StorageSharedBenchmarkingCli implements Runnable { @Option( names = "-object_size", - defaultValue = "1048576...1048576", - description = "Object size in bytes to use for the workload") + defaultValue = "1048576..1048576", + description = "any positive integer, or an inclusive range such as min..max where min and max are positive integers") String objectSize; @Option( @@ -94,9 +95,9 @@ public void run() { private void runWorkload1() { RetrySettings retrySettings = StorageOptions.getDefaultRetrySettings().toBuilder().build(); - StorageOptions alwaysRetryStorageOptions = + StorageOptions retryStorageOptions = StorageOptions.newBuilder().setProjectId(project).setRetrySettings(retrySettings).build(); - Storage storageClient = alwaysRetryStorageOptions.getService(); + Storage storageClient = retryStorageOptions.getService(); Path tempDir = Paths.get(System.getProperty("java.io.tmpdir")); ListeningExecutorService executorService = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(workers)); @@ -140,9 +141,9 @@ private Range(int min, int max) { public static Range of(int min, int max) { return new Range(min, max); } - // Takes an object size range of format min...max and creates a range object + // Takes an object size range of format min..max and creates a range object public static Range of(String range) { - Pattern p = Pattern.compile("\\.\\.\\."); + Pattern p = Pattern.compile("\\.\\."); String[] splitRangeVals = p.split(range); if (splitRangeVals.length == 2) { String min = splitRangeVals[0]; diff --git a/ssb/src/main/java/com/google/cloud/StorageSharedBenchmarkingUtils.java b/ssb/src/main/java/com/google/cloud/StorageSharedBenchmarkingUtils.java index 1eee4cb5ad..376bd80962 100644 --- a/ssb/src/main/java/com/google/cloud/StorageSharedBenchmarkingUtils.java +++ b/ssb/src/main/java/com/google/cloud/StorageSharedBenchmarkingUtils.java @@ -7,11 +7,12 @@ * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * */ package com.google.cloud; @@ -25,7 +26,7 @@ class StorageSharedBenchmarkingUtils { public static int DEFAULT_NUMBER_OF_READS = 3; public static void cleanupObject(Storage storage, Blob created) { - storage.delete(created.getBlobId()); + storage.delete(created.getBlobId(), Storage.BlobSourceOption.generationMatch(created.getGeneration())); } public static double calculateThroughput(long size, Duration elapsedTime) { diff --git a/ssb/src/main/java/com/google/cloud/Workload1.java b/ssb/src/main/java/com/google/cloud/Workload1.java index 4f40de9561..257176d9cd 100644 --- a/ssb/src/main/java/com/google/cloud/Workload1.java +++ b/ssb/src/main/java/com/google/cloud/Workload1.java @@ -7,19 +7,16 @@ * * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * */ package com.google.cloud; -import static com.google.cloud.StorageSharedBenchmarkingUtils.DEFAULT_NUMBER_OF_READS; -import static com.google.cloud.StorageSharedBenchmarkingUtils.calculateThroughput; -import static com.google.cloud.StorageSharedBenchmarkingUtils.cleanupObject; - import com.google.cloud.storage.Blob; import com.google.cloud.storage.BlobInfo; import com.google.cloud.storage.Storage; @@ -58,11 +55,11 @@ public String call() throws Exception { System.out.println( generateCloudMonitoringResult( "WRITE", - calculateThroughput(created.getSize().longValue(), elapsedTimeUpload), + StorageSharedBenchmarkingUtils.calculateThroughput(created.getSize().longValue(), elapsedTimeUpload), created) .toString()); Path tempDir = Paths.get(System.getProperty("java.io.tmpdir")); - for (int i = 0; i <= DEFAULT_NUMBER_OF_READS; i++) { + for (int i = 0; i <= StorageSharedBenchmarkingUtils.DEFAULT_NUMBER_OF_READS; i++) { TmpFile dest = TmpFile.of(tempDir, "prefix", "bin"); startTime = clock.instant(); storage.downloadTo(created.getBlobId(), dest.getPath()); @@ -71,11 +68,11 @@ public String call() throws Exception { System.out.println( generateCloudMonitoringResult( "READ[" + i + "]", - calculateThroughput(created.getSize().longValue(), elapsedTimeDownload), + StorageSharedBenchmarkingUtils.calculateThroughput(created.getSize().longValue(), elapsedTimeDownload), created) .toString()); } - cleanupObject(storage, created); + StorageSharedBenchmarkingUtils.cleanupObject(storage, created); return "OK"; } From 76d9c36aaa20bea70b2c69e46879596a4230e475 Mon Sep 17 00:00:00 2001 From: Sydney Munro Date: Thu, 17 Aug 2023 06:25:12 -0700 Subject: [PATCH 09/12] PR comments REV2 --- google-cloud-storage/pom.xml | 6 +++++ .../benchmarking}/CloudMonitoringResult.java | 15 +++++------ .../StorageSharedBenchmarkingCli.java | 27 +++++++++++-------- .../StorageSharedBenchmarkingUtils.java | 19 +++++++------ .../storage/benchmarking}/Workload1.java | 21 ++++++++------- ssb/pom.xml | 7 ++--- 6 files changed, 53 insertions(+), 42 deletions(-) rename {ssb/src/main/java/com/google/cloud => google-cloud-storage/src/test/java/com/google/cloud/storage/benchmarking}/CloudMonitoringResult.java (94%) rename {ssb/src/main/java/com/google/cloud => google-cloud-storage/src/test/java/com/google/cloud/storage/benchmarking}/StorageSharedBenchmarkingCli.java (86%) rename {ssb/src/main/java/com/google/cloud => google-cloud-storage/src/test/java/com/google/cloud/storage/benchmarking}/StorageSharedBenchmarkingUtils.java (60%) rename {ssb/src/main/java/com/google/cloud => google-cloud-storage/src/test/java/com/google/cloud/storage/benchmarking}/Workload1.java (83%) diff --git a/google-cloud-storage/pom.xml b/google-cloud-storage/pom.xml index 3a3c9010af..1e1bfa2868 100644 --- a/google-cloud-storage/pom.xml +++ b/google-cloud-storage/pom.xml @@ -251,6 +251,12 @@ 1.7.4 test + + info.picocli + picocli + 4.7.0 + test + diff --git a/ssb/src/main/java/com/google/cloud/CloudMonitoringResult.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/benchmarking/CloudMonitoringResult.java similarity index 94% rename from ssb/src/main/java/com/google/cloud/CloudMonitoringResult.java rename to google-cloud-storage/src/test/java/com/google/cloud/storage/benchmarking/CloudMonitoringResult.java index e16c564b06..f9884d12e5 100644 --- a/ssb/src/main/java/com/google/cloud/CloudMonitoringResult.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/benchmarking/CloudMonitoringResult.java @@ -5,17 +5,16 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ -package com.google.cloud; +package com.google.cloud.storage.benchmarking; import com.google.common.base.MoreObjects; import java.util.Objects; diff --git a/ssb/src/main/java/com/google/cloud/StorageSharedBenchmarkingCli.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/benchmarking/StorageSharedBenchmarkingCli.java similarity index 86% rename from ssb/src/main/java/com/google/cloud/StorageSharedBenchmarkingCli.java rename to google-cloud-storage/src/test/java/com/google/cloud/storage/benchmarking/StorageSharedBenchmarkingCli.java index b7d19634e5..387d39348f 100644 --- a/ssb/src/main/java/com/google/cloud/StorageSharedBenchmarkingCli.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/benchmarking/StorageSharedBenchmarkingCli.java @@ -5,17 +5,16 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ -package com.google.cloud; +package com.google.cloud.storage.benchmarking; import com.google.api.core.ApiFuture; import com.google.api.core.ApiFutures; @@ -66,7 +65,8 @@ public final class StorageSharedBenchmarkingCli implements Runnable { @Option( names = "-object_size", defaultValue = "1048576..1048576", - description = "any positive integer, or an inclusive range such as min..max where min and max are positive integers") + description = + "any positive integer, or an inclusive range such as min..max where min and max are positive integers") String objectSize; @Option( @@ -88,8 +88,13 @@ public static void main(String[] args) { @Override public void run() { - // TODO: Make this a switch once we add more workloads - runWorkload1(); + switch (testType) { + case "w1r3": + runWorkload1(); + break; + default: + throw new IllegalStateException("Specify a workload to run"); + } } private void runWorkload1() { diff --git a/ssb/src/main/java/com/google/cloud/StorageSharedBenchmarkingUtils.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/benchmarking/StorageSharedBenchmarkingUtils.java similarity index 60% rename from ssb/src/main/java/com/google/cloud/StorageSharedBenchmarkingUtils.java rename to google-cloud-storage/src/test/java/com/google/cloud/storage/benchmarking/StorageSharedBenchmarkingUtils.java index 376bd80962..0407cf7015 100644 --- a/ssb/src/main/java/com/google/cloud/StorageSharedBenchmarkingUtils.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/benchmarking/StorageSharedBenchmarkingUtils.java @@ -5,17 +5,15 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - -package com.google.cloud; +package com.google.cloud.storage.benchmarking; import com.google.cloud.storage.Blob; import com.google.cloud.storage.Storage; @@ -26,7 +24,8 @@ class StorageSharedBenchmarkingUtils { public static int DEFAULT_NUMBER_OF_READS = 3; public static void cleanupObject(Storage storage, Blob created) { - storage.delete(created.getBlobId(), Storage.BlobSourceOption.generationMatch(created.getGeneration())); + storage.delete( + created.getBlobId(), Storage.BlobSourceOption.generationMatch(created.getGeneration())); } public static double calculateThroughput(long size, Duration elapsedTime) { diff --git a/ssb/src/main/java/com/google/cloud/Workload1.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/benchmarking/Workload1.java similarity index 83% rename from ssb/src/main/java/com/google/cloud/Workload1.java rename to google-cloud-storage/src/test/java/com/google/cloud/storage/benchmarking/Workload1.java index 257176d9cd..9d3de47292 100644 --- a/ssb/src/main/java/com/google/cloud/Workload1.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/benchmarking/Workload1.java @@ -5,17 +5,16 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ -package com.google.cloud; +package com.google.cloud.storage.benchmarking; import com.google.cloud.storage.Blob; import com.google.cloud.storage.BlobInfo; @@ -55,7 +54,8 @@ public String call() throws Exception { System.out.println( generateCloudMonitoringResult( "WRITE", - StorageSharedBenchmarkingUtils.calculateThroughput(created.getSize().longValue(), elapsedTimeUpload), + StorageSharedBenchmarkingUtils.calculateThroughput( + created.getSize().longValue(), elapsedTimeUpload), created) .toString()); Path tempDir = Paths.get(System.getProperty("java.io.tmpdir")); @@ -68,7 +68,8 @@ public String call() throws Exception { System.out.println( generateCloudMonitoringResult( "READ[" + i + "]", - StorageSharedBenchmarkingUtils.calculateThroughput(created.getSize().longValue(), elapsedTimeDownload), + StorageSharedBenchmarkingUtils.calculateThroughput( + created.getSize().longValue(), elapsedTimeDownload), created) .toString()); } diff --git a/ssb/pom.xml b/ssb/pom.xml index d6d2cde589..6dd1fae472 100644 --- a/ssb/pom.xml +++ b/ssb/pom.xml @@ -3,6 +3,9 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 + storage-shared-benchmarking + 0.0.1-SNAPSHOT + com.google.cloud.samples shared-configuration @@ -10,8 +13,6 @@ - ssb - 1.8 1.8 @@ -76,7 +77,7 @@ true dependency-jars/ - com.google.cloud.StorageSharedBenchmarkingCli + com.google.cloud.storage.benchmarking.StorageSharedBenchmarkingCli From f1066c4d7abdcfa33cc823ba16921d21cdc99c88 Mon Sep 17 00:00:00 2001 From: Sydney Munro Date: Thu, 17 Aug 2023 13:03:16 -0700 Subject: [PATCH 10/12] Moving CLI files back into ssb module --- google-cloud-storage/pom.xml | 6 ----- pom.xml | 2 +- {ssb => storage-shared-benchmarking}/pom.xml | 24 ++++--------------- .../benchmarking/CloudMonitoringResult.java | 0 .../StorageSharedBenchmarkingCli.java | 0 .../StorageSharedBenchmarkingUtils.java | 0 .../cloud/storage/benchmarking/Workload1.java | 0 7 files changed, 6 insertions(+), 26 deletions(-) rename {ssb => storage-shared-benchmarking}/pom.xml (85%) rename {google-cloud-storage/src/test => storage-shared-benchmarking/src/main}/java/com/google/cloud/storage/benchmarking/CloudMonitoringResult.java (100%) rename {google-cloud-storage/src/test => storage-shared-benchmarking/src/main}/java/com/google/cloud/storage/benchmarking/StorageSharedBenchmarkingCli.java (100%) rename {google-cloud-storage/src/test => storage-shared-benchmarking/src/main}/java/com/google/cloud/storage/benchmarking/StorageSharedBenchmarkingUtils.java (100%) rename {google-cloud-storage/src/test => storage-shared-benchmarking/src/main}/java/com/google/cloud/storage/benchmarking/Workload1.java (100%) diff --git a/google-cloud-storage/pom.xml b/google-cloud-storage/pom.xml index 1e1bfa2868..3a3c9010af 100644 --- a/google-cloud-storage/pom.xml +++ b/google-cloud-storage/pom.xml @@ -251,12 +251,6 @@ 1.7.4 test - - info.picocli - picocli - 4.7.0 - test - diff --git a/pom.xml b/pom.xml index 48f17b3271..29b1fe580b 100644 --- a/pom.xml +++ b/pom.xml @@ -199,7 +199,7 @@ proto-google-cloud-storage-v2 gapic-google-cloud-storage-v2 google-cloud-storage-bom - ssb + storage-shared-benchmarking diff --git a/ssb/pom.xml b/storage-shared-benchmarking/pom.xml similarity index 85% rename from ssb/pom.xml rename to storage-shared-benchmarking/pom.xml index 6dd1fae472..f8b20c8dcc 100644 --- a/ssb/pom.xml +++ b/storage-shared-benchmarking/pom.xml @@ -3,14 +3,14 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 + com.google.cloud + jar storage-shared-benchmarking 0.0.1-SNAPSHOT - - com.google.cloud.samples - shared-configuration - 1.2.0 - + com.google.cloud + google-cloud-storage-parent + 2.26.2-SNAPSHOT @@ -86,20 +86,6 @@ org.apache.maven.plugins maven-dependency-plugin 3.6.0 - - - copy-dependencies - package - - copy-dependencies - - - - ${project.build.directory}/dependency-jars/ - - - - org.apache.maven.plugins diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/benchmarking/CloudMonitoringResult.java b/storage-shared-benchmarking/src/main/java/com/google/cloud/storage/benchmarking/CloudMonitoringResult.java similarity index 100% rename from google-cloud-storage/src/test/java/com/google/cloud/storage/benchmarking/CloudMonitoringResult.java rename to storage-shared-benchmarking/src/main/java/com/google/cloud/storage/benchmarking/CloudMonitoringResult.java diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/benchmarking/StorageSharedBenchmarkingCli.java b/storage-shared-benchmarking/src/main/java/com/google/cloud/storage/benchmarking/StorageSharedBenchmarkingCli.java similarity index 100% rename from google-cloud-storage/src/test/java/com/google/cloud/storage/benchmarking/StorageSharedBenchmarkingCli.java rename to storage-shared-benchmarking/src/main/java/com/google/cloud/storage/benchmarking/StorageSharedBenchmarkingCli.java diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/benchmarking/StorageSharedBenchmarkingUtils.java b/storage-shared-benchmarking/src/main/java/com/google/cloud/storage/benchmarking/StorageSharedBenchmarkingUtils.java similarity index 100% rename from google-cloud-storage/src/test/java/com/google/cloud/storage/benchmarking/StorageSharedBenchmarkingUtils.java rename to storage-shared-benchmarking/src/main/java/com/google/cloud/storage/benchmarking/StorageSharedBenchmarkingUtils.java diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/benchmarking/Workload1.java b/storage-shared-benchmarking/src/main/java/com/google/cloud/storage/benchmarking/Workload1.java similarity index 100% rename from google-cloud-storage/src/test/java/com/google/cloud/storage/benchmarking/Workload1.java rename to storage-shared-benchmarking/src/main/java/com/google/cloud/storage/benchmarking/Workload1.java From 2378090e5559f6ad154d2e9142c7e9ecb087849d Mon Sep 17 00:00:00 2001 From: Sydney Munro Date: Tue, 22 Aug 2023 13:40:53 -0700 Subject: [PATCH 11/12] Switch to using maven-shade-plugin --- storage-shared-benchmarking/pom.xml | 59 ++++++++++++++++------------- 1 file changed, 32 insertions(+), 27 deletions(-) diff --git a/storage-shared-benchmarking/pom.xml b/storage-shared-benchmarking/pom.xml index f8b20c8dcc..3a4f575452 100644 --- a/storage-shared-benchmarking/pom.xml +++ b/storage-shared-benchmarking/pom.xml @@ -18,18 +18,6 @@ 1.8 UTF-8 - - - - - com.google.cloud - libraries-bom - 26.22.0 - pom - import - - - info.picocli @@ -39,7 +27,6 @@ com.google.cloud google-cloud-storage - 2.26.2-SNAPSHOT com.google.cloud @@ -70,27 +57,46 @@ org.apache.maven.plugins - maven-jar-plugin - 3.3.0 - - - - true - dependency-jars/ - com.google.cloud.storage.benchmarking.StorageSharedBenchmarkingCli - - - + maven-shade-plugin + + + package + + shade + + + ${uberjar.name} + + + com.google.cloud.storage.benchmarking.StorageSharedBenchmarkingCli + + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + org.apache.maven.plugins maven-dependency-plugin - 3.6.0 org.apache.maven.plugins maven-deploy-plugin - 3.1.1 true @@ -98,7 +104,6 @@ org.sonatype.plugins nexus-staging-maven-plugin - 1.6.13 true From 9b354d538e6b50b13e2873fd8deafb39c3afcea8 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Tue, 22 Aug 2023 20:43:45 +0000 Subject: [PATCH 12/12] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 93d9ee636c..ea22fb0c5a 100644 --- a/README.md +++ b/README.md @@ -57,13 +57,13 @@ implementation 'com.google.cloud:google-cloud-storage' If you are using Gradle without BOM, add this to your dependencies: ```Groovy -implementation 'com.google.cloud:google-cloud-storage:2.26.0' +implementation 'com.google.cloud:google-cloud-storage:2.26.1' ``` If you are using SBT, add this to your dependencies: ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-storage" % "2.26.0" +libraryDependencies += "com.google.cloud" % "google-cloud-storage" % "2.26.1" ``` @@ -428,7 +428,7 @@ Java is a registered trademark of Oracle and/or its affiliates. [kokoro-badge-link-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-storage/java11.html [stability-image]: https://img.shields.io/badge/stability-stable-green [maven-version-image]: https://img.shields.io/maven-central/v/com.google.cloud/google-cloud-storage.svg -[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-storage/2.26.0 +[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-storage/2.26.1 [authentication]: https://github.com/googleapis/google-cloud-java#authentication [auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes [predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles