From 23ddd882ec7c36865141e8907a9e7178b32e345b Mon Sep 17 00:00:00 2001 From: Sergey Borisenko Date: Mon, 25 Apr 2022 16:06:47 +0300 Subject: [PATCH 01/15] Refactoring events package. --- .../java/events/ImportUserEventsBigQuery.java | 33 ++-- .../main/java/events/ImportUserEventsGcs.java | 148 +++++++++--------- .../java/events/ImportUserEventsInline.java | 8 +- .../src/main/java/events/PurgeUserEvent.java | 8 +- .../src/main/java/events/RejoinUserEvent.java | 8 +- .../src/main/java/events/WriteUserEvent.java | 8 +- .../setup/EventsCreateBigQueryTable.java | 7 +- .../events/setup/EventsCreateGcsBucket.java | 40 +++-- .../events/setup/RemoveEventsResources.java | 72 +++++++++ .../events/setup/UpdateUserEventsJson.java | 54 +++++++ .../events/ImportUserEventsBigQueryTest.java | 40 +++-- .../java/events/ImportUserEventsGcsTest.java | 43 +++-- .../events/ImportUserEventsInlineTest.java | 3 + .../test/java/events/PurgeUserEventTest.java | 3 + .../test/java/events/RejoinUserEventTest.java | 3 + .../test/java/events/WriteUserEventTest.java | 3 + .../setup/EventsCreateBigQueryTableTest.java | 91 +++++++++++ .../setup/EventsCreateGcsBucketTest.java | 68 ++++++++ .../setup/RemoveEventsResourcesTest.java | 56 +++++++ .../setup/UpdateUserEventsJsonTest.java | 67 ++++++++ .../src/test/java/resources/user_events.json | 4 + .../resources/user_events_some_invalid.json | 4 + 22 files changed, 624 insertions(+), 147 deletions(-) create mode 100644 samples/interactive-tutorials/src/main/java/events/setup/RemoveEventsResources.java create mode 100644 samples/interactive-tutorials/src/main/java/events/setup/UpdateUserEventsJson.java create mode 100644 samples/interactive-tutorials/src/test/java/events/setup/EventsCreateBigQueryTableTest.java create mode 100644 samples/interactive-tutorials/src/test/java/events/setup/EventsCreateGcsBucketTest.java create mode 100644 samples/interactive-tutorials/src/test/java/events/setup/RemoveEventsResourcesTest.java create mode 100644 samples/interactive-tutorials/src/test/java/events/setup/UpdateUserEventsJsonTest.java create mode 100644 samples/interactive-tutorials/src/test/java/resources/user_events.json create mode 100644 samples/interactive-tutorials/src/test/java/resources/user_events_some_invalid.json diff --git a/samples/interactive-tutorials/src/main/java/events/ImportUserEventsBigQuery.java b/samples/interactive-tutorials/src/main/java/events/ImportUserEventsBigQuery.java index 40d24e9f..16c91225 100644 --- a/samples/interactive-tutorials/src/main/java/events/ImportUserEventsBigQuery.java +++ b/samples/interactive-tutorials/src/main/java/events/ImportUserEventsBigQuery.java @@ -22,6 +22,7 @@ package events; +import com.google.api.gax.rpc.NotFoundException; import com.google.cloud.ServiceOptions; import com.google.cloud.bigquery.BigQueryException; import com.google.cloud.retail.v2.BigQuerySource; @@ -33,19 +34,20 @@ import com.google.longrunning.Operation; import com.google.longrunning.OperationsClient; import java.io.IOException; +import java.util.concurrent.TimeUnit; public class ImportUserEventsBigQuery { public static void main(String[] args) throws IOException, InterruptedException { - // TODO(developer): Replace these variables before running the sample. String projectId = ServiceOptions.getDefaultProjectId(); String defaultCatalog = String.format("projects/%s/locations/global/catalogs/default_catalog", projectId); - // TO CHECK ERROR HANDLING PASTE THE INVALID CATALOG NAME HERE: defaultCatalog = - // "invalid_catalog_name" + // TO CHECK ERROR HANDLING PASTE THE INVALID CATALOG NAME HERE: + // defaultCatalog = "invalid_catalog_name" String datasetId = "user_events"; String tableId = "events"; - // TO CHECK ERROR HANDLING USE THE TABLE OF INVALID USER EVENTS: tableId = "events_some_invalid" + // TO CHECK ERROR HANDLING USE THE TABLE OF INVALID USER EVENTS: + // tableId = "events_some_invalid" importUserEventsFromBigQuery(projectId, defaultCatalog, datasetId, tableId); } @@ -53,6 +55,7 @@ public static void main(String[] args) throws IOException, InterruptedException public static void importUserEventsFromBigQuery( String projectId, String defaultCatalog, String datasetId, String tableId) throws IOException, InterruptedException { + try { String dataSchema = "user_event"; @@ -75,30 +78,32 @@ public static void importUserEventsFromBigQuery( System.out.printf("Import user events from BigQuery source request: %s%n", importRequest); - // Initialize client that will be used to send requests. This client only needs to be created - // once, and can be reused for multiple requests. After completing all of your requests, call - // the "close" method on the client to safely clean up any remaining background resources. + // Initialize client that will be used to send requests. This client only + // needs to be created once, and can be reused for multiple requests. After + // completing all of your requests, call the "close" method on the client to + // safely clean up any remaining background resources. try (UserEventServiceClient serviceClient = UserEventServiceClient.create()) { String operationName = serviceClient.importUserEventsCallable().call(importRequest).getName(); - System.out.printf("OperationName = %s\n", operationName); + System.out.printf("OperationName = %s%n", operationName); OperationsClient operationsClient = serviceClient.getOperationsClient(); Operation operation = operationsClient.getOperation(operationName); - while (!operation.getDone()) { + long assuredBreak = System.currentTimeMillis() + 60000; // 60 seconds delay + + while (!operation.getDone() || System.currentTimeMillis() < assuredBreak) { // Keep polling the operation periodically until the import task is done. - int awaitDuration = 30000; - Thread.sleep(awaitDuration); + TimeUnit.SECONDS.sleep(30); operation = operationsClient.getOperation(operationName); } if (operation.hasMetadata()) { ImportMetadata metadata = operation.getMetadata().unpack(ImportMetadata.class); System.out.printf( - "Number of successfully imported events: %s\n", metadata.getSuccessCount()); + "Number of successfully imported events: %s%n", metadata.getSuccessCount()); System.out.printf( - "Number of failures during the importing: %s\n", metadata.getFailureCount()); + "Number of failures during the importing: %s%n", metadata.getFailureCount()); } if (operation.hasResponse()) { @@ -109,6 +114,8 @@ public static void importUserEventsFromBigQuery( } } catch (BigQueryException e) { System.out.printf("Exception message: %s", e.getMessage()); + } catch (NotFoundException e) { + System.out.printf("Catalog name is not found.%n%s%n", e.getMessage()); } } } diff --git a/samples/interactive-tutorials/src/main/java/events/ImportUserEventsGcs.java b/samples/interactive-tutorials/src/main/java/events/ImportUserEventsGcs.java index 5e64eb1d..3d514d72 100644 --- a/samples/interactive-tutorials/src/main/java/events/ImportUserEventsGcs.java +++ b/samples/interactive-tutorials/src/main/java/events/ImportUserEventsGcs.java @@ -23,8 +23,8 @@ package events; import com.google.api.gax.rpc.InvalidArgumentException; +import com.google.api.gax.rpc.PermissionDeniedException; import com.google.cloud.ServiceOptions; -import com.google.cloud.bigquery.BigQueryException; import com.google.cloud.retail.v2.GcsSource; import com.google.cloud.retail.v2.ImportErrorsConfig; import com.google.cloud.retail.v2.ImportMetadata; @@ -34,91 +34,95 @@ import com.google.cloud.retail.v2.UserEventServiceClient; import com.google.longrunning.Operation; import com.google.longrunning.OperationsClient; -import events.setup.EventsCreateGcsBucket; import java.io.IOException; +import java.util.concurrent.TimeUnit; public class ImportUserEventsGcs { public static void main(String[] args) throws IOException, InterruptedException { - // TODO(developer): Replace these variables before running the sample. String projectId = ServiceOptions.getDefaultProjectId(); String defaultCatalog = String.format("projects/%s/locations/global/catalogs/default_catalog", projectId); - // TO CHECK ERROR HANDLING PASTE THE INVALID CATALOG NAME HERE: defaultCatalog = - // "invalid_catalog_name" - String gcsEventsObject = "user_events.json"; - // TO CHECK ERROR HANDLING USE THE JSON WITH INVALID USER EVENT: gcsEventsObject = - // "user_events_some_invalid.json" + String bucketName = System.getenv("EVENTS_BUCKET_NAME"); + String gcsUserEventsObject = "user_events.json"; + // TO CHECK ERROR HANDLING USE THE JSON WITH INVALID USER EVENT: + // gcsEventsObject = "user_events_some_invalid.json" - importUserEventsFromGcs(gcsEventsObject, defaultCatalog); + importUserEventsFromGcs(defaultCatalog, bucketName, gcsUserEventsObject); } - public static void importUserEventsFromGcs(String gcsEventsObject, String defaultCatalog) + public static void importUserEventsFromGcs( + String defaultCatalog, String bucketName, String gcsUserEventsObject) throws IOException, InterruptedException { - try { - String gcsBucket = String.format("gs://%s", EventsCreateGcsBucket.getBucketName()); - String gcsErrorsBucket = String.format("%s/error", gcsBucket); - - GcsSource gcsSource = - GcsSource.newBuilder() - .addInputUris(String.format("%s/%s", gcsBucket, gcsEventsObject)) - .build(); - - UserEventInputConfig inputConfig = - UserEventInputConfig.newBuilder().setGcsSource(gcsSource).build(); - - ImportErrorsConfig errorsConfig = - ImportErrorsConfig.newBuilder().setGcsPrefix(gcsErrorsBucket).build(); - - ImportUserEventsRequest importRequest = - ImportUserEventsRequest.newBuilder() - .setParent(defaultCatalog) - .setInputConfig(inputConfig) - .setErrorsConfig(errorsConfig) - .build(); - - System.out.printf("Import user events from google cloud source request: %s%n", importRequest); - - // Initialize client that will be used to send requests. This client only needs to be created - // once, and can be reused for multiple requests. After completing all of your requests, call - // the "close" method on the client to safely clean up any remaining background resources. - try (UserEventServiceClient serviceClient = UserEventServiceClient.create()) { - String operationName = - serviceClient.importUserEventsCallable().call(importRequest).getName(); - - System.out.printf("OperationName = %s\n", operationName); - - OperationsClient operationsClient = serviceClient.getOperationsClient(); - Operation operation = operationsClient.getOperation(operationName); - - while (!operation.getDone()) { - // Keep polling the operation periodically until the import task is done. - int awaitDuration = 30000; - Thread.sleep(awaitDuration); - operation = operationsClient.getOperation(operationName); - } - - if (operation.hasMetadata()) { - ImportMetadata metadata = operation.getMetadata().unpack(ImportMetadata.class); - System.out.printf( - "Number of successfully imported events: %s\n", metadata.getSuccessCount()); - System.out.printf( - "Number of failures during the importing: %s\n", metadata.getFailureCount()); - } - - if (operation.hasResponse()) { - ImportUserEventsResponse response = - operation.getResponse().unpack(ImportUserEventsResponse.class); - System.out.printf("Operation result: %s%n", response); - } - } catch (InvalidArgumentException e) { + String gcsBucket = String.format("gs://%s", bucketName); + String gcsErrorsBucket = String.format("%s/error", gcsBucket); + + GcsSource gcsSource = + GcsSource.newBuilder() + .addInputUris(String.format("%s/%s", gcsBucket, gcsUserEventsObject)) + .build(); + + UserEventInputConfig inputConfig = + UserEventInputConfig.newBuilder().setGcsSource(gcsSource).build(); + + System.out.println("GRS source: " + gcsSource.getInputUrisList()); + + ImportErrorsConfig errorsConfig = + ImportErrorsConfig.newBuilder().setGcsPrefix(gcsErrorsBucket).build(); + + ImportUserEventsRequest importRequest = + ImportUserEventsRequest.newBuilder() + .setParent(defaultCatalog) + .setInputConfig(inputConfig) + .setErrorsConfig(errorsConfig) + .build(); + System.out.printf("Import user events from google cloud source request: %s%n", importRequest); + + // Initialize client that will be used to send requests. This client only + // needs to be created once, and can be reused for multiple requests. After + // completing all of your requests, call the "close" method on the client to + // safely clean up any remaining background resources. + try (UserEventServiceClient serviceClient = UserEventServiceClient.create()) { + String operationName = serviceClient.importUserEventsCallable().call(importRequest).getName(); + + System.out.println("The operation was started."); + System.out.printf("OperationName = %s%n", operationName); + + OperationsClient operationsClient = serviceClient.getOperationsClient(); + Operation operation = operationsClient.getOperation(operationName); + + long assuredBreak = System.currentTimeMillis() + 60000; // 60 seconds delay + + while (!operation.getDone() || System.currentTimeMillis() < assuredBreak) { + System.out.println("Please wait till operation is done."); + TimeUnit.SECONDS.sleep(30); + operation = operationsClient.getOperation(operationName); + } + + if (operation.hasMetadata()) { + ImportMetadata metadata = operation.getMetadata().unpack(ImportMetadata.class); System.out.printf( - "Given GCS input path was not found. %n%s%n " - + "Please run CreateTestResources class to create resources.", - e.getMessage()); + "Number of successfully imported events: %s%n", metadata.getSuccessCount()); + System.out.printf( + "Number of failures during the importing: %s%n", metadata.getFailureCount()); + } else { + System.out.println("Metadata is empty."); + } + + if (operation.hasResponse()) { + ImportUserEventsResponse response = + operation.getResponse().unpack(ImportUserEventsResponse.class); + System.out.printf("Operation result: %s%n", response); + } else { + System.out.println("Operation result is empty."); } - } catch (BigQueryException e) { - System.out.printf("Exception message: %s", e.getMessage()); + } catch (InvalidArgumentException e) { + System.out.printf( + "%s%n'%s' file does not exist in the bucket. Please " + + "make sure you have followed the setting up instructions.", + e.getMessage(), gcsUserEventsObject); + } catch (PermissionDeniedException e) { + System.out.println(e.getMessage()); } } } diff --git a/samples/interactive-tutorials/src/main/java/events/ImportUserEventsInline.java b/samples/interactive-tutorials/src/main/java/events/ImportUserEventsInline.java index 2a5124ab..8ef7118c 100644 --- a/samples/interactive-tutorials/src/main/java/events/ImportUserEventsInline.java +++ b/samples/interactive-tutorials/src/main/java/events/ImportUserEventsInline.java @@ -45,7 +45,6 @@ public class ImportUserEventsInline { public static void main(String[] args) throws IOException, ExecutionException, InterruptedException { - // TODO(developer): Replace these variables before running the sample. String projectId = ServiceOptions.getDefaultProjectId(); String defaultCatalog = String.format("projects/%s/locations/global/catalogs/default_catalog", projectId); @@ -90,9 +89,10 @@ public static void importUserEventsFromInlineSource(String defaultCatalog) .build(); System.out.printf("Import user events from inline source request: %s%n", importRequest); - // Initialize client that will be used to send requests. This client only needs to be created - // once, and can be reused for multiple requests. After completing all of your requests, call - // the "close" method on the client to safely clean up any remaining background resources. + // Initialize client that will be used to send requests. This client only + // needs to be created once, and can be reused for multiple requests. After + // completing all of your requests, call the "close" method on the client to + // safely clean up any remaining background resources. try (UserEventServiceClient userEventServiceClient = UserEventServiceClient.create()) { OperationFuture importOperation = userEventServiceClient.importUserEventsAsync(importRequest); diff --git a/samples/interactive-tutorials/src/main/java/events/PurgeUserEvent.java b/samples/interactive-tutorials/src/main/java/events/PurgeUserEvent.java index e55c4a66..fb34fe20 100644 --- a/samples/interactive-tutorials/src/main/java/events/PurgeUserEvent.java +++ b/samples/interactive-tutorials/src/main/java/events/PurgeUserEvent.java @@ -38,7 +38,6 @@ public class PurgeUserEvent { public static void main(String[] args) throws IOException, ExecutionException, InterruptedException { - // TODO(developer): Replace these variables before running the sample. String projectId = ServiceOptions.getDefaultProjectId(); String defaultCatalog = String.format("projects/%s/locations/global/catalogs/default_catalog", projectId); @@ -52,9 +51,10 @@ public static void callPurgeUserEvents(String defaultCatalog, String visitorId) throws IOException, ExecutionException, InterruptedException { writeUserEvent(visitorId); - // Initialize client that will be used to send requests. This client only needs to be created - // once, and can be reused for multiple requests. After completing all of your requests, call - // the "close" method on the client to safely clean up any remaining background resources. + // Initialize client that will be used to send requests. This client only + // needs to be created once, and can be reused for multiple requests. After + // completing all of your requests, call the "close" method on the client to + // safely clean up any remaining background resources. try (UserEventServiceClient userEventServiceClient = UserEventServiceClient.create()) { PurgeUserEventsRequest purgeUserEventsRequest = PurgeUserEventsRequest.newBuilder() diff --git a/samples/interactive-tutorials/src/main/java/events/RejoinUserEvent.java b/samples/interactive-tutorials/src/main/java/events/RejoinUserEvent.java index 90284f6e..776c443a 100644 --- a/samples/interactive-tutorials/src/main/java/events/RejoinUserEvent.java +++ b/samples/interactive-tutorials/src/main/java/events/RejoinUserEvent.java @@ -40,7 +40,6 @@ public class RejoinUserEvent { public static void main(String[] args) throws IOException, ExecutionException, InterruptedException { - // TODO(developer): Replace these variables before running the sample. String projectId = ServiceOptions.getDefaultProjectId(); String defaultCatalog = String.format("projects/%s/locations/global/catalogs/default_catalog", projectId); @@ -54,9 +53,10 @@ public static void callRejoinUserEvents(String defaultCatalog, String visitorId) throws IOException, ExecutionException, InterruptedException { writeUserEvent(visitorId); - // Initialize client that will be used to send requests. This client only needs to be created - // once, and can be reused for multiple requests. After completing all of your requests, call - // the "close" method on the client to safely clean up any remaining background resources. + // Initialize client that will be used to send requests. This client only + // needs to be created once, and can be reused for multiple requests. After + // completing all of your requests, call the "close" method on the client to + // safely clean up any remaining background resources. try (UserEventServiceClient userEventServiceClient = UserEventServiceClient.create()) { RejoinUserEventsRequest rejoinUserEventsRequest = RejoinUserEventsRequest.newBuilder() diff --git a/samples/interactive-tutorials/src/main/java/events/WriteUserEvent.java b/samples/interactive-tutorials/src/main/java/events/WriteUserEvent.java index 0f78fcf5..18379938 100644 --- a/samples/interactive-tutorials/src/main/java/events/WriteUserEvent.java +++ b/samples/interactive-tutorials/src/main/java/events/WriteUserEvent.java @@ -38,7 +38,6 @@ public class WriteUserEvent { public static void main(String[] args) throws IOException, ExecutionException, InterruptedException { - // TODO(developer): Replace these variables before running the sample. String projectId = ServiceOptions.getDefaultProjectId(); String defaultCatalog = String.format("projects/%s/locations/global/catalogs/default_catalog", projectId); @@ -50,9 +49,10 @@ public static void main(String[] args) public static void writeUserEvent(String defaultCatalog, String visitorId) throws IOException, ExecutionException, InterruptedException { - // Initialize client that will be used to send requests. This client only needs to be created - // once, and can be reused for multiple requests. After completing all of your requests, call - // the "close" method on the client to safely clean up any remaining background resources. + // Initialize client that will be used to send requests. This client only + // needs to be created once, and can be reused for multiple requests. After + // completing all of your requests, call the "close" method on the client to + // safely clean up any remaining background resources. try (UserEventServiceClient userEventServiceClient = UserEventServiceClient.create()) { Timestamp timestamp = Timestamp.newBuilder().setSeconds(Instant.now().getEpochSecond()).build(); diff --git a/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateBigQueryTable.java b/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateBigQueryTable.java index 5b0cce2f..3bedc9f1 100644 --- a/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateBigQueryTable.java +++ b/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateBigQueryTable.java @@ -30,16 +30,15 @@ public class EventsCreateBigQueryTable { - public static void main(String... args) throws IOException { + public static void main(String[] args) throws IOException { String dataset = "user_events"; String validEventsTable = "events"; String invalidEventsTable = "events_some_invalid"; String eventsSchemaFilePath = "src/main/resources/events_schema.json"; String validEventsSourceFile = - String.format("gs://%s/user_events.json", EventsCreateGcsBucket.getBucketName()); + String.format("gs://%s/user_events.json", System.getenv("EVENTS_BUCKET_NAME")); String invalidEventsSourceFile = - String.format( - "gs://%s/user_events_some_invalid.json", EventsCreateGcsBucket.getBucketName()); + String.format("gs://%s/user_events_some_invalid.json", System.getenv("EVENTS_BUCKET_NAME")); BufferedReader bufferedReader = new BufferedReader(new FileReader(eventsSchemaFilePath)); String jsonToString = bufferedReader.lines().collect(Collectors.joining()); diff --git a/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateGcsBucket.java b/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateGcsBucket.java index 33a2351e..0134b1eb 100644 --- a/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateGcsBucket.java +++ b/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateGcsBucket.java @@ -25,34 +25,30 @@ import java.time.Instant; public class EventsCreateGcsBucket { + public static void main(String[] args) throws IOException { + String projectId = ServiceOptions.getDefaultProjectId(); + Timestamp currentDate = + Timestamp.newBuilder() + .setSeconds(Instant.now().getEpochSecond()) + .setNanos(Instant.now().getNano()) + .build(); + String eventsBucketName = String.format("%s_events_%s", projectId, currentDate.getSeconds()); + + createGcsBucketAndUploadData(eventsBucketName); + } - private static final String PROJECT_ID = ServiceOptions.getDefaultProjectId(); - - private static final Timestamp CURRENT_DATE = - Timestamp.newBuilder() - .setSeconds(Instant.now().getEpochSecond()) - .setNanos(Instant.now().getNano()) - .build(); - - private static final String BUCKET_NAME = - String.format("%s_events_%s", PROJECT_ID, CURRENT_DATE.getSeconds()); - - public static void main(String... args) throws IOException { - createBucket(BUCKET_NAME); - System.out.printf("Events gcs bucket %s was created.", BUCKET_NAME); + public static void createGcsBucketAndUploadData(String bucketName) throws IOException { + createBucket(bucketName); + System.out.printf("Events gcs bucket %s was created.%n", bucketName); - uploadObject(BUCKET_NAME, "user_events.json", "src/main/resources/user_events.json"); - System.out.printf("File 'user_events.json' was uploaded into bucket '%s'.", BUCKET_NAME); + uploadObject(bucketName, "user_events.json", "src/main/resources/user_events.json"); + System.out.printf("File 'user_events.json' was uploaded into bucket '%s'.%n", bucketName); uploadObject( - BUCKET_NAME, + bucketName, "user_events_some_invalid.json", "src/main/resources/user_events_some_invalid.json"); System.out.printf( - "File 'user_events_some_invalid.json' was uploaded into bucket '%s'.", BUCKET_NAME); - } - - public static String getBucketName() { - return BUCKET_NAME; + "File 'user_events_some_invalid.json' was uploaded into bucket '%s'.%n", bucketName); } } diff --git a/samples/interactive-tutorials/src/main/java/events/setup/RemoveEventsResources.java b/samples/interactive-tutorials/src/main/java/events/setup/RemoveEventsResources.java new file mode 100644 index 00000000..fb2c4f7e --- /dev/null +++ b/samples/interactive-tutorials/src/main/java/events/setup/RemoveEventsResources.java @@ -0,0 +1,72 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package events.setup; + +import static setup.SetupCleanup.deleteBucket; +import static setup.SetupCleanup.deleteDataset; + +import com.google.api.gax.rpc.PermissionDeniedException; +import com.google.cloud.ServiceOptions; +import com.google.cloud.retail.v2.DeleteProductRequest; +import com.google.cloud.retail.v2.ListProductsRequest; +import com.google.cloud.retail.v2.Product; +import com.google.cloud.retail.v2.ProductServiceClient; +import com.google.cloud.retail.v2.ProductServiceClient.ListProductsPagedResponse; +import java.io.IOException; + +public class RemoveEventsResources { + + public static void main(String[] args) throws IOException { + String projectId = ServiceOptions.getDefaultProjectId(); + String bucketName = System.getenv("EVENTS_BUCKET_NAME"); + String branchName = + String.format( + "projects/%s/locations/global/catalogs/default_catalog/branches/0", projectId); + + deleteBucket(bucketName); + deleteAllProducts(branchName); + deleteDataset(projectId, "user_events"); + } + + public static void deleteAllProducts(String branchName) throws IOException { + System.out.println("Deleting products in process, please wait..."); + + try (ProductServiceClient productServiceClient = ProductServiceClient.create()) { + ListProductsRequest listRequest = + ListProductsRequest.newBuilder().setParent(branchName).build(); + ListProductsPagedResponse products = productServiceClient.listProducts(listRequest); + + int deleteCount = 0; + + for (Product product : products.iterateAll()) { + DeleteProductRequest deleteRequest = + DeleteProductRequest.newBuilder().setName(product.getName()).build(); + + try { + productServiceClient.deleteProduct(deleteRequest); + deleteCount++; + } catch (PermissionDeniedException e) { + System.out.println( + "Ignore PermissionDenied in case the product does not exist " + + "at time of deletion."); + } + } + + System.out.printf("%s products were deleted from %s%n", deleteCount, branchName); + } + } +} diff --git a/samples/interactive-tutorials/src/main/java/events/setup/UpdateUserEventsJson.java b/samples/interactive-tutorials/src/main/java/events/setup/UpdateUserEventsJson.java new file mode 100644 index 00000000..0651fa02 --- /dev/null +++ b/samples/interactive-tutorials/src/main/java/events/setup/UpdateUserEventsJson.java @@ -0,0 +1,54 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package events.setup; + +import java.io.BufferedWriter; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.sql.Timestamp; +import java.text.SimpleDateFormat; +import java.time.Instant; +import java.time.temporal.ChronoUnit; + +public class UpdateUserEventsJson { + + public static void main(String[] args) throws IOException { + String filePath = "src/main/resources/user_events.json"; + String invalidFilePath = "src/main/resources/user_events_some_invalid.json"; + + updateEventsTimestamp(filePath); + updateEventsTimestamp(invalidFilePath); + } + + public static void updateEventsTimestamp(String jsonFile) throws IOException { + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + Timestamp yesterdayDate = Timestamp.from(Instant.now().minus(1, ChronoUnit.DAYS)); + + String json = new String(Files.readAllBytes(Paths.get(jsonFile))); + json = + json.replaceAll( + "(\"eventTime\"\\s*:\\s*\"(\\d{4}-\\d{2}-\\d{2}(T.*Z)?))", + "\"eventTime\":\"" + dateFormat.format(yesterdayDate) + ""); + + BufferedWriter writer = new BufferedWriter(new FileWriter(jsonFile)); + writer.write(json); + System.out.printf("User events file '%s' updated.%n", jsonFile); + writer.close(); + } +} diff --git a/samples/interactive-tutorials/src/test/java/events/ImportUserEventsBigQueryTest.java b/samples/interactive-tutorials/src/test/java/events/ImportUserEventsBigQueryTest.java index 2f465467..82afd5ae 100644 --- a/samples/interactive-tutorials/src/test/java/events/ImportUserEventsBigQueryTest.java +++ b/samples/interactive-tutorials/src/test/java/events/ImportUserEventsBigQueryTest.java @@ -19,7 +19,6 @@ import static com.google.common.truth.Truth.assertThat; import com.google.cloud.ServiceOptions; -import events.setup.EventsCreateBigQueryTable; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.PrintStream; @@ -27,7 +26,10 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +@RunWith(JUnit4.class) public class ImportUserEventsBigQueryTest { private ByteArrayOutputStream bout; @@ -35,28 +37,46 @@ public class ImportUserEventsBigQueryTest { @Before public void setUp() throws IOException, InterruptedException, ExecutionException { + bout = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(bout); + originalPrintStream = System.out; + System.setOut(out); + } + + @Test + public void testValidImportUserEventsBigQuery() throws IOException, InterruptedException { String projectId = ServiceOptions.getDefaultProjectId(); String defaultCatalog = String.format("projects/%s/locations/global/catalogs/default_catalog", projectId); String datasetId = "user_events"; String tableId = "events"; - bout = new ByteArrayOutputStream(); - PrintStream out = new PrintStream(bout); - originalPrintStream = System.out; - System.setOut(out); - EventsCreateBigQueryTable.main(); ImportUserEventsBigQuery.importUserEventsFromBigQuery( projectId, defaultCatalog, datasetId, tableId); + + String outputResult = bout.toString(); + + assertThat(outputResult).contains("Import user events from BigQuery source request"); + assertThat(outputResult).contains("table_id: \"events\""); + assertThat(outputResult).contains("Number of successfully imported events:"); + assertThat(outputResult).contains("Number of failures during the importing: 0"); } @Test - public void testImportUserEventsBigQuery() { + public void testInvalidImportUserEventsBigQuery() throws IOException, InterruptedException { + String projectId = ServiceOptions.getDefaultProjectId(); + String defaultCatalog = + String.format("projects/%s/locations/global/catalogs/invalid_catalog_name", projectId); + String datasetId = "user_events"; + String tableId = "events_some_invalid"; + + ImportUserEventsBigQuery.importUserEventsFromBigQuery( + projectId, defaultCatalog, datasetId, tableId); + String outputResult = bout.toString(); - assertThat(outputResult).contains("Import user events from BigQuery source request"); - assertThat(outputResult).contains("Number of successfully imported events"); - assertThat(outputResult).contains("Number of failures during the importing"); + assertThat(outputResult).contains("table_id: \"events_some_invalid\""); + assertThat(outputResult).contains("Catalog name is not found."); } @After diff --git a/samples/interactive-tutorials/src/test/java/events/ImportUserEventsGcsTest.java b/samples/interactive-tutorials/src/test/java/events/ImportUserEventsGcsTest.java index 78db5ff9..fd55f5db 100644 --- a/samples/interactive-tutorials/src/test/java/events/ImportUserEventsGcsTest.java +++ b/samples/interactive-tutorials/src/test/java/events/ImportUserEventsGcsTest.java @@ -17,16 +17,20 @@ package events; import static com.google.common.truth.Truth.assertThat; +import static events.ImportUserEventsGcs.importUserEventsFromGcs; +import static events.setup.EventsCreateGcsBucket.createGcsBucketAndUploadData; import com.google.cloud.ServiceOptions; -import events.setup.EventsCreateGcsBucket; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.PrintStream; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +@RunWith(JUnit4.class) public class ImportUserEventsGcsTest { private ByteArrayOutputStream bout; @@ -34,27 +38,46 @@ public class ImportUserEventsGcsTest { @Before public void setUp() throws IOException, InterruptedException { - EventsCreateGcsBucket.main(); + bout = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(bout); + originalPrintStream = System.out; + System.setOut(out); + } + @Test + public void testValidImportUserEventsGcs() throws IOException, InterruptedException { String projectId = ServiceOptions.getDefaultProjectId(); String defaultCatalog = String.format("projects/%s/locations/global/catalogs/default_catalog", projectId); + String bucketName = "events_tests_bucket"; String gcsEventsObject = "user_events.json"; - bout = new ByteArrayOutputStream(); - PrintStream out = new PrintStream(bout); - originalPrintStream = System.out; - System.setOut(out); - ImportUserEventsGcs.importUserEventsFromGcs(gcsEventsObject, defaultCatalog); + createGcsBucketAndUploadData(bucketName); + importUserEventsFromGcs(defaultCatalog, bucketName, gcsEventsObject); + + String outputResult = bout.toString(); + + assertThat(outputResult).contains("Import user events from google cloud source request"); + assertThat(outputResult).contains("Number of successfully imported events:"); + assertThat(outputResult).contains("Number of failures during the importing: 0"); } @Test - public void testImportUserEventsGcs() { + public void testInvalidImportUserEventsGcs() throws IOException, InterruptedException { + String projectId = ServiceOptions.getDefaultProjectId(); + String defaultCatalog = + String.format("projects/%s/locations/global/catalogs/default_catalog", projectId); + String bucketName = "invalid_events_tests_bucket"; + String gcsEventsObject = "user_events_some_invalid.json"; + + createGcsBucketAndUploadData(bucketName); + importUserEventsFromGcs(defaultCatalog, bucketName, gcsEventsObject); + String outputResult = bout.toString(); assertThat(outputResult).contains("Import user events from google cloud source request"); - assertThat(outputResult).contains("Number of successfully imported events"); - assertThat(outputResult).contains("Number of failures during the importing"); + assertThat(outputResult).contains("Number of successfully imported events:"); + assertThat(outputResult).contains("Number of failures during the importing:"); } @After diff --git a/samples/interactive-tutorials/src/test/java/events/ImportUserEventsInlineTest.java b/samples/interactive-tutorials/src/test/java/events/ImportUserEventsInlineTest.java index f6db8372..f182a5ad 100644 --- a/samples/interactive-tutorials/src/test/java/events/ImportUserEventsInlineTest.java +++ b/samples/interactive-tutorials/src/test/java/events/ImportUserEventsInlineTest.java @@ -26,7 +26,10 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +@RunWith(JUnit4.class) public class ImportUserEventsInlineTest { private ByteArrayOutputStream bout; diff --git a/samples/interactive-tutorials/src/test/java/events/PurgeUserEventTest.java b/samples/interactive-tutorials/src/test/java/events/PurgeUserEventTest.java index 7c4eb446..13e68970 100644 --- a/samples/interactive-tutorials/src/test/java/events/PurgeUserEventTest.java +++ b/samples/interactive-tutorials/src/test/java/events/PurgeUserEventTest.java @@ -27,7 +27,10 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +@RunWith(JUnit4.class) public class PurgeUserEventTest { private ByteArrayOutputStream bout; diff --git a/samples/interactive-tutorials/src/test/java/events/RejoinUserEventTest.java b/samples/interactive-tutorials/src/test/java/events/RejoinUserEventTest.java index fd478dff..446ab2d9 100644 --- a/samples/interactive-tutorials/src/test/java/events/RejoinUserEventTest.java +++ b/samples/interactive-tutorials/src/test/java/events/RejoinUserEventTest.java @@ -27,7 +27,10 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +@RunWith(JUnit4.class) public class RejoinUserEventTest { private ByteArrayOutputStream bout; diff --git a/samples/interactive-tutorials/src/test/java/events/WriteUserEventTest.java b/samples/interactive-tutorials/src/test/java/events/WriteUserEventTest.java index ebff18ca..46f5a652 100644 --- a/samples/interactive-tutorials/src/test/java/events/WriteUserEventTest.java +++ b/samples/interactive-tutorials/src/test/java/events/WriteUserEventTest.java @@ -27,7 +27,10 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +@RunWith(JUnit4.class) public class WriteUserEventTest { private ByteArrayOutputStream bout; diff --git a/samples/interactive-tutorials/src/test/java/events/setup/EventsCreateBigQueryTableTest.java b/samples/interactive-tutorials/src/test/java/events/setup/EventsCreateBigQueryTableTest.java new file mode 100644 index 00000000..33727613 --- /dev/null +++ b/samples/interactive-tutorials/src/test/java/events/setup/EventsCreateBigQueryTableTest.java @@ -0,0 +1,91 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package events.setup; + +import static com.google.common.truth.Truth.assertThat; +import static events.setup.EventsCreateGcsBucket.createGcsBucketAndUploadData; +import static setup.SetupCleanup.createBqDataset; +import static setup.SetupCleanup.createBqTable; +import static setup.SetupCleanup.getGson; +import static setup.SetupCleanup.uploadDataToBqTable; + +import com.google.cloud.bigquery.Field; +import com.google.cloud.bigquery.Schema; +import java.io.BufferedReader; +import java.io.ByteArrayOutputStream; +import java.io.FileReader; +import java.io.IOException; +import java.io.PrintStream; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class EventsCreateBigQueryTableTest { + + private ByteArrayOutputStream bout; + private PrintStream originalPrintStream; + + @Before + public void setUp() throws IOException, InterruptedException, ExecutionException { + String dataset = "user_events"; + String validEventsTable = "events"; + String invalidEventsTable = "events_some_invalid"; + String bucketName = "events_tests_bucket"; + String eventsSchemaFilePath = "src/main/resources/events_schema.json"; + String validEventsSourceFile = String.format("gs://%s/user_events.json", bucketName); + String invalidEventsSourceFile = + String.format("gs://%s/user_events_some_invalid.json", bucketName); + + BufferedReader bufferedReader = new BufferedReader(new FileReader(eventsSchemaFilePath)); + String jsonToString = bufferedReader.lines().collect(Collectors.joining()); + jsonToString = jsonToString.replace("\"fields\"", "\"subFields\""); + Field[] fields = getGson().fromJson(jsonToString, Field[].class); + Schema eventsSchema = Schema.of(fields); + + bout = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(bout); + originalPrintStream = System.out; + System.setOut(out); + + createGcsBucketAndUploadData(bucketName); + createBqDataset(dataset); + createBqTable(dataset, validEventsTable, eventsSchema); + uploadDataToBqTable(dataset, validEventsTable, validEventsSourceFile, eventsSchema); + createBqTable(dataset, invalidEventsTable, eventsSchema); + uploadDataToBqTable(dataset, invalidEventsTable, invalidEventsSourceFile, eventsSchema); + } + + @Test + public void testEventsCreateBigQueryTable() { + String outputResult = bout.toString(); + + assertThat(outputResult).contains("Json from GCS successfully loaded in a table 'events'."); + assertThat(outputResult) + .contains("Json from GCS successfully loaded in a table 'events_some_invalid'."); + } + + @After + public void tearDown() { + System.out.flush(); + System.setOut(originalPrintStream); + } +} diff --git a/samples/interactive-tutorials/src/test/java/events/setup/EventsCreateGcsBucketTest.java b/samples/interactive-tutorials/src/test/java/events/setup/EventsCreateGcsBucketTest.java new file mode 100644 index 00000000..10d31f1a --- /dev/null +++ b/samples/interactive-tutorials/src/test/java/events/setup/EventsCreateGcsBucketTest.java @@ -0,0 +1,68 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package events.setup; + +import static com.google.common.truth.Truth.assertThat; +import static events.setup.EventsCreateGcsBucket.createGcsBucketAndUploadData; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.util.concurrent.ExecutionException; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class EventsCreateGcsBucketTest { + + private ByteArrayOutputStream bout; + private PrintStream originalPrintStream; + + @Before + public void setUp() throws IOException, InterruptedException, ExecutionException { + + String bucketName = "events_tests_bucket"; + + bout = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(bout); + originalPrintStream = System.out; + System.setOut(out); + + createGcsBucketAndUploadData(bucketName); + } + + @Test + public void testEventsCreateGcsBucket() { + String outputResult = bout.toString(); + + assertThat(outputResult).contains("Events gcs bucket events_tests_bucket was created."); + assertThat(outputResult) + .contains("File 'user_events.json' was uploaded into bucket 'events_tests_bucket'."); + assertThat(outputResult) + .contains( + "File 'user_events_some_invalid.json' was uploaded into bucket 'events_tests_bucket'."); + } + + @After + public void tearDown() { + System.out.flush(); + System.setOut(originalPrintStream); + } +} diff --git a/samples/interactive-tutorials/src/test/java/events/setup/RemoveEventsResourcesTest.java b/samples/interactive-tutorials/src/test/java/events/setup/RemoveEventsResourcesTest.java new file mode 100644 index 00000000..8614483d --- /dev/null +++ b/samples/interactive-tutorials/src/test/java/events/setup/RemoveEventsResourcesTest.java @@ -0,0 +1,56 @@ +package events.setup; + +import static com.google.common.truth.Truth.assertThat; +import static events.setup.RemoveEventsResources.deleteAllProducts; +import static setup.SetupCleanup.deleteBucket; +import static setup.SetupCleanup.deleteDataset; + +import com.google.cloud.ServiceOptions; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.util.concurrent.ExecutionException; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class RemoveEventsResourcesTest { + + private ByteArrayOutputStream bout; + private PrintStream originalPrintStream; + + @Before + public void setUp() throws IOException, InterruptedException, ExecutionException { + String projectId = ServiceOptions.getDefaultProjectId(); + String bucketName = "events_tests_bucket"; + String branchName = + String.format( + "projects/%s/locations/global/catalogs/default_catalog/branches/0", projectId); + + bout = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(bout); + originalPrintStream = System.out; + System.setOut(out); + + deleteBucket(bucketName); + deleteAllProducts(branchName); + deleteDataset(projectId, "user_events"); + } + + @Test + public void testRemoveEventsResources() { + String outputResult = bout.toString(); + + assertThat(outputResult).contains("Deleting products in process, please wait..."); + assertThat(outputResult).contains("products were deleted from"); + } + + @After + public void tearDown() { + System.out.flush(); + System.setOut(originalPrintStream); + } +} diff --git a/samples/interactive-tutorials/src/test/java/events/setup/UpdateUserEventsJsonTest.java b/samples/interactive-tutorials/src/test/java/events/setup/UpdateUserEventsJsonTest.java new file mode 100644 index 00000000..21623889 --- /dev/null +++ b/samples/interactive-tutorials/src/test/java/events/setup/UpdateUserEventsJsonTest.java @@ -0,0 +1,67 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package events.setup; + +import static com.google.common.truth.Truth.assertThat; +import static events.setup.UpdateUserEventsJson.updateEventsTimestamp; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.util.concurrent.ExecutionException; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class UpdateUserEventsJsonTest { + + private ByteArrayOutputStream bout; + private PrintStream originalPrintStream; + + @Before + public void setUp() throws IOException, InterruptedException, ExecutionException { + String filePath = "src/test/java/resources/user_events.json"; + String invalidFilePath = "src/test/java/resources/user_events_some_invalid.json"; + bout = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(bout); + originalPrintStream = System.out; + System.setOut(out); + + updateEventsTimestamp(filePath); + updateEventsTimestamp(invalidFilePath); + } + + @Test + public void testUpdateUserEventsJson() { + String outputResult = bout.toString(); + + assertThat(outputResult) + .contains("User events file 'src/test/java/resources/user_events.json' updated."); + assertThat(outputResult) + .contains( + "User events file 'src/test/java/resources/user_events_some_invalid.json' updated."); + } + + @After + public void tearDown() { + System.out.flush(); + System.setOut(originalPrintStream); + } +} diff --git a/samples/interactive-tutorials/src/test/java/resources/user_events.json b/samples/interactive-tutorials/src/test/java/resources/user_events.json new file mode 100644 index 00000000..76d7c0df --- /dev/null +++ b/samples/interactive-tutorials/src/test/java/resources/user_events.json @@ -0,0 +1,4 @@ +{"eventType":"home-page-view","visitorId":"bjbs_group1_visitor1","eventTime":"2022-04-13T10:27:42+00:00"} +{"eventType":"search","visitorId":"bjbs_group1_visitor1","eventTime":"2022-04-13T10:27:42+00:00","searchQuery":"RockerJeans teenagers blue jeans"} +{"eventType":"search","visitorId":"bjbs_group1_visitor1","eventTime":"2022-04-13T10:27:42+00:00","searchQuery":"SocksUnlimited teenagers black socks"} +{"eventType":"detail-page-view","visitorId":"bjbs_group1_visitor1","eventTime":"2022-04-13T10:27:42+00:00","productDetails":{"product":{"id":"GGCOGAEC100616"},"quantity":3}} \ No newline at end of file diff --git a/samples/interactive-tutorials/src/test/java/resources/user_events_some_invalid.json b/samples/interactive-tutorials/src/test/java/resources/user_events_some_invalid.json new file mode 100644 index 00000000..27e00a0c --- /dev/null +++ b/samples/interactive-tutorials/src/test/java/resources/user_events_some_invalid.json @@ -0,0 +1,4 @@ +{"eventType":"home-page-view","visitorId":"bjbs_group1_visitor1","eventTime":"2022-04-13T10:27:42+00:00"} +{"eventType":"invalid","visitorId":"bjbs_group1_visitor1","eventTime":"2022-04-13T10:27:42+00:00","searchQuery":"RockerJeans teenagers blue jeans"} +{"eventType":"search","visitorId":"bjbs_group1_visitor1","eventTime":"2022-04-13T10:27:42+00:00","searchQuery":"SocksUnlimited teenagers black socks"} +{"eventType":"detail-page-view","visitorId":"bjbs_group1_visitor1","eventTime":"2022-04-13T10:27:42+00:00","productDetails":{"product":{"id":"GGCOGAEC100616"},"quantity":3}} From 9bade5d8c78dc7c81a03c95cae39c4e48f742add Mon Sep 17 00:00:00 2001 From: Sergey Borisenko Date: Mon, 25 Apr 2022 16:07:08 +0300 Subject: [PATCH 02/15] Remove init package. --- .../main/java/init/CreateTestResources.java | 110 ------------------ .../main/java/init/RemoveTestResources.java | 74 ------------ .../java/init/TEST_RESOURCES_SETUP_CLEANUP.md | 49 -------- 3 files changed, 233 deletions(-) delete mode 100644 samples/interactive-tutorials/src/main/java/init/CreateTestResources.java delete mode 100644 samples/interactive-tutorials/src/main/java/init/RemoveTestResources.java delete mode 100644 samples/interactive-tutorials/src/main/java/init/TEST_RESOURCES_SETUP_CLEANUP.md diff --git a/samples/interactive-tutorials/src/main/java/init/CreateTestResources.java b/samples/interactive-tutorials/src/main/java/init/CreateTestResources.java deleted file mode 100644 index 5a0f6cc6..00000000 --- a/samples/interactive-tutorials/src/main/java/init/CreateTestResources.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package init; - -import com.google.cloud.ServiceOptions; -import com.google.cloud.retail.v2.GcsSource; -import com.google.cloud.retail.v2.ImportErrorsConfig; -import com.google.cloud.retail.v2.ImportMetadata; -import com.google.cloud.retail.v2.ImportProductsRequest; -import com.google.cloud.retail.v2.ImportProductsRequest.ReconciliationMode; -import com.google.cloud.retail.v2.ImportProductsResponse; -import com.google.cloud.retail.v2.ProductInputConfig; -import com.google.cloud.retail.v2.ProductServiceClient; -import com.google.longrunning.Operation; -import com.google.longrunning.OperationsClient; -import events.setup.EventsCreateBigQueryTable; -import events.setup.EventsCreateGcsBucket; -import java.io.IOException; -import java.util.Collections; -import product.setup.ProductsCreateBigqueryTable; -import product.setup.ProductsCreateGcsBucket; - -public class CreateTestResources { - - public static void main(String... args) throws IOException, InterruptedException { - // TODO(developer): Replace these variables before running the sample. - String projectId = ServiceOptions.getDefaultProjectId(); - String bucketName = System.getenv("BUCKET_NAME"); - String gcsBucket = String.format("gs://%s", System.getenv("BUCKET_NAME")); - String gcsErrorBucket = String.format("%s/errors", gcsBucket); - String branchName = - String.format( - "projects/%s/locations/global/catalogs/default_catalog/branches/0", projectId); - - ProductsCreateGcsBucket.main(); - EventsCreateGcsBucket.main(); - importProductsFromGcs(bucketName, gcsErrorBucket, branchName); - ProductsCreateBigqueryTable.main(); - EventsCreateBigQueryTable.main(); - } - - public static void importProductsFromGcs( - String bucketName, String gcsErrorBucket, String branchName) - throws IOException, InterruptedException { - GcsSource gcsSource = - GcsSource.newBuilder() - .addAllInputUris( - Collections.singleton(String.format("gs://%s/%s", bucketName, "products.json"))) - .build(); - ProductInputConfig inputConfig = - ProductInputConfig.newBuilder().setGcsSource(gcsSource).build(); - System.out.println("GRS source: " + gcsSource.getInputUrisList()); - - ImportErrorsConfig errorsConfig = - ImportErrorsConfig.newBuilder().setGcsPrefix(gcsErrorBucket).build(); - ImportProductsRequest importRequest = - ImportProductsRequest.newBuilder() - .setParent(branchName) - .setReconciliationMode(ReconciliationMode.INCREMENTAL) - .setInputConfig(inputConfig) - .setErrorsConfig(errorsConfig) - .build(); - System.out.println("Import products from google cloud source request: " + importRequest); - - try (ProductServiceClient serviceClient = ProductServiceClient.create()) { - String operationName = serviceClient.importProductsCallable().call(importRequest).getName(); - System.out.printf("OperationName = %s\n", operationName); - - OperationsClient operationsClient = serviceClient.getOperationsClient(); - Operation operation = operationsClient.getOperation(operationName); - - while (!operation.getDone()) { - System.out.println("Please wait till operation is completed."); - // Keep polling the operation periodically until the import task is done. - Thread.sleep(30_000); - operation = operationsClient.getOperation(operationName); - } - - System.out.println("Import products operation is completed."); - - if (operation.hasMetadata()) { - ImportMetadata metadata = operation.getMetadata().unpack(ImportMetadata.class); - System.out.printf( - "Number of successfully imported products: %s\n", metadata.getSuccessCount()); - System.out.printf( - "Number of failures during the importing: %s\n", metadata.getFailureCount()); - } - - if (operation.hasResponse()) { - ImportProductsResponse response = - operation.getResponse().unpack(ImportProductsResponse.class); - System.out.printf("Operation result: %s", response); - } - } - } -} diff --git a/samples/interactive-tutorials/src/main/java/init/RemoveTestResources.java b/samples/interactive-tutorials/src/main/java/init/RemoveTestResources.java deleted file mode 100644 index c48ceff2..00000000 --- a/samples/interactive-tutorials/src/main/java/init/RemoveTestResources.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package init; - -import static setup.SetupCleanup.deleteBucket; -import static setup.SetupCleanup.deleteDataset; - -import com.google.api.gax.rpc.PermissionDeniedException; -import com.google.cloud.ServiceOptions; -import com.google.cloud.retail.v2.DeleteProductRequest; -import com.google.cloud.retail.v2.ListProductsRequest; -import com.google.cloud.retail.v2.Product; -import com.google.cloud.retail.v2.ProductServiceClient; -import com.google.cloud.retail.v2.ProductServiceClient.ListProductsPagedResponse; -import java.io.IOException; - -public class RemoveTestResources { - - public static void main(String... args) throws IOException { - // TODO(developer): Replace these variables before running the sample. - String projectId = ServiceOptions.getDefaultProjectId(); - String bucketName = System.getenv("BUCKET_NAME"); - String branchName = - String.format( - "projects/%s/locations/global/catalogs/default_catalog/branches/0", projectId); - - deleteBucket(bucketName); - deleteAllProducts(branchName); - deleteDataset(projectId, "products"); - deleteDataset(projectId, "user_events"); - } - - public static void deleteAllProducts(String branchName) throws IOException { - System.out.println("Deleting products in process, please wait..."); - - try (ProductServiceClient productServiceClient = ProductServiceClient.create()) { - ListProductsRequest listRequest = - ListProductsRequest.newBuilder().setParent(branchName).build(); - ListProductsPagedResponse products = productServiceClient.listProducts(listRequest); - - int deleteCount = 0; - - for (Product product : products.iterateAll()) { - DeleteProductRequest deleteRequest = - DeleteProductRequest.newBuilder().setName(product.getName()).build(); - - try { - productServiceClient.deleteProduct(deleteRequest); - deleteCount++; - } catch (PermissionDeniedException e) { - System.out.println( - "Ignore PermissionDenied in case the product does not exist " - + "at time of deletion."); - } - } - - System.out.printf("%s products were deleted from %s%n", deleteCount, branchName); - } - } -} diff --git a/samples/interactive-tutorials/src/main/java/init/TEST_RESOURCES_SETUP_CLEANUP.md b/samples/interactive-tutorials/src/main/java/init/TEST_RESOURCES_SETUP_CLEANUP.md deleted file mode 100644 index 0b9df6d1..00000000 --- a/samples/interactive-tutorials/src/main/java/init/TEST_RESOURCES_SETUP_CLEANUP.md +++ /dev/null @@ -1,49 +0,0 @@ -# How to set up/ tear down the test resources - -## Required environment variables - -To successfully import the catalog data for tests, the following environment variables should be -set: - -- PROJECT_ID -- PROJECT_NUMBER -- BUCKET_NAME - -The Secret Manager name is set in .kokoro/presubmit/common.cfg file, SECRET_MANAGER_KEYS variable. - -## Import catalog data - -There is a JSON file with valid products prepared in the `product` directory: -`resources/products.json`. - -Run the `CreateTestResources` to perform the following actions: - -- create the GCS bucket ; -- upload the product data from `resources/products.json` file to the bucket; -- import products to the default branch of the Retail catalog; -- upload the product data from `resources/user_events.json` file to the bucket; -- create a BigQuery dataset and table `products`; -- insert products from resources/products.json to the created products table; -- create a BigQuery dataset and table `events`; -- insert user events from resources/user_events.json to the created events table; - -``` -mvn compile exec:java -Dexec.mainClass="init.CreateTestResources" -``` - -In the result 316 products should be created in the test project catalog. - -## Remove catalog data - -Run the `RemoveTestResources` to perform the following actions: - -- remove all objects from the GCS bucket ; -- remove the bucket; -- delete all products from the Retail catalog; -- remove all objects from the GCS bucket ; -- remove dataset `products` along with tables; -- remove dataset `user_events` along with tables; - -``` -mvn compile exec:java -Dexec.mainClass="init.RemoveTestResources" -``` \ No newline at end of file From c32a7b05e8793acf33ba108ffd2e80754da46c98 Mon Sep 17 00:00:00 2001 From: Daria Firova Date: Fri, 15 Jul 2022 13:19:48 +0300 Subject: [PATCH 03/15] PR fixes: Fixed ImportUserEventsBigQuery. --- .../main/java/events/ImportUserEventsBigQuery.java | 9 +++------ .../java/events/ImportUserEventsBigQueryTest.java | 13 +++++++++++++ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/samples/interactive-tutorials/src/main/java/events/ImportUserEventsBigQuery.java b/samples/interactive-tutorials/src/main/java/events/ImportUserEventsBigQuery.java index 16c91225..62825cee 100644 --- a/samples/interactive-tutorials/src/main/java/events/ImportUserEventsBigQuery.java +++ b/samples/interactive-tutorials/src/main/java/events/ImportUserEventsBigQuery.java @@ -34,6 +34,7 @@ import com.google.longrunning.Operation; import com.google.longrunning.OperationsClient; import java.io.IOException; +import java.time.Instant; import java.util.concurrent.TimeUnit; public class ImportUserEventsBigQuery { @@ -42,12 +43,8 @@ public static void main(String[] args) throws IOException, InterruptedException String projectId = ServiceOptions.getDefaultProjectId(); String defaultCatalog = String.format("projects/%s/locations/global/catalogs/default_catalog", projectId); - // TO CHECK ERROR HANDLING PASTE THE INVALID CATALOG NAME HERE: - // defaultCatalog = "invalid_catalog_name" String datasetId = "user_events"; String tableId = "events"; - // TO CHECK ERROR HANDLING USE THE TABLE OF INVALID USER EVENTS: - // tableId = "events_some_invalid" importUserEventsFromBigQuery(projectId, defaultCatalog, datasetId, tableId); } @@ -90,9 +87,9 @@ public static void importUserEventsFromBigQuery( OperationsClient operationsClient = serviceClient.getOperationsClient(); Operation operation = operationsClient.getOperation(operationName); - long assuredBreak = System.currentTimeMillis() + 60000; // 60 seconds delay + Instant deadline = Instant.now().plusSeconds(60); - while (!operation.getDone() || System.currentTimeMillis() < assuredBreak) { + while (!operation.getDone() || Instant.now().isBefore(deadline)) { // Keep polling the operation periodically until the import task is done. TimeUnit.SECONDS.sleep(30); operation = operationsClient.getOperation(operationName); diff --git a/samples/interactive-tutorials/src/test/java/events/ImportUserEventsBigQueryTest.java b/samples/interactive-tutorials/src/test/java/events/ImportUserEventsBigQueryTest.java index 82afd5ae..8fb1c308 100644 --- a/samples/interactive-tutorials/src/test/java/events/ImportUserEventsBigQueryTest.java +++ b/samples/interactive-tutorials/src/test/java/events/ImportUserEventsBigQueryTest.java @@ -17,7 +17,9 @@ package events; import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; +import com.google.api.gax.rpc.InvalidArgumentException; import com.google.cloud.ServiceOptions; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -79,6 +81,17 @@ public void testInvalidImportUserEventsBigQuery() throws IOException, Interrupte assertThat(outputResult).contains("Catalog name is not found."); } + @Test + public void testInvalidDefaultCatalogBigQuery() { + String projectId = ServiceOptions.getDefaultProjectId(); + String defaultCatalog = "invalid_catalog_name"; + String datasetId = "user_events"; + String tableId = "events"; + + assertThrows(InvalidArgumentException.class, () -> ImportUserEventsBigQuery.importUserEventsFromBigQuery( + projectId, defaultCatalog, datasetId, tableId)); + } + @After public void tearDown() { System.out.flush(); From ff7ce0e18c7baec26eeb93095b9befeb30c15fec Mon Sep 17 00:00:00 2001 From: Daria Firova Date: Fri, 15 Jul 2022 15:46:43 +0300 Subject: [PATCH 04/15] PR fix: fixed delete of buckets and events. --- .../events/setup/RemoveEventsResources.java | 17 ++++++++++++----- .../setup/EventsCreateBigQueryTableTest.java | 12 +++++++----- .../events/setup/EventsCreateGcsBucketTest.java | 9 +++++++-- 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/samples/interactive-tutorials/src/main/java/events/setup/RemoveEventsResources.java b/samples/interactive-tutorials/src/main/java/events/setup/RemoveEventsResources.java index fb2c4f7e..f4e99742 100644 --- a/samples/interactive-tutorials/src/main/java/events/setup/RemoveEventsResources.java +++ b/samples/interactive-tutorials/src/main/java/events/setup/RemoveEventsResources.java @@ -21,10 +21,7 @@ import com.google.api.gax.rpc.PermissionDeniedException; import com.google.cloud.ServiceOptions; -import com.google.cloud.retail.v2.DeleteProductRequest; -import com.google.cloud.retail.v2.ListProductsRequest; -import com.google.cloud.retail.v2.Product; -import com.google.cloud.retail.v2.ProductServiceClient; +import com.google.cloud.retail.v2.*; import com.google.cloud.retail.v2.ProductServiceClient.ListProductsPagedResponse; import java.io.IOException; @@ -38,10 +35,20 @@ public static void main(String[] args) throws IOException { "projects/%s/locations/global/catalogs/default_catalog/branches/0", projectId); deleteBucket(bucketName); - deleteAllProducts(branchName); + deleteAllEvents(branchName); deleteDataset(projectId, "user_events"); } + public static void deleteAllEvents(String branchName) throws IOException { + System.out.println("Deleting events in process, please wait..."); + + try (UserEventServiceClient eventServiceClient = UserEventServiceClient.create()) { + PurgeUserEventsRequest purgeUserEventsRequest = PurgeUserEventsRequest.newBuilder().setParent(branchName).build(); + eventServiceClient.purgeUserEventsAsync(purgeUserEventsRequest); + System.out.printf("Events were deleted from %s%n", branchName); + } + } + public static void deleteAllProducts(String branchName) throws IOException { System.out.println("Deleting products in process, please wait..."); diff --git a/samples/interactive-tutorials/src/test/java/events/setup/EventsCreateBigQueryTableTest.java b/samples/interactive-tutorials/src/test/java/events/setup/EventsCreateBigQueryTableTest.java index 33727613..0fe3ae7d 100644 --- a/samples/interactive-tutorials/src/test/java/events/setup/EventsCreateBigQueryTableTest.java +++ b/samples/interactive-tutorials/src/test/java/events/setup/EventsCreateBigQueryTableTest.java @@ -18,10 +18,7 @@ import static com.google.common.truth.Truth.assertThat; import static events.setup.EventsCreateGcsBucket.createGcsBucketAndUploadData; -import static setup.SetupCleanup.createBqDataset; -import static setup.SetupCleanup.createBqTable; -import static setup.SetupCleanup.getGson; -import static setup.SetupCleanup.uploadDataToBqTable; +import static setup.SetupCleanup.*; import com.google.cloud.bigquery.Field; import com.google.cloud.bigquery.Schema; @@ -43,13 +40,13 @@ public class EventsCreateBigQueryTableTest { private ByteArrayOutputStream bout; private PrintStream originalPrintStream; + private static final String bucketName = "events_tests_bucket"; @Before public void setUp() throws IOException, InterruptedException, ExecutionException { String dataset = "user_events"; String validEventsTable = "events"; String invalidEventsTable = "events_some_invalid"; - String bucketName = "events_tests_bucket"; String eventsSchemaFilePath = "src/main/resources/events_schema.json"; String validEventsSourceFile = String.format("gs://%s/user_events.json", bucketName); String invalidEventsSourceFile = @@ -74,6 +71,11 @@ public void setUp() throws IOException, InterruptedException, ExecutionException uploadDataToBqTable(dataset, invalidEventsTable, invalidEventsSourceFile, eventsSchema); } + @After + public void cleanUp() { + deleteBucket(bucketName); + } + @Test public void testEventsCreateBigQueryTable() { String outputResult = bout.toString(); diff --git a/samples/interactive-tutorials/src/test/java/events/setup/EventsCreateGcsBucketTest.java b/samples/interactive-tutorials/src/test/java/events/setup/EventsCreateGcsBucketTest.java index 10d31f1a..b02ab017 100644 --- a/samples/interactive-tutorials/src/test/java/events/setup/EventsCreateGcsBucketTest.java +++ b/samples/interactive-tutorials/src/test/java/events/setup/EventsCreateGcsBucketTest.java @@ -18,6 +18,7 @@ import static com.google.common.truth.Truth.assertThat; import static events.setup.EventsCreateGcsBucket.createGcsBucketAndUploadData; +import static setup.SetupCleanup.deleteBucket; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -34,12 +35,11 @@ public class EventsCreateGcsBucketTest { private ByteArrayOutputStream bout; private PrintStream originalPrintStream; + private static final String bucketName = "events_tests_bucket"; @Before public void setUp() throws IOException, InterruptedException, ExecutionException { - String bucketName = "events_tests_bucket"; - bout = new ByteArrayOutputStream(); PrintStream out = new PrintStream(bout); originalPrintStream = System.out; @@ -48,6 +48,11 @@ public void setUp() throws IOException, InterruptedException, ExecutionException createGcsBucketAndUploadData(bucketName); } + @After + public void cleanUp() { + deleteBucket(bucketName); + } + @Test public void testEventsCreateGcsBucket() { String outputResult = bout.toString(); From a0b6e37be75a6c08eab46648338b713cfdf111a8 Mon Sep 17 00:00:00 2001 From: Daria Firova Date: Mon, 18 Jul 2022 09:13:50 +0300 Subject: [PATCH 05/15] PR fix: removed tests for setup/cleanup. --- .../setup/EventsCreateBigQueryTableTest.java | 93 ------------------- .../setup/EventsCreateGcsBucketTest.java | 73 --------------- .../setup/RemoveEventsResourcesTest.java | 56 ----------- .../setup/UpdateUserEventsJsonTest.java | 67 ------------- 4 files changed, 289 deletions(-) delete mode 100644 samples/interactive-tutorials/src/test/java/events/setup/EventsCreateBigQueryTableTest.java delete mode 100644 samples/interactive-tutorials/src/test/java/events/setup/EventsCreateGcsBucketTest.java delete mode 100644 samples/interactive-tutorials/src/test/java/events/setup/RemoveEventsResourcesTest.java delete mode 100644 samples/interactive-tutorials/src/test/java/events/setup/UpdateUserEventsJsonTest.java diff --git a/samples/interactive-tutorials/src/test/java/events/setup/EventsCreateBigQueryTableTest.java b/samples/interactive-tutorials/src/test/java/events/setup/EventsCreateBigQueryTableTest.java deleted file mode 100644 index 0fe3ae7d..00000000 --- a/samples/interactive-tutorials/src/test/java/events/setup/EventsCreateBigQueryTableTest.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package events.setup; - -import static com.google.common.truth.Truth.assertThat; -import static events.setup.EventsCreateGcsBucket.createGcsBucketAndUploadData; -import static setup.SetupCleanup.*; - -import com.google.cloud.bigquery.Field; -import com.google.cloud.bigquery.Schema; -import java.io.BufferedReader; -import java.io.ByteArrayOutputStream; -import java.io.FileReader; -import java.io.IOException; -import java.io.PrintStream; -import java.util.concurrent.ExecutionException; -import java.util.stream.Collectors; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -@RunWith(JUnit4.class) -public class EventsCreateBigQueryTableTest { - - private ByteArrayOutputStream bout; - private PrintStream originalPrintStream; - private static final String bucketName = "events_tests_bucket"; - - @Before - public void setUp() throws IOException, InterruptedException, ExecutionException { - String dataset = "user_events"; - String validEventsTable = "events"; - String invalidEventsTable = "events_some_invalid"; - String eventsSchemaFilePath = "src/main/resources/events_schema.json"; - String validEventsSourceFile = String.format("gs://%s/user_events.json", bucketName); - String invalidEventsSourceFile = - String.format("gs://%s/user_events_some_invalid.json", bucketName); - - BufferedReader bufferedReader = new BufferedReader(new FileReader(eventsSchemaFilePath)); - String jsonToString = bufferedReader.lines().collect(Collectors.joining()); - jsonToString = jsonToString.replace("\"fields\"", "\"subFields\""); - Field[] fields = getGson().fromJson(jsonToString, Field[].class); - Schema eventsSchema = Schema.of(fields); - - bout = new ByteArrayOutputStream(); - PrintStream out = new PrintStream(bout); - originalPrintStream = System.out; - System.setOut(out); - - createGcsBucketAndUploadData(bucketName); - createBqDataset(dataset); - createBqTable(dataset, validEventsTable, eventsSchema); - uploadDataToBqTable(dataset, validEventsTable, validEventsSourceFile, eventsSchema); - createBqTable(dataset, invalidEventsTable, eventsSchema); - uploadDataToBqTable(dataset, invalidEventsTable, invalidEventsSourceFile, eventsSchema); - } - - @After - public void cleanUp() { - deleteBucket(bucketName); - } - - @Test - public void testEventsCreateBigQueryTable() { - String outputResult = bout.toString(); - - assertThat(outputResult).contains("Json from GCS successfully loaded in a table 'events'."); - assertThat(outputResult) - .contains("Json from GCS successfully loaded in a table 'events_some_invalid'."); - } - - @After - public void tearDown() { - System.out.flush(); - System.setOut(originalPrintStream); - } -} diff --git a/samples/interactive-tutorials/src/test/java/events/setup/EventsCreateGcsBucketTest.java b/samples/interactive-tutorials/src/test/java/events/setup/EventsCreateGcsBucketTest.java deleted file mode 100644 index b02ab017..00000000 --- a/samples/interactive-tutorials/src/test/java/events/setup/EventsCreateGcsBucketTest.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package events.setup; - -import static com.google.common.truth.Truth.assertThat; -import static events.setup.EventsCreateGcsBucket.createGcsBucketAndUploadData; -import static setup.SetupCleanup.deleteBucket; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.PrintStream; -import java.util.concurrent.ExecutionException; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -@RunWith(JUnit4.class) -public class EventsCreateGcsBucketTest { - - private ByteArrayOutputStream bout; - private PrintStream originalPrintStream; - private static final String bucketName = "events_tests_bucket"; - - @Before - public void setUp() throws IOException, InterruptedException, ExecutionException { - - bout = new ByteArrayOutputStream(); - PrintStream out = new PrintStream(bout); - originalPrintStream = System.out; - System.setOut(out); - - createGcsBucketAndUploadData(bucketName); - } - - @After - public void cleanUp() { - deleteBucket(bucketName); - } - - @Test - public void testEventsCreateGcsBucket() { - String outputResult = bout.toString(); - - assertThat(outputResult).contains("Events gcs bucket events_tests_bucket was created."); - assertThat(outputResult) - .contains("File 'user_events.json' was uploaded into bucket 'events_tests_bucket'."); - assertThat(outputResult) - .contains( - "File 'user_events_some_invalid.json' was uploaded into bucket 'events_tests_bucket'."); - } - - @After - public void tearDown() { - System.out.flush(); - System.setOut(originalPrintStream); - } -} diff --git a/samples/interactive-tutorials/src/test/java/events/setup/RemoveEventsResourcesTest.java b/samples/interactive-tutorials/src/test/java/events/setup/RemoveEventsResourcesTest.java deleted file mode 100644 index 8614483d..00000000 --- a/samples/interactive-tutorials/src/test/java/events/setup/RemoveEventsResourcesTest.java +++ /dev/null @@ -1,56 +0,0 @@ -package events.setup; - -import static com.google.common.truth.Truth.assertThat; -import static events.setup.RemoveEventsResources.deleteAllProducts; -import static setup.SetupCleanup.deleteBucket; -import static setup.SetupCleanup.deleteDataset; - -import com.google.cloud.ServiceOptions; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.PrintStream; -import java.util.concurrent.ExecutionException; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -@RunWith(JUnit4.class) -public class RemoveEventsResourcesTest { - - private ByteArrayOutputStream bout; - private PrintStream originalPrintStream; - - @Before - public void setUp() throws IOException, InterruptedException, ExecutionException { - String projectId = ServiceOptions.getDefaultProjectId(); - String bucketName = "events_tests_bucket"; - String branchName = - String.format( - "projects/%s/locations/global/catalogs/default_catalog/branches/0", projectId); - - bout = new ByteArrayOutputStream(); - PrintStream out = new PrintStream(bout); - originalPrintStream = System.out; - System.setOut(out); - - deleteBucket(bucketName); - deleteAllProducts(branchName); - deleteDataset(projectId, "user_events"); - } - - @Test - public void testRemoveEventsResources() { - String outputResult = bout.toString(); - - assertThat(outputResult).contains("Deleting products in process, please wait..."); - assertThat(outputResult).contains("products were deleted from"); - } - - @After - public void tearDown() { - System.out.flush(); - System.setOut(originalPrintStream); - } -} diff --git a/samples/interactive-tutorials/src/test/java/events/setup/UpdateUserEventsJsonTest.java b/samples/interactive-tutorials/src/test/java/events/setup/UpdateUserEventsJsonTest.java deleted file mode 100644 index 21623889..00000000 --- a/samples/interactive-tutorials/src/test/java/events/setup/UpdateUserEventsJsonTest.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package events.setup; - -import static com.google.common.truth.Truth.assertThat; -import static events.setup.UpdateUserEventsJson.updateEventsTimestamp; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.PrintStream; -import java.util.concurrent.ExecutionException; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -@RunWith(JUnit4.class) -public class UpdateUserEventsJsonTest { - - private ByteArrayOutputStream bout; - private PrintStream originalPrintStream; - - @Before - public void setUp() throws IOException, InterruptedException, ExecutionException { - String filePath = "src/test/java/resources/user_events.json"; - String invalidFilePath = "src/test/java/resources/user_events_some_invalid.json"; - bout = new ByteArrayOutputStream(); - PrintStream out = new PrintStream(bout); - originalPrintStream = System.out; - System.setOut(out); - - updateEventsTimestamp(filePath); - updateEventsTimestamp(invalidFilePath); - } - - @Test - public void testUpdateUserEventsJson() { - String outputResult = bout.toString(); - - assertThat(outputResult) - .contains("User events file 'src/test/java/resources/user_events.json' updated."); - assertThat(outputResult) - .contains( - "User events file 'src/test/java/resources/user_events_some_invalid.json' updated."); - } - - @After - public void tearDown() { - System.out.flush(); - System.setOut(originalPrintStream); - } -} From ea8622e3fd5ea34844d3f5b2584a1ff7cf733a94 Mon Sep 17 00:00:00 2001 From: Daria Firova Date: Thu, 28 Jul 2022 08:23:01 +0300 Subject: [PATCH 06/15] PR fix: added comments and removed unnecessary code lines. --- .../src/main/java/events/ImportUserEventsBigQuery.java | 1 + .../src/main/java/events/ImportUserEventsGcs.java | 8 ++++---- .../src/main/java/events/setup/RemoveEventsResources.java | 4 ++++ .../src/main/java/events/setup/UpdateUserEventsJson.java | 3 +-- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/samples/interactive-tutorials/src/main/java/events/ImportUserEventsBigQuery.java b/samples/interactive-tutorials/src/main/java/events/ImportUserEventsBigQuery.java index 62825cee..ee1665f3 100644 --- a/samples/interactive-tutorials/src/main/java/events/ImportUserEventsBigQuery.java +++ b/samples/interactive-tutorials/src/main/java/events/ImportUserEventsBigQuery.java @@ -40,6 +40,7 @@ public class ImportUserEventsBigQuery { public static void main(String[] args) throws IOException, InterruptedException { + // TODO(developer): Replace these variables before running the sample. String projectId = ServiceOptions.getDefaultProjectId(); String defaultCatalog = String.format("projects/%s/locations/global/catalogs/default_catalog", projectId); diff --git a/samples/interactive-tutorials/src/main/java/events/ImportUserEventsGcs.java b/samples/interactive-tutorials/src/main/java/events/ImportUserEventsGcs.java index 3d514d72..5c240d70 100644 --- a/samples/interactive-tutorials/src/main/java/events/ImportUserEventsGcs.java +++ b/samples/interactive-tutorials/src/main/java/events/ImportUserEventsGcs.java @@ -35,18 +35,18 @@ import com.google.longrunning.Operation; import com.google.longrunning.OperationsClient; import java.io.IOException; +import java.time.Instant; import java.util.concurrent.TimeUnit; public class ImportUserEventsGcs { public static void main(String[] args) throws IOException, InterruptedException { + // TODO(developer): Replace these variables before running the sample. String projectId = ServiceOptions.getDefaultProjectId(); String defaultCatalog = String.format("projects/%s/locations/global/catalogs/default_catalog", projectId); String bucketName = System.getenv("EVENTS_BUCKET_NAME"); String gcsUserEventsObject = "user_events.json"; - // TO CHECK ERROR HANDLING USE THE JSON WITH INVALID USER EVENT: - // gcsEventsObject = "user_events_some_invalid.json" importUserEventsFromGcs(defaultCatalog, bucketName, gcsUserEventsObject); } @@ -91,9 +91,9 @@ public static void importUserEventsFromGcs( OperationsClient operationsClient = serviceClient.getOperationsClient(); Operation operation = operationsClient.getOperation(operationName); - long assuredBreak = System.currentTimeMillis() + 60000; // 60 seconds delay + Instant deadline = Instant.now().plusSeconds(60); - while (!operation.getDone() || System.currentTimeMillis() < assuredBreak) { + while (!operation.getDone() || Instant.now().isBefore(deadline)) { System.out.println("Please wait till operation is done."); TimeUnit.SECONDS.sleep(30); operation = operationsClient.getOperation(operationName); diff --git a/samples/interactive-tutorials/src/main/java/events/setup/RemoveEventsResources.java b/samples/interactive-tutorials/src/main/java/events/setup/RemoveEventsResources.java index f4e99742..e4e4b70f 100644 --- a/samples/interactive-tutorials/src/main/java/events/setup/RemoveEventsResources.java +++ b/samples/interactive-tutorials/src/main/java/events/setup/RemoveEventsResources.java @@ -42,6 +42,10 @@ public static void main(String[] args) throws IOException { public static void deleteAllEvents(String branchName) throws IOException { System.out.println("Deleting events in process, please wait..."); + // Initialize client that will be used to send requests. This client only + // needs to be created once, and can be reused for multiple requests. After + // completing all of your requests, call the "close" method on the client to + // safely clean up any remaining background resources. try (UserEventServiceClient eventServiceClient = UserEventServiceClient.create()) { PurgeUserEventsRequest purgeUserEventsRequest = PurgeUserEventsRequest.newBuilder().setParent(branchName).build(); eventServiceClient.purgeUserEventsAsync(purgeUserEventsRequest); diff --git a/samples/interactive-tutorials/src/main/java/events/setup/UpdateUserEventsJson.java b/samples/interactive-tutorials/src/main/java/events/setup/UpdateUserEventsJson.java index 0651fa02..e1fd5590 100644 --- a/samples/interactive-tutorials/src/main/java/events/setup/UpdateUserEventsJson.java +++ b/samples/interactive-tutorials/src/main/java/events/setup/UpdateUserEventsJson.java @@ -29,11 +29,10 @@ public class UpdateUserEventsJson { public static void main(String[] args) throws IOException { + // TODO(developer): Replace these variables before running the sample. String filePath = "src/main/resources/user_events.json"; - String invalidFilePath = "src/main/resources/user_events_some_invalid.json"; updateEventsTimestamp(filePath); - updateEventsTimestamp(invalidFilePath); } public static void updateEventsTimestamp(String jsonFile) throws IOException { From 2ae878df2f05ceb001e722cb0dc0c62c8861f870 Mon Sep 17 00:00:00 2001 From: dfirova <93149631+dfirova@users.noreply.github.com> Date: Fri, 29 Jul 2022 11:29:33 +0300 Subject: [PATCH 07/15] PR fix: typo Co-authored-by: Kurtis Van Gent <31518063+kurtisvg@users.noreply.github.com> --- .../src/main/java/events/ImportUserEventsGcs.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/interactive-tutorials/src/main/java/events/ImportUserEventsGcs.java b/samples/interactive-tutorials/src/main/java/events/ImportUserEventsGcs.java index 5c240d70..fb22daed 100644 --- a/samples/interactive-tutorials/src/main/java/events/ImportUserEventsGcs.java +++ b/samples/interactive-tutorials/src/main/java/events/ImportUserEventsGcs.java @@ -65,7 +65,7 @@ public static void importUserEventsFromGcs( UserEventInputConfig inputConfig = UserEventInputConfig.newBuilder().setGcsSource(gcsSource).build(); - System.out.println("GRS source: " + gcsSource.getInputUrisList()); + System.out.println("GCS source: " + gcsSource.getInputUrisList()); ImportErrorsConfig errorsConfig = ImportErrorsConfig.newBuilder().setGcsPrefix(gcsErrorsBucket).build(); From b9ed4441d70ea3596caaadadaea97cd9f188190c Mon Sep 17 00:00:00 2001 From: Daria Firova Date: Mon, 1 Aug 2022 09:10:49 +0300 Subject: [PATCH 08/15] PR fixes: changed code according to the guidelines. --- .../src/main/java/events/ImportUserEventsBigQuery.java | 2 +- .../src/main/java/events/ImportUserEventsGcs.java | 2 +- .../src/main/java/events/ImportUserEventsInline.java | 3 ++- .../src/main/java/events/PurgeUserEvent.java | 3 ++- .../src/main/java/events/RejoinUserEvent.java | 3 ++- .../src/main/java/events/WriteUserEvent.java | 3 ++- .../main/java/events/setup/EventsCreateGcsBucket.java | 10 ++-------- 7 files changed, 12 insertions(+), 14 deletions(-) diff --git a/samples/interactive-tutorials/src/main/java/events/ImportUserEventsBigQuery.java b/samples/interactive-tutorials/src/main/java/events/ImportUserEventsBigQuery.java index ee1665f3..1e0bbb23 100644 --- a/samples/interactive-tutorials/src/main/java/events/ImportUserEventsBigQuery.java +++ b/samples/interactive-tutorials/src/main/java/events/ImportUserEventsBigQuery.java @@ -41,7 +41,7 @@ public class ImportUserEventsBigQuery { public static void main(String[] args) throws IOException, InterruptedException { // TODO(developer): Replace these variables before running the sample. - String projectId = ServiceOptions.getDefaultProjectId(); + String projectId = "your-project-id"; String defaultCatalog = String.format("projects/%s/locations/global/catalogs/default_catalog", projectId); String datasetId = "user_events"; diff --git a/samples/interactive-tutorials/src/main/java/events/ImportUserEventsGcs.java b/samples/interactive-tutorials/src/main/java/events/ImportUserEventsGcs.java index fb22daed..b02c1333 100644 --- a/samples/interactive-tutorials/src/main/java/events/ImportUserEventsGcs.java +++ b/samples/interactive-tutorials/src/main/java/events/ImportUserEventsGcs.java @@ -42,7 +42,7 @@ public class ImportUserEventsGcs { public static void main(String[] args) throws IOException, InterruptedException { // TODO(developer): Replace these variables before running the sample. - String projectId = ServiceOptions.getDefaultProjectId(); + String projectId = "your-project-id"; String defaultCatalog = String.format("projects/%s/locations/global/catalogs/default_catalog", projectId); String bucketName = System.getenv("EVENTS_BUCKET_NAME"); diff --git a/samples/interactive-tutorials/src/main/java/events/ImportUserEventsInline.java b/samples/interactive-tutorials/src/main/java/events/ImportUserEventsInline.java index 8ef7118c..f1d2dfcb 100644 --- a/samples/interactive-tutorials/src/main/java/events/ImportUserEventsInline.java +++ b/samples/interactive-tutorials/src/main/java/events/ImportUserEventsInline.java @@ -45,7 +45,8 @@ public class ImportUserEventsInline { public static void main(String[] args) throws IOException, ExecutionException, InterruptedException { - String projectId = ServiceOptions.getDefaultProjectId(); + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; String defaultCatalog = String.format("projects/%s/locations/global/catalogs/default_catalog", projectId); diff --git a/samples/interactive-tutorials/src/main/java/events/PurgeUserEvent.java b/samples/interactive-tutorials/src/main/java/events/PurgeUserEvent.java index fb34fe20..7bd93c41 100644 --- a/samples/interactive-tutorials/src/main/java/events/PurgeUserEvent.java +++ b/samples/interactive-tutorials/src/main/java/events/PurgeUserEvent.java @@ -38,7 +38,8 @@ public class PurgeUserEvent { public static void main(String[] args) throws IOException, ExecutionException, InterruptedException { - String projectId = ServiceOptions.getDefaultProjectId(); + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; String defaultCatalog = String.format("projects/%s/locations/global/catalogs/default_catalog", projectId); // visitorId generated randomly. diff --git a/samples/interactive-tutorials/src/main/java/events/RejoinUserEvent.java b/samples/interactive-tutorials/src/main/java/events/RejoinUserEvent.java index 776c443a..a03806b3 100644 --- a/samples/interactive-tutorials/src/main/java/events/RejoinUserEvent.java +++ b/samples/interactive-tutorials/src/main/java/events/RejoinUserEvent.java @@ -40,7 +40,8 @@ public class RejoinUserEvent { public static void main(String[] args) throws IOException, ExecutionException, InterruptedException { - String projectId = ServiceOptions.getDefaultProjectId(); + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; String defaultCatalog = String.format("projects/%s/locations/global/catalogs/default_catalog", projectId); // visitorId generated randomly. diff --git a/samples/interactive-tutorials/src/main/java/events/WriteUserEvent.java b/samples/interactive-tutorials/src/main/java/events/WriteUserEvent.java index 18379938..4537cdd3 100644 --- a/samples/interactive-tutorials/src/main/java/events/WriteUserEvent.java +++ b/samples/interactive-tutorials/src/main/java/events/WriteUserEvent.java @@ -38,7 +38,8 @@ public class WriteUserEvent { public static void main(String[] args) throws IOException, ExecutionException, InterruptedException { - String projectId = ServiceOptions.getDefaultProjectId(); + // TODO(developer): Replace these variables before running the sample. + String projectId = "your-project-id"; String defaultCatalog = String.format("projects/%s/locations/global/catalogs/default_catalog", projectId); // visitorId generated randomly. diff --git a/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateGcsBucket.java b/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateGcsBucket.java index 0134b1eb..9a63072b 100644 --- a/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateGcsBucket.java +++ b/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateGcsBucket.java @@ -26,14 +26,8 @@ public class EventsCreateGcsBucket { public static void main(String[] args) throws IOException { - String projectId = ServiceOptions.getDefaultProjectId(); - Timestamp currentDate = - Timestamp.newBuilder() - .setSeconds(Instant.now().getEpochSecond()) - .setNanos(Instant.now().getNano()) - .build(); - String eventsBucketName = String.format("%s_events_%s", projectId, currentDate.getSeconds()); - + // TODO(developer): Replace these variables before running the sample. + String eventsBucketName = String.format("your-bucket-prefix_%s", Instant.now().getEpochSecond()); createGcsBucketAndUploadData(eventsBucketName); } From d31b3c7960317ad642361d9a5b5a2cec741debf9 Mon Sep 17 00:00:00 2001 From: Daria Firova Date: Mon, 1 Aug 2022 09:51:59 +0300 Subject: [PATCH 09/15] PR fixes: returned error handling steps from the tutorial. --- .../src/main/java/events/ImportUserEventsBigQuery.java | 4 ++++ .../src/main/java/events/ImportUserEventsGcs.java | 4 ++++ .../src/main/java/product/ImportProductsGcs.java | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/samples/interactive-tutorials/src/main/java/events/ImportUserEventsBigQuery.java b/samples/interactive-tutorials/src/main/java/events/ImportUserEventsBigQuery.java index 1e0bbb23..e6a1f3f6 100644 --- a/samples/interactive-tutorials/src/main/java/events/ImportUserEventsBigQuery.java +++ b/samples/interactive-tutorials/src/main/java/events/ImportUserEventsBigQuery.java @@ -44,8 +44,12 @@ public static void main(String[] args) throws IOException, InterruptedException String projectId = "your-project-id"; String defaultCatalog = String.format("projects/%s/locations/global/catalogs/default_catalog", projectId); + // TO CHECK ERROR HANDLING PASTE THE INVALID CATALOG NAME HERE: + // defaultCatalog = "invalid_catalog_name"; String datasetId = "user_events"; String tableId = "events"; + // TO CHECK ERROR HANDLING USE THE TABLE OF INVALID USER EVENTS: + // tableId = "events_some_invalid"; importUserEventsFromBigQuery(projectId, defaultCatalog, datasetId, tableId); } diff --git a/samples/interactive-tutorials/src/main/java/events/ImportUserEventsGcs.java b/samples/interactive-tutorials/src/main/java/events/ImportUserEventsGcs.java index b02c1333..a1fd128e 100644 --- a/samples/interactive-tutorials/src/main/java/events/ImportUserEventsGcs.java +++ b/samples/interactive-tutorials/src/main/java/events/ImportUserEventsGcs.java @@ -45,8 +45,12 @@ public static void main(String[] args) throws IOException, InterruptedException String projectId = "your-project-id"; String defaultCatalog = String.format("projects/%s/locations/global/catalogs/default_catalog", projectId); + // TO CHECK ERROR HANDLING PASTE THE INVALID CATALOG NAME HERE: + // defaultCatalog = "invalid_catalog_name"; String bucketName = System.getenv("EVENTS_BUCKET_NAME"); String gcsUserEventsObject = "user_events.json"; + // TO CHECK ERROR HANDLING USE THE JSON WITH INVALID USER EVENT: + // gcsUserEventsObject = "user_events_some_invalid.json"; importUserEventsFromGcs(defaultCatalog, bucketName, gcsUserEventsObject); } diff --git a/samples/interactive-tutorials/src/main/java/product/ImportProductsGcs.java b/samples/interactive-tutorials/src/main/java/product/ImportProductsGcs.java index f2ce4e18..19bc1794 100644 --- a/samples/interactive-tutorials/src/main/java/product/ImportProductsGcs.java +++ b/samples/interactive-tutorials/src/main/java/product/ImportProductsGcs.java @@ -48,7 +48,7 @@ public static void main(String[] args) throws IOException, InterruptedException String gcsErrorBucket = String.format("%s/errors", gcsBucket); String gscProductsObject = "products.json"; // TO CHECK ERROR HANDLING USE THE JSON WITH INVALID PRODUCT - // GCS_PRODUCTS_OBJECT = "products_some_invalid.json" + // gscProductsObject = "products_some_invalid.json"; ImportProductsRequest importGcsRequest = getImportProductsGcsRequest(gscProductsObject, gcsBucket, gcsErrorBucket, branchName); From 79072a5fb8ab175531c284edab36882e94d97820 Mon Sep 17 00:00:00 2001 From: Daria Firova Date: Tue, 9 Aug 2022 12:55:09 +0300 Subject: [PATCH 10/15] PR fix: removed test. --- .../java/events/ImportUserEventsGcsTest.java | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/samples/interactive-tutorials/src/test/java/events/ImportUserEventsGcsTest.java b/samples/interactive-tutorials/src/test/java/events/ImportUserEventsGcsTest.java index fd55f5db..a580da3a 100644 --- a/samples/interactive-tutorials/src/test/java/events/ImportUserEventsGcsTest.java +++ b/samples/interactive-tutorials/src/test/java/events/ImportUserEventsGcsTest.java @@ -62,24 +62,6 @@ public void testValidImportUserEventsGcs() throws IOException, InterruptedExcept assertThat(outputResult).contains("Number of failures during the importing: 0"); } - @Test - public void testInvalidImportUserEventsGcs() throws IOException, InterruptedException { - String projectId = ServiceOptions.getDefaultProjectId(); - String defaultCatalog = - String.format("projects/%s/locations/global/catalogs/default_catalog", projectId); - String bucketName = "invalid_events_tests_bucket"; - String gcsEventsObject = "user_events_some_invalid.json"; - - createGcsBucketAndUploadData(bucketName); - importUserEventsFromGcs(defaultCatalog, bucketName, gcsEventsObject); - - String outputResult = bout.toString(); - - assertThat(outputResult).contains("Import user events from google cloud source request"); - assertThat(outputResult).contains("Number of successfully imported events:"); - assertThat(outputResult).contains("Number of failures during the importing:"); - } - @After public void tearDown() { System.out.flush(); From bde0349fdd8d515a103912646808d7c9a90ccebb Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Wed, 10 Aug 2022 19:17:07 +0000 Subject: [PATCH 11/15] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- README.md | 8 ++++---- .../src/main/java/events/ImportUserEventsBigQuery.java | 1 - .../src/main/java/events/ImportUserEventsGcs.java | 1 - .../src/main/java/events/ImportUserEventsInline.java | 1 - .../src/main/java/events/PurgeUserEvent.java | 1 - .../src/main/java/events/RejoinUserEvent.java | 1 - .../src/main/java/events/WriteUserEvent.java | 1 - .../src/main/java/events/setup/EventsCreateGcsBucket.java | 5 ++--- .../src/main/java/events/setup/RemoveEventsResources.java | 3 ++- .../test/java/events/ImportUserEventsBigQueryTest.java | 7 +++++-- 10 files changed, 13 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index bb0be0e3..99d787ec 100644 --- a/README.md +++ b/README.md @@ -27,13 +27,13 @@ If you are using Maven, add this to your pom.xml file: If you are using Gradle without BOM, add this to your dependencies ```Groovy -implementation 'com.google.cloud:google-cloud-retail:2.2.4' +implementation 'com.google.cloud:google-cloud-retail:2.3.0' ``` If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-retail" % "2.2.4" +libraryDependencies += "com.google.cloud" % "google-cloud-retail" % "2.3.0" ``` ## Authentication @@ -85,8 +85,8 @@ Samples are in the [`samples/`](https://github.com/googleapis/java-retail/tree/m | Write User Event | [source code](https://github.com/googleapis/java-retail/blob/main/samples/interactive-tutorials/src/main/java/events/WriteUserEvent.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-retail&page=editor&open_in_editor=samples/interactive-tutorials/src/main/java/events/WriteUserEvent.java) | | Events Create Big Query Table | [source code](https://github.com/googleapis/java-retail/blob/main/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateBigQueryTable.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-retail&page=editor&open_in_editor=samples/interactive-tutorials/src/main/java/events/setup/EventsCreateBigQueryTable.java) | | Events Create Gcs Bucket | [source code](https://github.com/googleapis/java-retail/blob/main/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateGcsBucket.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-retail&page=editor&open_in_editor=samples/interactive-tutorials/src/main/java/events/setup/EventsCreateGcsBucket.java) | -| Create Test Resources | [source code](https://github.com/googleapis/java-retail/blob/main/samples/interactive-tutorials/src/main/java/init/CreateTestResources.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-retail&page=editor&open_in_editor=samples/interactive-tutorials/src/main/java/init/CreateTestResources.java) | -| Remove Test Resources | [source code](https://github.com/googleapis/java-retail/blob/main/samples/interactive-tutorials/src/main/java/init/RemoveTestResources.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-retail&page=editor&open_in_editor=samples/interactive-tutorials/src/main/java/init/RemoveTestResources.java) | +| Remove Events Resources | [source code](https://github.com/googleapis/java-retail/blob/main/samples/interactive-tutorials/src/main/java/events/setup/RemoveEventsResources.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-retail&page=editor&open_in_editor=samples/interactive-tutorials/src/main/java/events/setup/RemoveEventsResources.java) | +| Update User Events Json | [source code](https://github.com/googleapis/java-retail/blob/main/samples/interactive-tutorials/src/main/java/events/setup/UpdateUserEventsJson.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-retail&page=editor&open_in_editor=samples/interactive-tutorials/src/main/java/events/setup/UpdateUserEventsJson.java) | | Add Fulfillment Places | [source code](https://github.com/googleapis/java-retail/blob/main/samples/interactive-tutorials/src/main/java/product/AddFulfillmentPlaces.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-retail&page=editor&open_in_editor=samples/interactive-tutorials/src/main/java/product/AddFulfillmentPlaces.java) | | Create Product | [source code](https://github.com/googleapis/java-retail/blob/main/samples/interactive-tutorials/src/main/java/product/CreateProduct.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-retail&page=editor&open_in_editor=samples/interactive-tutorials/src/main/java/product/CreateProduct.java) | | Crud Product | [source code](https://github.com/googleapis/java-retail/blob/main/samples/interactive-tutorials/src/main/java/product/CrudProduct.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-retail&page=editor&open_in_editor=samples/interactive-tutorials/src/main/java/product/CrudProduct.java) | diff --git a/samples/interactive-tutorials/src/main/java/events/ImportUserEventsBigQuery.java b/samples/interactive-tutorials/src/main/java/events/ImportUserEventsBigQuery.java index e6a1f3f6..a415567f 100644 --- a/samples/interactive-tutorials/src/main/java/events/ImportUserEventsBigQuery.java +++ b/samples/interactive-tutorials/src/main/java/events/ImportUserEventsBigQuery.java @@ -23,7 +23,6 @@ package events; import com.google.api.gax.rpc.NotFoundException; -import com.google.cloud.ServiceOptions; import com.google.cloud.bigquery.BigQueryException; import com.google.cloud.retail.v2.BigQuerySource; import com.google.cloud.retail.v2.ImportMetadata; diff --git a/samples/interactive-tutorials/src/main/java/events/ImportUserEventsGcs.java b/samples/interactive-tutorials/src/main/java/events/ImportUserEventsGcs.java index a1fd128e..b53750b6 100644 --- a/samples/interactive-tutorials/src/main/java/events/ImportUserEventsGcs.java +++ b/samples/interactive-tutorials/src/main/java/events/ImportUserEventsGcs.java @@ -24,7 +24,6 @@ import com.google.api.gax.rpc.InvalidArgumentException; import com.google.api.gax.rpc.PermissionDeniedException; -import com.google.cloud.ServiceOptions; import com.google.cloud.retail.v2.GcsSource; import com.google.cloud.retail.v2.ImportErrorsConfig; import com.google.cloud.retail.v2.ImportMetadata; diff --git a/samples/interactive-tutorials/src/main/java/events/ImportUserEventsInline.java b/samples/interactive-tutorials/src/main/java/events/ImportUserEventsInline.java index f1d2dfcb..1f3630ad 100644 --- a/samples/interactive-tutorials/src/main/java/events/ImportUserEventsInline.java +++ b/samples/interactive-tutorials/src/main/java/events/ImportUserEventsInline.java @@ -23,7 +23,6 @@ package events; import com.google.api.gax.longrunning.OperationFuture; -import com.google.cloud.ServiceOptions; import com.google.cloud.bigquery.BigQueryException; import com.google.cloud.retail.v2.ImportMetadata; import com.google.cloud.retail.v2.ImportUserEventsRequest; diff --git a/samples/interactive-tutorials/src/main/java/events/PurgeUserEvent.java b/samples/interactive-tutorials/src/main/java/events/PurgeUserEvent.java index 7bd93c41..49060aee 100644 --- a/samples/interactive-tutorials/src/main/java/events/PurgeUserEvent.java +++ b/samples/interactive-tutorials/src/main/java/events/PurgeUserEvent.java @@ -25,7 +25,6 @@ import static setup.SetupCleanup.writeUserEvent; import com.google.api.gax.longrunning.OperationFuture; -import com.google.cloud.ServiceOptions; import com.google.cloud.retail.v2.PurgeMetadata; import com.google.cloud.retail.v2.PurgeUserEventsRequest; import com.google.cloud.retail.v2.PurgeUserEventsResponse; diff --git a/samples/interactive-tutorials/src/main/java/events/RejoinUserEvent.java b/samples/interactive-tutorials/src/main/java/events/RejoinUserEvent.java index a03806b3..0d29dcb9 100644 --- a/samples/interactive-tutorials/src/main/java/events/RejoinUserEvent.java +++ b/samples/interactive-tutorials/src/main/java/events/RejoinUserEvent.java @@ -26,7 +26,6 @@ import static setup.SetupCleanup.writeUserEvent; import com.google.api.gax.longrunning.OperationFuture; -import com.google.cloud.ServiceOptions; import com.google.cloud.retail.v2.RejoinUserEventsMetadata; import com.google.cloud.retail.v2.RejoinUserEventsRequest; import com.google.cloud.retail.v2.RejoinUserEventsRequest.UserEventRejoinScope; diff --git a/samples/interactive-tutorials/src/main/java/events/WriteUserEvent.java b/samples/interactive-tutorials/src/main/java/events/WriteUserEvent.java index 4537cdd3..f6f7d34a 100644 --- a/samples/interactive-tutorials/src/main/java/events/WriteUserEvent.java +++ b/samples/interactive-tutorials/src/main/java/events/WriteUserEvent.java @@ -24,7 +24,6 @@ import static setup.SetupCleanup.purgeUserEvent; -import com.google.cloud.ServiceOptions; import com.google.cloud.retail.v2.UserEvent; import com.google.cloud.retail.v2.UserEventServiceClient; import com.google.cloud.retail.v2.WriteUserEventRequest; diff --git a/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateGcsBucket.java b/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateGcsBucket.java index 9a63072b..1ec7bbcb 100644 --- a/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateGcsBucket.java +++ b/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateGcsBucket.java @@ -19,15 +19,14 @@ import static setup.SetupCleanup.createBucket; import static setup.SetupCleanup.uploadObject; -import com.google.cloud.ServiceOptions; -import com.google.protobuf.Timestamp; import java.io.IOException; import java.time.Instant; public class EventsCreateGcsBucket { public static void main(String[] args) throws IOException { // TODO(developer): Replace these variables before running the sample. - String eventsBucketName = String.format("your-bucket-prefix_%s", Instant.now().getEpochSecond()); + String eventsBucketName = + String.format("your-bucket-prefix_%s", Instant.now().getEpochSecond()); createGcsBucketAndUploadData(eventsBucketName); } diff --git a/samples/interactive-tutorials/src/main/java/events/setup/RemoveEventsResources.java b/samples/interactive-tutorials/src/main/java/events/setup/RemoveEventsResources.java index e4e4b70f..05b19e33 100644 --- a/samples/interactive-tutorials/src/main/java/events/setup/RemoveEventsResources.java +++ b/samples/interactive-tutorials/src/main/java/events/setup/RemoveEventsResources.java @@ -47,7 +47,8 @@ public static void deleteAllEvents(String branchName) throws IOException { // completing all of your requests, call the "close" method on the client to // safely clean up any remaining background resources. try (UserEventServiceClient eventServiceClient = UserEventServiceClient.create()) { - PurgeUserEventsRequest purgeUserEventsRequest = PurgeUserEventsRequest.newBuilder().setParent(branchName).build(); + PurgeUserEventsRequest purgeUserEventsRequest = + PurgeUserEventsRequest.newBuilder().setParent(branchName).build(); eventServiceClient.purgeUserEventsAsync(purgeUserEventsRequest); System.out.printf("Events were deleted from %s%n", branchName); } diff --git a/samples/interactive-tutorials/src/test/java/events/ImportUserEventsBigQueryTest.java b/samples/interactive-tutorials/src/test/java/events/ImportUserEventsBigQueryTest.java index 8fb1c308..a7f9a7a6 100644 --- a/samples/interactive-tutorials/src/test/java/events/ImportUserEventsBigQueryTest.java +++ b/samples/interactive-tutorials/src/test/java/events/ImportUserEventsBigQueryTest.java @@ -88,8 +88,11 @@ public void testInvalidDefaultCatalogBigQuery() { String datasetId = "user_events"; String tableId = "events"; - assertThrows(InvalidArgumentException.class, () -> ImportUserEventsBigQuery.importUserEventsFromBigQuery( - projectId, defaultCatalog, datasetId, tableId)); + assertThrows( + InvalidArgumentException.class, + () -> + ImportUserEventsBigQuery.importUserEventsFromBigQuery( + projectId, defaultCatalog, datasetId, tableId)); } @After From 325e6dc29fec7d62f821c8c83d4cecb31ca99703 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Thu, 11 Aug 2022 14:46:33 +0000 Subject: [PATCH 12/15] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7513e046..7ef426a7 100644 --- a/README.md +++ b/README.md @@ -27,13 +27,13 @@ If you are using Maven, add this to your pom.xml file: If you are using Gradle without BOM, add this to your dependencies: ```Groovy -implementation 'com.google.cloud:google-cloud-retail:2.3.0' +implementation 'com.google.cloud:google-cloud-retail:2.4.0' ``` If you are using SBT, add this to your dependencies: ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-retail" % "2.3.0" +libraryDependencies += "com.google.cloud" % "google-cloud-retail" % "2.4.0" ``` ## Authentication From e37c0acec7ee49c00619abe5d726079b96b41367 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Fri, 12 Aug 2022 21:26:44 +0000 Subject: [PATCH 13/15] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index d2b18b02..7ef426a7 100644 --- a/README.md +++ b/README.md @@ -85,9 +85,8 @@ Samples are in the [`samples/`](https://github.com/googleapis/java-retail/tree/m | Write User Event | [source code](https://github.com/googleapis/java-retail/blob/main/samples/interactive-tutorials/src/main/java/events/WriteUserEvent.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-retail&page=editor&open_in_editor=samples/interactive-tutorials/src/main/java/events/WriteUserEvent.java) | | Events Create Big Query Table | [source code](https://github.com/googleapis/java-retail/blob/main/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateBigQueryTable.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-retail&page=editor&open_in_editor=samples/interactive-tutorials/src/main/java/events/setup/EventsCreateBigQueryTable.java) | | Events Create Gcs Bucket | [source code](https://github.com/googleapis/java-retail/blob/main/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateGcsBucket.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-retail&page=editor&open_in_editor=samples/interactive-tutorials/src/main/java/events/setup/EventsCreateGcsBucket.java) | +| Remove Events Resources | [source code](https://github.com/googleapis/java-retail/blob/main/samples/interactive-tutorials/src/main/java/events/setup/RemoveEventsResources.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-retail&page=editor&open_in_editor=samples/interactive-tutorials/src/main/java/events/setup/RemoveEventsResources.java) | | Update User Events Json | [source code](https://github.com/googleapis/java-retail/blob/main/samples/interactive-tutorials/src/main/java/events/setup/UpdateUserEventsJson.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-retail&page=editor&open_in_editor=samples/interactive-tutorials/src/main/java/events/setup/UpdateUserEventsJson.java) | -| Create Test Resources | [source code](https://github.com/googleapis/java-retail/blob/main/samples/interactive-tutorials/src/main/java/init/CreateTestResources.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-retail&page=editor&open_in_editor=samples/interactive-tutorials/src/main/java/init/CreateTestResources.java) | -| Remove Test Resources | [source code](https://github.com/googleapis/java-retail/blob/main/samples/interactive-tutorials/src/main/java/init/RemoveTestResources.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-retail&page=editor&open_in_editor=samples/interactive-tutorials/src/main/java/init/RemoveTestResources.java) | | Add Fulfillment Places | [source code](https://github.com/googleapis/java-retail/blob/main/samples/interactive-tutorials/src/main/java/product/AddFulfillmentPlaces.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-retail&page=editor&open_in_editor=samples/interactive-tutorials/src/main/java/product/AddFulfillmentPlaces.java) | | Create Product | [source code](https://github.com/googleapis/java-retail/blob/main/samples/interactive-tutorials/src/main/java/product/CreateProduct.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-retail&page=editor&open_in_editor=samples/interactive-tutorials/src/main/java/product/CreateProduct.java) | | Crud Product | [source code](https://github.com/googleapis/java-retail/blob/main/samples/interactive-tutorials/src/main/java/product/CrudProduct.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-retail&page=editor&open_in_editor=samples/interactive-tutorials/src/main/java/product/CrudProduct.java) | From 4aaa3df75ed5280c4584398c4958dc06aaf02b1f Mon Sep 17 00:00:00 2001 From: Daria Firova Date: Thu, 1 Sep 2022 09:10:38 +0300 Subject: [PATCH 14/15] PR fix: returned test to previous state. --- .../events/setup/EventsCreateGcsBucket.java | 42 ++++++++++++------- .../java/events/ImportUserEventsGcsTest.java | 23 +++++----- 2 files changed, 39 insertions(+), 26 deletions(-) diff --git a/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateGcsBucket.java b/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateGcsBucket.java index 1ec7bbcb..d4a17377 100644 --- a/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateGcsBucket.java +++ b/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateGcsBucket.java @@ -16,6 +16,9 @@ package events.setup; +import com.google.cloud.ServiceOptions; +import com.google.protobuf.Timestamp; + import static setup.SetupCleanup.createBucket; import static setup.SetupCleanup.uploadObject; @@ -23,25 +26,34 @@ import java.time.Instant; public class EventsCreateGcsBucket { - public static void main(String[] args) throws IOException { - // TODO(developer): Replace these variables before running the sample. - String eventsBucketName = - String.format("your-bucket-prefix_%s", Instant.now().getEpochSecond()); - createGcsBucketAndUploadData(eventsBucketName); - } - public static void createGcsBucketAndUploadData(String bucketName) throws IOException { - createBucket(bucketName); - System.out.printf("Events gcs bucket %s was created.%n", bucketName); + private static final String PROJECT_ID = ServiceOptions.getDefaultProjectId(); + + private static final Timestamp CURRENT_DATE = + Timestamp.newBuilder() + .setSeconds(Instant.now().getEpochSecond()) + .setNanos(Instant.now().getNano()) + .build(); + + private static final String BUCKET_NAME = + String.format("%s_events_%s", PROJECT_ID, CURRENT_DATE.getSeconds()); - uploadObject(bucketName, "user_events.json", "src/main/resources/user_events.json"); - System.out.printf("File 'user_events.json' was uploaded into bucket '%s'.%n", bucketName); + public static void main(String... args) throws IOException { + createBucket(BUCKET_NAME); + System.out.printf("Events gcs bucket %s was created.", BUCKET_NAME); + + uploadObject(BUCKET_NAME, "user_events.json", "src/main/resources/user_events.json"); + System.out.printf("File 'user_events.json' was uploaded into bucket '%s'.", BUCKET_NAME); uploadObject( - bucketName, - "user_events_some_invalid.json", - "src/main/resources/user_events_some_invalid.json"); + BUCKET_NAME, + "user_events_some_invalid.json", + "src/main/resources/user_events_some_invalid.json"); System.out.printf( - "File 'user_events_some_invalid.json' was uploaded into bucket '%s'.%n", bucketName); + "File 'user_events_some_invalid.json' was uploaded into bucket '%s'.", BUCKET_NAME); + } + + public static String getBucketName() { + return BUCKET_NAME; } } diff --git a/samples/interactive-tutorials/src/test/java/events/ImportUserEventsGcsTest.java b/samples/interactive-tutorials/src/test/java/events/ImportUserEventsGcsTest.java index a580da3a..fe340d50 100644 --- a/samples/interactive-tutorials/src/test/java/events/ImportUserEventsGcsTest.java +++ b/samples/interactive-tutorials/src/test/java/events/ImportUserEventsGcsTest.java @@ -18,12 +18,13 @@ import static com.google.common.truth.Truth.assertThat; import static events.ImportUserEventsGcs.importUserEventsFromGcs; -import static events.setup.EventsCreateGcsBucket.createGcsBucketAndUploadData; import com.google.cloud.ServiceOptions; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.PrintStream; + +import events.setup.EventsCreateGcsBucket; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -38,23 +39,23 @@ public class ImportUserEventsGcsTest { @Before public void setUp() throws IOException, InterruptedException { - bout = new ByteArrayOutputStream(); - PrintStream out = new PrintStream(bout); - originalPrintStream = System.out; - System.setOut(out); - } + EventsCreateGcsBucket.main(); - @Test - public void testValidImportUserEventsGcs() throws IOException, InterruptedException { String projectId = ServiceOptions.getDefaultProjectId(); String defaultCatalog = - String.format("projects/%s/locations/global/catalogs/default_catalog", projectId); - String bucketName = "events_tests_bucket"; + String.format("projects/%s/locations/global/catalogs/default_catalog", projectId); + String bucketName = EventsCreateGcsBucket.getBucketName(); String gcsEventsObject = "user_events.json"; + bout = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(bout); + originalPrintStream = System.out; + System.setOut(out); - createGcsBucketAndUploadData(bucketName); importUserEventsFromGcs(defaultCatalog, bucketName, gcsEventsObject); + } + @Test + public void testValidImportUserEventsGcs() { String outputResult = bout.toString(); assertThat(outputResult).contains("Import user events from google cloud source request"); From 69ac31f72a032605dbe1a21fb31cda04d07bbb66 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Mon, 12 Sep 2022 16:17:49 +0000 Subject: [PATCH 15/15] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- .../events/setup/EventsCreateGcsBucket.java | 23 +++++++++---------- .../java/events/ImportUserEventsGcsTest.java | 5 ++-- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateGcsBucket.java b/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateGcsBucket.java index d4a17377..33a2351e 100644 --- a/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateGcsBucket.java +++ b/samples/interactive-tutorials/src/main/java/events/setup/EventsCreateGcsBucket.java @@ -16,12 +16,11 @@ package events.setup; -import com.google.cloud.ServiceOptions; -import com.google.protobuf.Timestamp; - import static setup.SetupCleanup.createBucket; import static setup.SetupCleanup.uploadObject; +import com.google.cloud.ServiceOptions; +import com.google.protobuf.Timestamp; import java.io.IOException; import java.time.Instant; @@ -30,13 +29,13 @@ public class EventsCreateGcsBucket { private static final String PROJECT_ID = ServiceOptions.getDefaultProjectId(); private static final Timestamp CURRENT_DATE = - Timestamp.newBuilder() - .setSeconds(Instant.now().getEpochSecond()) - .setNanos(Instant.now().getNano()) - .build(); + Timestamp.newBuilder() + .setSeconds(Instant.now().getEpochSecond()) + .setNanos(Instant.now().getNano()) + .build(); private static final String BUCKET_NAME = - String.format("%s_events_%s", PROJECT_ID, CURRENT_DATE.getSeconds()); + String.format("%s_events_%s", PROJECT_ID, CURRENT_DATE.getSeconds()); public static void main(String... args) throws IOException { createBucket(BUCKET_NAME); @@ -46,11 +45,11 @@ public static void main(String... args) throws IOException { System.out.printf("File 'user_events.json' was uploaded into bucket '%s'.", BUCKET_NAME); uploadObject( - BUCKET_NAME, - "user_events_some_invalid.json", - "src/main/resources/user_events_some_invalid.json"); + BUCKET_NAME, + "user_events_some_invalid.json", + "src/main/resources/user_events_some_invalid.json"); System.out.printf( - "File 'user_events_some_invalid.json' was uploaded into bucket '%s'.", BUCKET_NAME); + "File 'user_events_some_invalid.json' was uploaded into bucket '%s'.", BUCKET_NAME); } public static String getBucketName() { diff --git a/samples/interactive-tutorials/src/test/java/events/ImportUserEventsGcsTest.java b/samples/interactive-tutorials/src/test/java/events/ImportUserEventsGcsTest.java index fe340d50..8ee3488f 100644 --- a/samples/interactive-tutorials/src/test/java/events/ImportUserEventsGcsTest.java +++ b/samples/interactive-tutorials/src/test/java/events/ImportUserEventsGcsTest.java @@ -20,11 +20,10 @@ import static events.ImportUserEventsGcs.importUserEventsFromGcs; import com.google.cloud.ServiceOptions; +import events.setup.EventsCreateGcsBucket; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.PrintStream; - -import events.setup.EventsCreateGcsBucket; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -43,7 +42,7 @@ public void setUp() throws IOException, InterruptedException { String projectId = ServiceOptions.getDefaultProjectId(); String defaultCatalog = - String.format("projects/%s/locations/global/catalogs/default_catalog", projectId); + String.format("projects/%s/locations/global/catalogs/default_catalog", projectId); String bucketName = EventsCreateGcsBucket.getBucketName(); String gcsEventsObject = "user_events.json"; bout = new ByteArrayOutputStream();