From d28532901b5caf07b92481442956640f3277cd42 Mon Sep 17 00:00:00 2001 From: nnegrey Date: Thu, 9 Jan 2020 11:29:50 -0700 Subject: [PATCH 1/2] translate: add v3 translate text samples with glossary and model --- translate/cloud-client/pom.xml | 4 + ...atchTranslateTextWithGlossaryAndModel.java | 113 ++++++++++++++ .../TranslateTextWithGlossaryAndModel.java | 93 +++++++++++ ...ranslateTextWithGlossaryAndModelTests.java | 145 ++++++++++++++++++ ...ranslateTextWithGlossaryAndModelTests.java | 107 +++++++++++++ 5 files changed, 462 insertions(+) create mode 100644 translate/cloud-client/src/main/java/com/example/translate/BatchTranslateTextWithGlossaryAndModel.java create mode 100644 translate/cloud-client/src/main/java/com/example/translate/TranslateTextWithGlossaryAndModel.java create mode 100644 translate/cloud-client/src/test/java/com/example/translate/BatchTranslateTextWithGlossaryAndModelTests.java create mode 100644 translate/cloud-client/src/test/java/com/example/translate/TranslateTextWithGlossaryAndModelTests.java diff --git a/translate/cloud-client/pom.xml b/translate/cloud-client/pom.xml index 7e689cacae1..7f3e05c0152 100644 --- a/translate/cloud-client/pom.xml +++ b/translate/cloud-client/pom.xml @@ -56,6 +56,10 @@ See https://github.com/GoogleCloudPlatform/cloud-opensource-java/wiki/The-Google google-cloud-translate + + com.google.cloud + google-cloud-storage + diff --git a/translate/cloud-client/src/main/java/com/example/translate/BatchTranslateTextWithGlossaryAndModel.java b/translate/cloud-client/src/main/java/com/example/translate/BatchTranslateTextWithGlossaryAndModel.java new file mode 100644 index 00000000000..897d4807d3a --- /dev/null +++ b/translate/cloud-client/src/main/java/com/example/translate/BatchTranslateTextWithGlossaryAndModel.java @@ -0,0 +1,113 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.translate; + +// [START translate_v3_batch_translate_text_with_glossary_and_model] +import com.google.api.gax.longrunning.OperationFuture; +import com.google.cloud.translate.v3.BatchTranslateMetadata; +import com.google.cloud.translate.v3.BatchTranslateResponse; +import com.google.cloud.translate.v3.BatchTranslateTextRequest; +import com.google.cloud.translate.v3.GcsDestination; +import com.google.cloud.translate.v3.GcsSource; +import com.google.cloud.translate.v3.GlossaryName; +import com.google.cloud.translate.v3.InputConfig; +import com.google.cloud.translate.v3.LocationName; +import com.google.cloud.translate.v3.OutputConfig; +import com.google.cloud.translate.v3.TranslateTextGlossaryConfig; +import com.google.cloud.translate.v3.TranslationServiceClient; + +import java.io.IOException; +import java.util.concurrent.ExecutionException; + +public class BatchTranslateTextWithGlossaryAndModel { + + public static void batchTranslateTextWithGlossaryAndModel() + throws InterruptedException, ExecutionException, IOException { + // TODO(developer): Replace these variables before running the sample. + String projectId = "YOUR-PROJECT-ID"; + // Supported Languages: https://cloud.google.com/translate/docs/languages + String sourceLanguage = "your-source-language"; + String targetLanguage = "your-target-language"; + String inputUri = "gs://your-gcs-bucket/path/to/input/file.txt"; + String outputUri = "gs://your-gcs-bucket/path/to/results/"; + String glossaryId = "your-glossary-display-name"; + String modelId = "YOUR-MODEL-ID"; + batchTranslateTextWithGlossaryAndModel( + projectId, sourceLanguage, targetLanguage, inputUri, outputUri, glossaryId, modelId); + } + + // Batch translate text with Model and Glossary + public static void batchTranslateTextWithGlossaryAndModel( + String projectId, + String sourceLanguage, + String targetLanguage, + String inputUri, + String outputUri, + String glossaryId, + String modelId) + throws IOException, ExecutionException, InterruptedException { + + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (TranslationServiceClient client = TranslationServiceClient.create()) { + // Supported Locations: `global`, [glossary location], or [model location] + // Glossaries must be hosted in `us-central1` + // Custom Models must use the same location as your model. (us-central1) + String location = "us-central1"; + LocationName parent = LocationName.of(projectId, location); + + GcsSource gcsSource = GcsSource.newBuilder().setInputUri(inputUri).build(); + // Supported Mime Types: https://cloud.google.com/translate/docs/supported-formats + InputConfig inputConfig = + InputConfig.newBuilder().setGcsSource(gcsSource).setMimeType("text/plain").build(); + + GcsDestination gcsDestination = + GcsDestination.newBuilder().setOutputUriPrefix(outputUri).build(); + OutputConfig outputConfig = + OutputConfig.newBuilder().setGcsDestination(gcsDestination).build(); + + GlossaryName glossaryName = GlossaryName.of(projectId, location, glossaryId); + TranslateTextGlossaryConfig glossaryConfig = + TranslateTextGlossaryConfig.newBuilder().setGlossary(glossaryName.toString()).build(); + + String modelPath = + String.format("projects/%s/locations/%s/models/%s", projectId, location, modelId); + + BatchTranslateTextRequest request = + BatchTranslateTextRequest.newBuilder() + .setParent(parent.toString()) + .setSourceLanguageCode(sourceLanguage) + .addTargetLanguageCodes(targetLanguage) + .addInputConfigs(inputConfig) + .setOutputConfig(outputConfig) + .putGlossaries(targetLanguage, glossaryConfig) + .putModels(targetLanguage, modelPath) + .build(); + + OperationFuture future = + client.batchTranslateTextAsync(request); + + System.out.println("Waiting for operation to complete..."); + BatchTranslateResponse response = future.get(); + // Display the translation for each input text provided + System.out.printf("Total Characters: %s\n", response.getTotalCharacters()); + System.out.printf("Translated Characters: %s\n", response.getTranslatedCharacters()); + } + } +} +// [END translate_v3_batch_translate_text_with_glossary_and_model] diff --git a/translate/cloud-client/src/main/java/com/example/translate/TranslateTextWithGlossaryAndModel.java b/translate/cloud-client/src/main/java/com/example/translate/TranslateTextWithGlossaryAndModel.java new file mode 100644 index 00000000000..473d3042eba --- /dev/null +++ b/translate/cloud-client/src/main/java/com/example/translate/TranslateTextWithGlossaryAndModel.java @@ -0,0 +1,93 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.translate; + +// [START translate_v3_translate_text_with_glossary_and_model] +import com.google.cloud.translate.v3.GlossaryName; +import com.google.cloud.translate.v3.LocationName; +import com.google.cloud.translate.v3.TranslateTextGlossaryConfig; +import com.google.cloud.translate.v3.TranslateTextRequest; +import com.google.cloud.translate.v3.TranslateTextResponse; +import com.google.cloud.translate.v3.Translation; +import com.google.cloud.translate.v3.TranslationServiceClient; + +import java.io.IOException; + +public class TranslateTextWithGlossaryAndModel { + + public static void translateTextWithGlossaryAndModel() throws IOException { + // TODO(developer): Replace these variables before running the sample. + String projectId = "YOUR-PROJECT-ID"; + // Supported Languages: https://cloud.google.com/translate/docs/languages + String sourceLanguage = "your-source-language"; + String targetLanguage = "your-target-language"; + String text = "your-text"; + String glossaryId = "your-glossary-display-name"; + String modelId = "YOUR-MODEL-ID"; + translateTextWithGlossaryAndModel( + projectId, sourceLanguage, targetLanguage, text, glossaryId, modelId); + } + + // Translating Text with Glossary and Model + public static void translateTextWithGlossaryAndModel( + String projectId, + String sourceLanguage, + String targetLanguage, + String text, + String glossaryId, + String modelId) + throws IOException { + + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. After completing all of your requests, call + // the "close" method on the client to safely clean up any remaining background resources. + try (TranslationServiceClient client = TranslationServiceClient.create()) { + // Supported Locations: `global`, [glossary location], or [model location] + // Glossaries must be hosted in `us-central1` + // Custom Models must use the same location as your model. (us-central1) + String location = "us-central1"; + LocationName parent = LocationName.of(projectId, location); + + GlossaryName glossaryName = GlossaryName.of(projectId, location, glossaryId); + TranslateTextGlossaryConfig glossaryConfig = + TranslateTextGlossaryConfig.newBuilder().setGlossary(glossaryName.toString()).build(); + + String modelPath = + String.format("projects/%s/locations/%s/models/%s", projectId, location, modelId); + + // Supported Mime Types: https://cloud.google.com/translate/docs/supported-formats + TranslateTextRequest request = + TranslateTextRequest.newBuilder() + .setParent(parent.toString()) + .setMimeType("text/plain") + .setSourceLanguageCode(sourceLanguage) + .setTargetLanguageCode(targetLanguage) + .addContents(text) + .setGlossaryConfig(glossaryConfig) + .setModel(modelPath) + .build(); + + TranslateTextResponse response = client.translateText(request); + + // Display the translation for each input text provided + for (Translation translation : response.getGlossaryTranslationsList()) { + System.out.printf("Translated text: %s\n", translation.getTranslatedText()); + } + } + } +} +// [END translate_v3_translate_text_with_glossary_and_model] diff --git a/translate/cloud-client/src/test/java/com/example/translate/BatchTranslateTextWithGlossaryAndModelTests.java b/translate/cloud-client/src/test/java/com/example/translate/BatchTranslateTextWithGlossaryAndModelTests.java new file mode 100644 index 00000000000..a91e530a0fb --- /dev/null +++ b/translate/cloud-client/src/test/java/com/example/translate/BatchTranslateTextWithGlossaryAndModelTests.java @@ -0,0 +1,145 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.translate; + +import static com.google.common.truth.Truth.assertThat; +import static junit.framework.TestCase.assertNotNull; + +import com.google.api.gax.longrunning.OperationFuture; +import com.google.api.gax.paging.Page; +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageOptions; +import com.google.cloud.translate.v3.CreateGlossaryMetadata; +import com.google.cloud.translate.v3.CreateGlossaryRequest; +import com.google.cloud.translate.v3.DeleteGlossaryMetadata; +import com.google.cloud.translate.v3.DeleteGlossaryRequest; +import com.google.cloud.translate.v3.DeleteGlossaryResponse; +import com.google.cloud.translate.v3.GcsSource; +import com.google.cloud.translate.v3.Glossary; +import com.google.cloud.translate.v3.GlossaryInputConfig; +import com.google.cloud.translate.v3.GlossaryName; +import com.google.cloud.translate.v3.LocationName; +import com.google.cloud.translate.v3.TranslationServiceClient; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ExecutionException; + +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Tests for Batch Translate Text With Glossary sample. */ +@RunWith(JUnit4.class) +@SuppressWarnings("checkstyle:abbreviationaswordinname") +public class BatchTranslateTextWithGlossaryAndModelTests { + private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); + private static final String INPUT_URI = + "gs://cloud-samples-data/translation/text_with_custom_model_and_glossary.txt"; + private static final String GLOSSARY_ID = + String.format("test_%s", UUID.randomUUID().toString().replace("-", "_").substring(0, 26)); + private static final String MODEL_ID = "TRL2188848820815848149"; + private static final String GLOSSARY_INPUT_URI = + "gs://cloud-samples-data/translation/glossary_ja.csv"; + + private ByteArrayOutputStream bout; + private PrintStream out; + + private static final void cleanUpBucket() { + Storage storage = StorageOptions.getDefaultInstance().getService(); + Page blobs = + storage.list( + PROJECT_ID, + Storage.BlobListOption.currentDirectory(), + Storage.BlobListOption.prefix("BATCH_TRANSLATION_OUTPUT/")); + + deleteDirectory(storage, blobs); + } + + private static void deleteDirectory(Storage storage, Page blobs) { + for (Blob blob : blobs.iterateAll()) { + System.out.println(blob.getBlobId()); + if (!blob.delete()) { + Page subBlobs = + storage.list( + PROJECT_ID, + Storage.BlobListOption.currentDirectory(), + Storage.BlobListOption.prefix(blob.getName())); + + deleteDirectory(storage, subBlobs); + } + } + } + + private static void requireEnvVar(String varName) { + assertNotNull( + "Environment variable '%s' is required to perform these tests.".format(varName), + System.getenv(varName)); + } + + @BeforeClass + public static void checkRequirements() { + requireEnvVar("GOOGLE_APPLICATION_CREDENTIALS"); + requireEnvVar("GOOGLE_CLOUD_PROJECT"); + } + + @Before + public void setUp() throws InterruptedException, ExecutionException, IOException { + // Create a glossary that can be used in the test + PrintStream temp = new PrintStream(new ByteArrayOutputStream()); + System.setOut(temp); + List languageCodes = new ArrayList<>(); + languageCodes.add("en"); + languageCodes.add("ja"); + CreateGlossary.createGlossary(PROJECT_ID, GLOSSARY_ID, languageCodes, GLOSSARY_INPUT_URI); + + bout = new ByteArrayOutputStream(); + out = new PrintStream(bout); + System.setOut(out); + } + + @After + public void tearDown() throws InterruptedException, ExecutionException, IOException { + cleanUpBucket(); + // Delete the created glossary + DeleteGlossary.deleteGlossary(PROJECT_ID, GLOSSARY_ID); + System.setOut(null); + } + + @Test + public void testBatchTranslateTextWithGlossaryAndModel() + throws InterruptedException, ExecutionException, IOException { + BatchTranslateTextWithGlossaryAndModel.batchTranslateTextWithGlossaryAndModel( + PROJECT_ID, + "en", + "ja", + INPUT_URI, + "gs://" + PROJECT_ID + "/BATCH_TRANSLATION_OUTPUT/", + GLOSSARY_ID, + MODEL_ID); + String got = bout.toString(); + assertThat(got).contains("Total Characters: 25"); + } +} diff --git a/translate/cloud-client/src/test/java/com/example/translate/TranslateTextWithGlossaryAndModelTests.java b/translate/cloud-client/src/test/java/com/example/translate/TranslateTextWithGlossaryAndModelTests.java new file mode 100644 index 00000000000..dc68fa0fa92 --- /dev/null +++ b/translate/cloud-client/src/test/java/com/example/translate/TranslateTextWithGlossaryAndModelTests.java @@ -0,0 +1,107 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.translate; + +import static com.google.common.truth.Truth.assertThat; +import static junit.framework.TestCase.assertNotNull; + +import com.google.api.gax.longrunning.OperationFuture; +import com.google.cloud.translate.v3.CreateGlossaryMetadata; +import com.google.cloud.translate.v3.CreateGlossaryRequest; +import com.google.cloud.translate.v3.DeleteGlossaryMetadata; +import com.google.cloud.translate.v3.DeleteGlossaryRequest; +import com.google.cloud.translate.v3.DeleteGlossaryResponse; +import com.google.cloud.translate.v3.GcsSource; +import com.google.cloud.translate.v3.Glossary; +import com.google.cloud.translate.v3.GlossaryInputConfig; +import com.google.cloud.translate.v3.GlossaryName; +import com.google.cloud.translate.v3.LocationName; +import com.google.cloud.translate.v3.TranslationServiceClient; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ExecutionException; + +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Tests for Batch Translate Text With Glossary and Model sample. */ +@RunWith(JUnit4.class) +@SuppressWarnings("checkstyle:abbreviationaswordinname") +public class TranslateTextWithGlossaryAndModelTests { + private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); + private static final String MODEL_ID = "TRL2188848820815848149"; + private static final String GLOSSARY_INPUT_URI = + "gs://cloud-samples-data/translation/glossary_ja.csv"; + private static final String GLOSSARY_ID = + String.format("test_%s", UUID.randomUUID().toString().replace("-", "_").substring(0, 26)); + + private ByteArrayOutputStream bout; + private PrintStream out; + + private static void requireEnvVar(String varName) { + assertNotNull( + "Environment variable '%s' is required to perform these tests.".format(varName), + System.getenv(varName)); + } + + @BeforeClass + public static void checkRequirements() { + requireEnvVar("GOOGLE_APPLICATION_CREDENTIALS"); + requireEnvVar("GOOGLE_CLOUD_PROJECT"); + } + + @Before + public void setUp() throws InterruptedException, ExecutionException, IOException { + // Create a glossary that can be used in the test + PrintStream temp = new PrintStream(new ByteArrayOutputStream()); + System.setOut(temp); + List languageCodes = new ArrayList<>(); + languageCodes.add("en"); + languageCodes.add("ja"); + CreateGlossary.createGlossary(PROJECT_ID, GLOSSARY_ID, languageCodes, GLOSSARY_INPUT_URI); + + bout = new ByteArrayOutputStream(); + out = new PrintStream(bout); + System.setOut(out); + } + + @After + public void tearDown() throws InterruptedException, ExecutionException, IOException { + // Clean up + // Delete the created glossary + DeleteGlossary.deleteGlossary(PROJECT_ID, GLOSSARY_ID); + System.setOut(null); + } + + @Test + public void testTranslateTextWithGlossaryAndModel() throws IOException { + TranslateTextWithGlossaryAndModel.translateTextWithGlossaryAndModel( + PROJECT_ID, "en", "ja", "That' il do it. deception", GLOSSARY_ID, MODEL_ID); + String got = bout.toString(); + assertThat(got).contains("それはそうだ"); // custom model + assertThat(got).contains("欺く"); // glossary + } +} From 9b5cc83e169efba62fd9e66175f2a9db17e4f93e Mon Sep 17 00:00:00 2001 From: nnegrey Date: Tue, 14 Jan 2020 11:36:58 -0700 Subject: [PATCH 2/2] Update pom to java 11 --- translate/cloud-client/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/translate/cloud-client/pom.xml b/translate/cloud-client/pom.xml index 65f74557325..11fe6fbed34 100644 --- a/translate/cloud-client/pom.xml +++ b/translate/cloud-client/pom.xml @@ -30,8 +30,8 @@ - 1.8 - 1.8 + 11 + 11 UTF-8