rowFuture = dataClient.readRowAsync("fake-table", "fake-key");
+
+ Assert.assertEquals("value", rowFuture.get().getCells().get(0).getValue().toStringUtf8());
+ }
+}
+```
+
+## Java Versions
+
+Java 7 or above is required for using this emulator.
+
+## Versioning
+
+This library follows [Semantic Versioning](http://semver.org/).
+
+It is currently in major version zero (`0.y.z`), which means that anything may
+change at any time and the public API should not be considered stable.
+
+## Contributing
+
+Contributions to this library are always welcome and highly encouraged.
+
+See [CONTRIBUTING] for more information on how to get started and [DEVELOPING] for a layout of the
+codebase.
+
+## License
+
+Apache 2.0 - See [LICENSE] for more information.
+
+[CONTRIBUTING]:https://github.com/googleapis/google-cloud-java/blob/master/CONTRIBUTING.md
+[LICENSE]: https://github.com/googleapis/google-cloud-java/blob/master/LICENSE
+[cloud-bigtable]: https://cloud.google.com/bigtable/
+
diff --git a/google-cloud-testing/google-cloud-bigtable-emulator/pom.xml b/google-cloud-testing/google-cloud-bigtable-emulator/pom.xml
new file mode 100644
index 000000000000..2cec7e6cc4ec
--- /dev/null
+++ b/google-cloud-testing/google-cloud-bigtable-emulator/pom.xml
@@ -0,0 +1,92 @@
+
+
+
+ google-cloud-testing
+ com.google.cloud
+ 0.70.1-alpha-SNAPSHOT
+
+ 4.0.0
+
+ google-cloud-bigtable-emulator
+ 0.70.1-alpha-SNAPSHOT
+
+
+
+
+ com.google.cloud
+ google-cloud-gcloud-maven-plugin
+ 0.70.1-alpha-SNAPSHOT
+
+
+
+ gen-sources
+ generate-resources
+
+ download
+
+
+
+ bigtable-darwin-x86
+ bigtable-darwin-x86_64
+ bigtable-linux-x86
+ bigtable-linux-x86_64
+ bigtable-windows-x86
+ bigtable-windows-x86_64
+
+
+
+
+
+
+
+
+
+
+
+ io.grpc
+ grpc-core
+ 1.13.1
+ provided
+
+
+
+ io.grpc
+ grpc-netty-shaded
+ 1.13.1
+ provided
+
+
+
+ com.google.api
+ api-common
+
+
+
+
+ junit
+ junit
+ compile
+ true
+
+
+
+ com.google.truth
+ truth
+ test
+
+
+
+ com.google.api.grpc
+ grpc-google-cloud-bigtable-v2
+ test
+
+
+
+ com.google.api.grpc
+ grpc-google-cloud-bigtable-admin-v2
+ test
+
+
+
diff --git a/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/BigtableEmulatorRule.java b/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/BigtableEmulatorRule.java
new file mode 100644
index 000000000000..b02887abd9e9
--- /dev/null
+++ b/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/BigtableEmulatorRule.java
@@ -0,0 +1,89 @@
+/*
+ * Copyright 2018 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.cloud.bigtable.emulator.v2;
+
+import com.google.api.core.BetaApi;
+import io.grpc.ManagedChannel;
+import org.junit.rules.ExternalResource;
+
+/**
+ * The BigtableEmulatorRule manages the lifecycle of the Bigtable {@link Emulator}. Before the
+ * start of a test, the emulator will be started on a random port and will be shutdown after the
+ * test finishes.
+ *
+ * Example usage:
+ *
+ * {@code
+ * {@literal @RunWith(JUnit4.class)}
+ * public class MyTest {
+ * {@literal @Rule}
+ * public final BigtableEmulatorRule bigtableEmulator = BigtableEmulatorRule.create();
+ *
+ * {@literal @Test}
+ * public void testUsingEmulator() {
+ * ManagedChannel adminChannel = bigtableEmulator.getAdminChannel();
+ * // Do something with channel
+ * }
+ * }
+ * }
+ */
+@BetaApi("Surface for Bigtable emulator is not yet stable")
+public class BigtableEmulatorRule extends ExternalResource {
+ private Emulator emulator;
+
+ public static BigtableEmulatorRule create() {
+ return new BigtableEmulatorRule();
+ }
+
+ private BigtableEmulatorRule() { }
+
+ /** Initializes the Bigtable emulator before a test runs. */
+ @Override
+ protected void before() throws Throwable {
+ emulator = Emulator.createBundled();
+ emulator.start();
+ }
+
+ /** Stops the Bigtable emulator after a test finishes. */
+ @Override
+ protected void after() {
+ emulator.stop();
+ emulator = null;
+ }
+
+ /**
+ * Gets a {@link ManagedChannel} connected to the Emulator. The channel is configured for data
+ * operations.
+ */
+ public ManagedChannel getDataChannel() {
+ return emulator.getDataChannel();
+ }
+
+ /**
+ * Gets a {@link ManagedChannel} connected to the Emulator. This channel should be used for admin
+ * operations.
+ */
+ public ManagedChannel getAdminChannel() {
+ return emulator.getAdminChannel();
+ }
+
+ /**
+ * Gets the port of the emulator, allowing the caller to create their own {@link ManagedChannel}.
+ */
+ public int getPort() {
+ return emulator.getPort();
+ }
+}
diff --git a/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/Emulator.java b/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/Emulator.java
new file mode 100644
index 000000000000..c69500faaeea
--- /dev/null
+++ b/google-cloud-testing/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/Emulator.java
@@ -0,0 +1,282 @@
+/*
+ * Copyright 2018 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.cloud.bigtable.emulator.v2;
+
+
+import com.google.api.core.BetaApi;
+import io.grpc.ManagedChannel;
+import io.grpc.ManagedChannelBuilder;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.nio.file.Path;
+import java.util.Locale;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Wraps the Bigtable emulator in a java api.
+ *
+ *
This class will use the golang binaries embedded in this jar to launch the emulator as an
+ * external process and redirect its output to a {@link Logger}.
+ *
+ */
+@BetaApi("Surface for Bigtable emulator is not yet stable")
+public class Emulator {
+
+ private static final Logger LOGGER = Logger.getLogger(Emulator.class.getName());
+
+ private final Path executable;
+ private Process process;
+ private boolean isStopped = true;
+ private Thread shutdownHook;
+
+ private int port;
+ private ManagedChannel dataChannel;
+ private ManagedChannel adminChannel;
+
+ /**
+ * Create a new instance of emulator. The emulator will use the bundled binaries in this jar.
+ * Please note that the emulator is created in a stopped state, please use {@link #start()} after
+ * creating it.
+ */
+ public static Emulator createBundled() throws IOException {
+ String resourcePath = getBundledResourcePath();
+
+ File tmpEmulator = File.createTempFile("cbtemulator", "");
+ tmpEmulator.deleteOnExit();
+
+ try (InputStream is = Emulator.class.getResourceAsStream(resourcePath);
+ FileOutputStream os = new FileOutputStream(tmpEmulator)) {
+
+ if (is == null) {
+ throw new FileNotFoundException("Failed to find the bundled emulator binary: " + resourcePath);
+ }
+
+ byte[] buff = new byte[2048];
+ int length;
+
+ while ((length = is.read(buff)) != -1) {
+ os.write(buff, 0, length);
+ }
+ }
+ tmpEmulator.setExecutable(true);
+
+ return new Emulator(tmpEmulator.toPath());
+ }
+
+ private Emulator(Path executable) {
+ this.executable = executable;
+ }
+
+ /** Starts the emulator process and waits for it to be ready. */
+ public synchronized void start() throws IOException, TimeoutException, InterruptedException {
+ if (!isStopped) {
+ throw new IllegalStateException("Emulator is already started");
+ }
+ this.port = getAvailablePort();
+
+ process = Runtime.getRuntime().exec(String.format("%s -port %d", executable, port));
+ pipeStreamToLog(process.getInputStream(), Level.INFO);
+ pipeStreamToLog(process.getErrorStream(), Level.WARNING);
+ isStopped = false;
+
+ shutdownHook = new Thread() {
+ @Override
+ public void run() {
+ if (!isStopped) {
+ isStopped = true;
+ process.destroy();
+ }
+ }
+ };
+
+ Runtime.getRuntime().addShutdownHook(shutdownHook);
+
+ waitForPort(port);
+ }
+
+ /** Stops the emulator process. */
+ public synchronized void stop() {
+ if (isStopped) {
+ throw new IllegalStateException("Emulator already stopped");
+ }
+
+ try {
+ Runtime.getRuntime().removeShutdownHook(shutdownHook);
+ shutdownHook = null;
+
+ // Shutdown channels in parallel
+ if (dataChannel != null) {
+ dataChannel.shutdownNow();
+ }
+ if (adminChannel != null) {
+ adminChannel.shutdownNow();
+ }
+
+ // Then wait for actual shutdown
+ if (dataChannel != null) {
+ dataChannel.awaitTermination(1, TimeUnit.MINUTES);
+ dataChannel = null;
+ }
+ if (adminChannel != null) {
+ adminChannel.awaitTermination(1, TimeUnit.MINUTES);
+ adminChannel = null;
+ }
+ } catch (InterruptedException e) {
+ LOGGER.warning("Interrupted while waiting for client channels to close");
+ Thread.currentThread().interrupt();
+ } finally {
+ isStopped = true;
+ process.destroy();
+ }
+ }
+
+ public synchronized int getPort() {
+ if (isStopped) {
+ throw new IllegalStateException("Emulator is not running");
+ }
+ return port;
+ }
+
+ public synchronized ManagedChannel getDataChannel() {
+ if (isStopped){
+ throw new IllegalStateException("Emulator is not running");
+ }
+
+ if (dataChannel == null) {
+ dataChannel = newChannelBuilder(port)
+ .maxInboundMessageSize(256 * 1024 * 1024)
+ .build();
+ }
+ return dataChannel;
+ }
+
+ public synchronized ManagedChannel getAdminChannel() {
+ if (isStopped) {
+ throw new IllegalStateException("Emulator is not running");
+ }
+
+ if (adminChannel == null) {
+ adminChannel = newChannelBuilder(port)
+ .build();
+ }
+ return adminChannel;
+ }
+
+ //
+
+ /** Gets the current platform, which will be used to select the appropriate emulator binary. */
+ private static String getBundledResourcePath() {
+ String unformattedOs = System.getProperty("os.name", "unknown").toLowerCase(Locale.ENGLISH);
+ String os;
+ String suffix = "";
+
+ if (unformattedOs.contains("mac") || unformattedOs.contains("darwin")) {
+ os = "darwin";
+ } else if (unformattedOs.contains("win")) {
+ os = "windows";
+ suffix = ".exe";
+ } else if (unformattedOs.contains("linux")) {
+ os = "linux";
+ } else {
+ throw new UnsupportedOperationException(
+ "Emulator is not supported on your platform: " + unformattedOs);
+ }
+
+ String unformattedArch = System.getProperty("os.arch");
+ String arch;
+
+ switch (unformattedArch) {
+ case "x86":
+ arch = "x86";
+ break;
+ case "x86_64":
+ case "amd64":
+ arch = "x86_64";
+ break;
+ default:
+ throw new UnsupportedOperationException("Unsupported architecture: " + unformattedArch);
+ }
+
+ return String.format(
+ "/gcloud/bigtable-%s-%s/platform/bigtable-emulator/cbtemulator%s", os, arch, suffix);
+ }
+
+ /** Gets a random open port number. */
+ private static int getAvailablePort() {
+ try (ServerSocket serverSocket = new ServerSocket(0)) {
+ return serverSocket.getLocalPort();
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to find open port");
+ }
+ }
+
+ /** Waits for a port to open. It's used to wait for the emulator's gRPC server to be ready. */
+ private static void waitForPort(int port) throws InterruptedException, TimeoutException {
+ for (int i = 0; i < 100; i++) {
+ try (Socket ignored = new Socket("localhost", port)) {
+ return;
+ } catch (IOException e) {
+ Thread.sleep(200);
+ }
+ }
+
+ throw new TimeoutException("Timed out waiting for server to start");
+ }
+
+ /** Creates a {@link io.grpc.ManagedChannelBuilder} preconfigured for the emulator's port. */
+ private static ManagedChannelBuilder newChannelBuilder(int port) {
+ // NOTE: usePlaintext is currently @ExperimentalAPI.
+ // See https://github.com/grpc/grpc-java/issues/1772 for discussion
+ return ManagedChannelBuilder.forAddress("localhost", port)
+ .usePlaintext();
+ }
+
+ /** Creates a thread that will pipe an {@link java.io.InputStream} to this class' Logger. */
+ private static void pipeStreamToLog(final InputStream stream, final Level level) {
+ final BufferedReader reader = new BufferedReader(new InputStreamReader(stream));
+
+ Thread thread =
+ new Thread(
+ new Runnable() {
+ @Override
+ public void run() {
+ try {
+ String line;
+ while ((line = reader.readLine()) != null) {
+ LOGGER.log(level, line);
+ }
+ } catch (IOException e) {
+ if (!"Stream closed".equals(e.getMessage())) {
+ LOGGER.log(Level.WARNING, "Failed to read process stream", e);
+ }
+ }
+ }
+ });
+ thread.setDaemon(true);
+ thread.start();
+ }
+ //
+}
diff --git a/google-cloud-testing/google-cloud-bigtable-emulator/src/test/java/com/google/cloud/bigtable/emulator/v2/BigtableEmulatorRuleTest.java b/google-cloud-testing/google-cloud-bigtable-emulator/src/test/java/com/google/cloud/bigtable/emulator/v2/BigtableEmulatorRuleTest.java
new file mode 100644
index 000000000000..03fb139c1f4c
--- /dev/null
+++ b/google-cloud-testing/google-cloud-bigtable-emulator/src/test/java/com/google/cloud/bigtable/emulator/v2/BigtableEmulatorRuleTest.java
@@ -0,0 +1,108 @@
+/*
+ * Copyright 2018 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.cloud.bigtable.emulator.v2;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.bigtable.admin.v2.BigtableTableAdminGrpc;
+import com.google.bigtable.admin.v2.BigtableTableAdminGrpc.BigtableTableAdminBlockingStub;
+import com.google.bigtable.admin.v2.ColumnFamily;
+import com.google.bigtable.admin.v2.CreateTableRequest;
+import com.google.bigtable.admin.v2.Table;
+import com.google.bigtable.v2.BigtableGrpc;
+import com.google.bigtable.v2.BigtableGrpc.BigtableBlockingStub;
+import com.google.bigtable.v2.MutateRowRequest;
+import com.google.bigtable.v2.Mutation;
+import com.google.bigtable.v2.Mutation.SetCell;
+import com.google.bigtable.v2.ReadRowsRequest;
+import com.google.bigtable.v2.ReadRowsResponse;
+import com.google.protobuf.ByteString;
+import java.util.Iterator;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public class BigtableEmulatorRuleTest {
+ @Rule
+ public BigtableEmulatorRule bigtableRule = BigtableEmulatorRule.create();
+ private BigtableTableAdminBlockingStub tableAdminStub;
+ private BigtableBlockingStub dataStub;
+
+
+ @Before
+ public void setUp() {
+ tableAdminStub = BigtableTableAdminGrpc.newBlockingStub(bigtableRule.getAdminChannel());
+ dataStub = BigtableGrpc.newBlockingStub(bigtableRule.getDataChannel());
+ }
+
+ @Test
+ public void testTableAdminClient() {
+ Table table = tableAdminStub.createTable(
+ CreateTableRequest.newBuilder()
+ .setParent("projects/fake-project/instances/fake-instance")
+ .setTableId("fake-table")
+ .setTable(
+ Table.newBuilder()
+ .putColumnFamilies("cf", ColumnFamily.getDefaultInstance())
+ )
+ .build()
+ );
+
+ assertThat(table.getName())
+ .isEqualTo("projects/fake-project/instances/fake-instance/tables/fake-table");
+ }
+
+ @Test
+ public void testDataClient() throws Exception {
+ tableAdminStub.createTable(
+ CreateTableRequest.newBuilder()
+ .setParent("projects/fake-project/instances/fake-instance")
+ .setTableId("fake-table")
+ .setTable(
+ Table.newBuilder()
+ .putColumnFamilies("cf", ColumnFamily.getDefaultInstance())
+ )
+ .build()
+ );
+
+ dataStub.mutateRow(
+ MutateRowRequest.newBuilder()
+ .setTableName("projects/fake-project/instances/fake-instance/tables/fake-table")
+ .setRowKey(ByteString.copyFromUtf8("fake-key"))
+ .addMutations(
+ Mutation.newBuilder().setSetCell(
+ SetCell.newBuilder()
+ .setFamilyName("cf")
+ .setColumnQualifier(ByteString.EMPTY)
+ .setValue(ByteString.copyFromUtf8("value"))
+ )
+ )
+ .build()
+ );
+
+ Iterator results = dataStub.readRows(
+ ReadRowsRequest.newBuilder()
+ .setTableName("projects/fake-project/instances/fake-instance/tables/fake-table")
+ .build()
+ );
+
+ ReadRowsResponse row = results.next();
+ assertThat(row.getChunks(0).getValue()).isEqualTo(ByteString.copyFromUtf8("value"));
+ }
+}
\ No newline at end of file
diff --git a/google-cloud-testing/google-cloud-bigtable-emulator/src/test/java/com/google/cloud/bigtable/emulator/v2/EmulatorTest.java b/google-cloud-testing/google-cloud-bigtable-emulator/src/test/java/com/google/cloud/bigtable/emulator/v2/EmulatorTest.java
new file mode 100644
index 000000000000..46139b4f0377
--- /dev/null
+++ b/google-cloud-testing/google-cloud-bigtable-emulator/src/test/java/com/google/cloud/bigtable/emulator/v2/EmulatorTest.java
@@ -0,0 +1,116 @@
+/*
+ * Copyright 2018 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.cloud.bigtable.emulator.v2;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.bigtable.admin.v2.BigtableTableAdminGrpc;
+import com.google.bigtable.admin.v2.BigtableTableAdminGrpc.BigtableTableAdminBlockingStub;
+import com.google.bigtable.admin.v2.ColumnFamily;
+import com.google.bigtable.admin.v2.CreateTableRequest;
+import com.google.bigtable.admin.v2.Table;
+import com.google.bigtable.v2.BigtableGrpc;
+import com.google.bigtable.v2.BigtableGrpc.BigtableBlockingStub;
+import com.google.bigtable.v2.MutateRowRequest;
+import com.google.bigtable.v2.Mutation;
+import com.google.bigtable.v2.Mutation.SetCell;
+import com.google.bigtable.v2.ReadRowsRequest;
+import com.google.bigtable.v2.ReadRowsResponse;
+import com.google.protobuf.ByteString;
+import java.util.Iterator;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public class EmulatorTest {
+
+ private Emulator emulator;
+ private BigtableTableAdminBlockingStub tableAdminStub;
+ private BigtableBlockingStub dataStub;
+
+
+ @Before
+ public void setUp() throws Exception {
+ emulator = Emulator.createBundled();
+ emulator.start();
+ tableAdminStub = BigtableTableAdminGrpc.newBlockingStub(emulator.getAdminChannel());
+ dataStub = BigtableGrpc.newBlockingStub(emulator.getDataChannel());
+ }
+
+ @After
+ public void tearDown() {
+ emulator.stop();
+ emulator = null;
+ }
+
+ @Test
+ public void testTableAdminClient() {
+ Table table = tableAdminStub.createTable(
+ CreateTableRequest.newBuilder()
+ .setParent("projects/fake-project/instances/fake-instance")
+ .setTableId("fake-table")
+ .setTable(
+ Table.newBuilder()
+ .putColumnFamilies("cf", ColumnFamily.getDefaultInstance())
+ )
+ .build()
+ );
+
+ assertThat(table.getName())
+ .isEqualTo("projects/fake-project/instances/fake-instance/tables/fake-table");
+ }
+
+ @Test
+ public void testDataClient() {
+ tableAdminStub.createTable(
+ CreateTableRequest.newBuilder()
+ .setParent("projects/fake-project/instances/fake-instance")
+ .setTableId("fake-table")
+ .setTable(
+ Table.newBuilder()
+ .putColumnFamilies("cf", ColumnFamily.getDefaultInstance())
+ )
+ .build()
+ );
+
+ dataStub.mutateRow(
+ MutateRowRequest.newBuilder()
+ .setTableName("projects/fake-project/instances/fake-instance/tables/fake-table")
+ .setRowKey(ByteString.copyFromUtf8("fake-key"))
+ .addMutations(
+ Mutation.newBuilder().setSetCell(
+ SetCell.newBuilder()
+ .setFamilyName("cf")
+ .setColumnQualifier(ByteString.EMPTY)
+ .setValue(ByteString.copyFromUtf8("value"))
+ )
+ )
+ .build()
+ );
+
+ Iterator results = dataStub.readRows(
+ ReadRowsRequest.newBuilder()
+ .setTableName("projects/fake-project/instances/fake-instance/tables/fake-table")
+ .build()
+ );
+
+ ReadRowsResponse row = results.next();
+ assertThat(row.getChunks(0).getValue()).isEqualTo(ByteString.copyFromUtf8("value"));
+ }
+}
diff --git a/google-cloud-testing/google-cloud-gcloud-maven-plugin/README.md b/google-cloud-testing/google-cloud-gcloud-maven-plugin/README.md
new file mode 100644
index 000000000000..676659962a6c
--- /dev/null
+++ b/google-cloud-testing/google-cloud-gcloud-maven-plugin/README.md
@@ -0,0 +1,9 @@
+# Google Cloud SDK tools
+
+This module is considered an internal implementation detail of
+google-cloud-java and should not be used by other projects. It contains
+maven plugins that integrate with the gcloud SDK.
+
+## Download plugin
+
+A maven plugin to download components published in the gcloud sdk repository.
diff --git a/google-cloud-testing/google-cloud-gcloud-maven-plugin/pom.xml b/google-cloud-testing/google-cloud-gcloud-maven-plugin/pom.xml
new file mode 100644
index 000000000000..ce677348cac9
--- /dev/null
+++ b/google-cloud-testing/google-cloud-gcloud-maven-plugin/pom.xml
@@ -0,0 +1,93 @@
+
+
+ 4.0.0
+
+
+ google-cloud-testing
+ com.google.cloud
+ 0.70.1-alpha-SNAPSHOT
+
+
+ google-cloud-gcloud-maven-plugin
+ 0.70.1-alpha-SNAPSHOT
+ maven-plugin
+ Experimental Maven plugin to interact with the Google Cloud SDK
+ (https://cloud.google.com/sdk/). Currently this plugin is meant to be an internal implementation
+ detail for google-cloud-java.
+
+
+
+ 3.3.9
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-plugin-plugin
+ 3.5.1
+
+ true
+
+
+
+ mojo-descriptor
+
+ descriptor
+
+
+
+
+
+
+
+
+
+
+ org.apache.maven
+ maven-plugin-api
+ ${maven.version}
+
+
+
+ org.apache.maven
+ maven-core
+ ${maven.version}
+
+
+
+ org.apache.maven.plugin-tools
+ maven-plugin-annotations
+ 3.5.1
+ provided
+
+
+
+
+ com.google.code.gson
+ gson
+ 2.7
+
+
+
+ org.apache.commons
+ commons-compress
+ 1.11
+
+
+
+ commons-io
+ commons-io
+ 2.5
+
+
+
+
+ junit
+ junit
+ test
+
+
+
diff --git a/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/Component.java b/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/Component.java
new file mode 100644
index 000000000000..65032d664efd
--- /dev/null
+++ b/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/Component.java
@@ -0,0 +1,93 @@
+/*
+ * Copyright 2018 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.cloud;
+
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import java.io.IOException;
+import java.net.URL;
+import java.util.Objects;
+
+/** Representation of a gcloud component */
+public class Component {
+ private final String id;
+ private final String checksum;
+ private final URL source;
+ private final String fileType;
+
+ static Component fromJson(URL baseUrl, JsonObject componentObj) throws IOException {
+ String id = componentObj.get("id").getAsString();
+
+ JsonElement data = componentObj.get("data");
+ if (data == null) {
+ throw new NullPointerException("Component " + id + " is missing a data section");
+ }
+
+ return create(
+ id,
+ data.getAsJsonObject().get("checksum").getAsString(),
+ new URL(baseUrl, data.getAsJsonObject().get("source").getAsString()),
+ data.getAsJsonObject().get("type").getAsString()
+ );
+ }
+
+ static Component create(String id, String checksum, URL source, String fileType) {
+ return new Component(id, checksum, source, fileType);
+ }
+
+ private Component(String id, String checksum, URL source, String fileType) {
+ this.id = id;
+ this.checksum = checksum;
+ this.source = source;
+ this.fileType = fileType;
+ }
+
+ String getId() {
+ return id;
+ }
+
+ String getChecksum() {
+ return checksum;
+ }
+
+ URL getSource() {
+ return source;
+ }
+
+ String getFileType() {
+ return fileType;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ Component component = (Component) o;
+ return Objects.equals(id, component.id) &&
+ Objects.equals(checksum, component.checksum) &&
+ Objects.equals(source, component.source) &&
+ Objects.equals(fileType, component.fileType);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(id, checksum, source, fileType);
+ }
+}
diff --git a/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/DownloadComponentsMojo.java b/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/DownloadComponentsMojo.java
new file mode 100644
index 000000000000..491aaf284efb
--- /dev/null
+++ b/google-cloud-testing/google-cloud-gcloud-maven-plugin/src/main/java/com/google/cloud/DownloadComponentsMojo.java
@@ -0,0 +1,347 @@
+/*
+ * Copyright 2018 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.cloud;
+
+import com.google.gson.Gson;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import com.google.gson.JsonPrimitive;
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.Writer;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import org.apache.commons.compress.archivers.ArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+import org.apache.commons.compress.utils.IOUtils;
+import org.apache.commons.io.FileUtils;
+import org.apache.maven.execution.MavenSession;
+import org.apache.maven.plugin.AbstractMojo;
+import org.apache.maven.plugin.MojoExecutionException;
+import org.apache.maven.plugins.annotations.Mojo;
+import org.apache.maven.plugins.annotations.Parameter;
+
+/**
+ * Goal that downloads gcloud components and embeds them as resources in a jar. This is mainly
+ * intended to be used to wrapped native emulators in jars. This Mojo makes extensive use of caching
+ * which makes it ok to include in the regular build cycle. The initial manifest fetch is cached for
+ * 2 hours and the actual binaries are downloaded only when their checksum changed in the manifest.
+ */
+@Mojo(name = "download")
+public class DownloadComponentsMojo extends AbstractMojo {
+
+ private static final long STALE_MS = TimeUnit.HOURS.toMillis(2);
+
+ @Parameter(defaultValue = "https://dl.google.com/dl/cloudsdk/channels/rapid/", required = true)
+ private URL baseUrl;
+
+ @Parameter(required = true)
+ private List componentNames;
+
+ @Parameter(defaultValue = "${project.build.outputDirectory}/gcloud", required = true)
+ private File destinationDir;
+
+ @Parameter(defaultValue = "false", required = true, property = "gcloud.download.force")
+ private boolean forceRefresh;
+
+ @Parameter(defaultValue = "${session}", readonly = true)
+ private MavenSession session;
+
+ public void execute() throws MojoExecutionException {
+ if (session.isOffline() && forceRefresh) {
+ throw new MojoExecutionException("Can't force refresh when offline");
+ }
+
+ // Ensure that the output directory exists
+ destinationDir.mkdirs();
+
+ // Update the cached manifest
+ try {
+ updateCachedManifest();
+ } catch (Exception e) {
+ throw new MojoExecutionException("Failed to update the cached manifest", e);
+ }
+
+ // Parse the relevant components
+ List components;
+ try {
+ components = parseManifest();
+ } catch (Exception e) {
+ throw new MojoExecutionException("Failed to parse the manifest", e);
+ }
+
+ // Get the checksums of the existing components
+ Map checksums;
+ try {
+ checksums = parseLocalChecksums();
+ } catch (Exception e) {
+ getLog().warn("Failed to parse local checksums, ignoring", e);
+ checksums = new HashMap<>();
+ }
+
+ // Download any updated components
+ for (Component component : components) {
+ if (!forceRefresh && component.getChecksum().equals(checksums.get(component.getId()))) {
+ continue;
+ }
+
+ try {
+ downloadComponent(component);
+ } catch (Exception e) {
+ throw new MojoExecutionException("Failed to download component " + component.getId(), e);
+ }
+ }
+
+ // Write the checksums of the newly updated components.
+ try {
+ writeLocalChecksums(components);
+ } catch (IOException e) {
+ throw new MojoExecutionException("Failed to update the local checksum cache", e);
+ }
+ }
+
+ private URL getManifestUrl() throws MalformedURLException {
+ return new URL(baseUrl, "components-2.json");
+ }
+
+ private File getManifestCache() {
+ return new File(destinationDir, "components-2.json");
+ }
+
+ private File getComponentPath(Component component) {
+ return new File(destinationDir, component.getId());
+ }
+
+ private File getChecksumFile() {
+ return new File(destinationDir, "checksums.json");
+ }
+
+ /**
+ * Try to update the locally cached cache manifest. This will be a noop if the cached manifest is
+ * fresh enough.
+ */
+ private void updateCachedManifest() throws IOException {
+ URL manifestUrl = getManifestUrl();
+ File localCache = getManifestCache();
+
+ boolean isStale = !localCache.exists()
+ || new Date().getTime() - localCache.lastModified() < STALE_MS;
+
+ if (forceRefresh && session.isOffline()) {
+ throw new IllegalStateException("Can't force manifest refresh while offline");
+ }
+
+ if (session.isOffline() && localCache.exists() && isStale) {
+ getLog().info("Using stale manifest because offline mode is enabled");
+ return;
+ }
+
+ if (!forceRefresh && !isStale) {
+ getLog().debug("Manifest is up to date, skipping manifest download");
+ return;
+ }
+
+ if (session.isOffline()) {
+ throw new IllegalStateException("Can't download manifest in offline mode");
+ }
+
+ getLog().debug("Downloading fresh manifest");
+
+ File tempFile = File.createTempFile(localCache.getName(), "");
+
+ try (BufferedInputStream in = new BufferedInputStream(manifestUrl.openStream());
+ FileOutputStream fileOutputStream = new FileOutputStream(tempFile)) {
+ byte dataBuffer[] = new byte[1024];
+ int bytesRead;
+ while ((bytesRead = in.read(dataBuffer, 0, 1024)) != -1) {
+ fileOutputStream.write(dataBuffer, 0, bytesRead);
+ }
+ }
+
+ @SuppressWarnings("unused")
+ boolean ignored = localCache.delete();
+
+ FileUtils.moveFile(tempFile, localCache);
+ }
+
+ /**
+ * Parse the locally cached manifest and extract the relevant components.
+ */
+ private List parseManifest() throws IOException {
+ JsonParser parser = new JsonParser();
+ JsonElement json;
+ try (FileReader reader = new FileReader(getManifestCache())) {
+ json = parser.parse(reader);
+ }
+
+ JsonArray jsonComponents = json.getAsJsonObject().get("components").getAsJsonArray();
+ List results = new ArrayList<>();
+
+ for (JsonElement jsonComponent : jsonComponents) {
+ JsonObject componentObj = jsonComponent.getAsJsonObject();
+ String id = componentObj.get("id").getAsString();
+
+ if (!componentNames.contains(id)) {
+ continue;
+ }
+ results.add(Component.fromJson(baseUrl, componentObj));
+ }
+
+ return results;
+ }
+
+ /**
+ * Parses a local manifest of the downloaded component checksums
+ */
+ private Map parseLocalChecksums() throws IOException {
+ JsonElement json = new JsonObject();
+
+ if (getChecksumFile().exists()) {
+ JsonParser parser = new JsonParser();
+
+ try (FileReader reader = new FileReader(getChecksumFile())) {
+ json = parser.parse(reader);
+ }
+ }
+
+ Map results = new HashMap<>();
+ JsonObject checksumMap = json.getAsJsonObject();
+ for (String componentName : componentNames) {
+ JsonElement checksumJson = checksumMap.get(componentName);
+ if (checksumJson != null) {
+ results.put(componentName, checksumJson.getAsString());
+ }
+ }
+ return results;
+ }
+
+ /**
+ * Downloads and extracts the component into the destinationDir.
+ */
+ private void downloadComponent(Component component) throws IOException, NoSuchAlgorithmException {
+ getLog().info("Downloading " + component.getId());
+
+ if (!"tar".equals(component.getFileType())) {
+ throw new UnsupportedOperationException(
+ "Only tarballs are supported, got: " + component.getFileType());
+ }
+
+ // Download and verify the component
+ File tmpArchive = File.createTempFile(component.getId(), "");
+ tmpArchive.deleteOnExit();
+
+ MessageDigest digest = MessageDigest.getInstance("SHA-256");
+ try (InputStream is = component.getSource().openStream();
+ FileOutputStream os = new FileOutputStream(tmpArchive)) {
+
+ byte dataBuffer[] = new byte[1024];
+ int bytesRead;
+ while ((bytesRead = is.read(dataBuffer, 0, 1024)) != -1) {
+ digest.update(dataBuffer, 0, bytesRead);
+ os.write(dataBuffer, 0, bytesRead);
+ }
+ }
+ String checksum = byteArrayToHex(digest.digest());
+
+ if (!checksum.equals(component.getChecksum())) {
+ throw new RuntimeException(String
+ .format("Checksum mismatch for %s %s != %s", component.getId(), component.getChecksum(),
+ checksum));
+ }
+
+ // Stage the expanded archive
+ File tmpPath = Files.createTempDirectory(component.getId()).toFile();
+
+ try (TarArchiveInputStream stream = new TarArchiveInputStream(
+ new GzipCompressorInputStream(new FileInputStream(tmpArchive)))) {
+
+ ArchiveEntry entry;
+
+ while ((entry = stream.getNextEntry()) != null) {
+ File dest = new File(tmpPath, entry.getName());
+
+ if (entry.isDirectory()) {
+ if (!dest.mkdirs()) {
+ getLog().warn("Failed to expand the directory " + dest);
+ }
+ } else {
+ try (OutputStream outputFileStream = new FileOutputStream(dest)) {
+ IOUtils.copy(stream, outputFileStream);
+ }
+ }
+ }
+ }
+
+ // Move it into place
+ File localPath = getComponentPath(component);
+ FileUtils.deleteDirectory(localPath);
+ FileUtils.moveDirectory(tmpPath, localPath);
+ }
+
+ private static String byteArrayToHex(byte[] a) {
+ StringBuilder sb = new StringBuilder(a.length * 2);
+ for (byte b : a) {
+ sb.append(String.format("%02x", b));
+ }
+ return sb.toString();
+ }
+
+
+ /**
+ * Update the checksums of the downloaded components. This will avoid the need to download them in
+ * the future.
+ */
+ private void writeLocalChecksums(List components) throws IOException {
+ JsonObject results = new JsonObject();
+
+ try {
+ JsonParser parser = new JsonParser();
+ try (FileReader reader = new FileReader(getChecksumFile())) {
+ results = parser.parse(reader).getAsJsonObject();
+ }
+ } catch (FileNotFoundException e) {
+ // ignored
+ }
+
+ for (Component component : components) {
+ results.add(component.getId(), new JsonPrimitive(component.getChecksum()));
+ }
+
+ try (Writer writer = new FileWriter(getChecksumFile())) {
+ new Gson().toJson(results, writer);
+ }
+ }
+}
diff --git a/google-cloud-testing/pom.xml b/google-cloud-testing/pom.xml
index b644d4b8ed8f..8eef85982c9e 100644
--- a/google-cloud-testing/pom.xml
+++ b/google-cloud-testing/pom.xml
@@ -24,6 +24,8 @@
google-cloud-appengineflexjava
google-cloud-appenginejava8
google-cloud-managedtest
+ google-cloud-gcloud-maven-plugin
+ google-cloud-bigtable-emulator
diff --git a/versions.txt b/versions.txt
index 945c7629e378..4e77eaeb30d0 100644
--- a/versions.txt
+++ b/versions.txt
@@ -147,6 +147,8 @@ google-cloud-storage:1.52.0:1.52.1-SNAPSHOT
google-cloud-tasks:0.70.0-beta:0.70.1-beta-SNAPSHOT
google-cloud-texttospeech:0.70.0-beta:0.70.1-beta-SNAPSHOT
google-cloud-testing:0.70.0-alpha:0.70.1-alpha-SNAPSHOT
+google-cloud-gcloud-maven-plugin:0.70.0-alpha:0.70.1-alpha-SNAPSHOT
+google-cloud-bigtable-emulator:0.70.0-alpha:0.70.1-alpha-SNAPSHOT
google-cloud-trace:0.70.0-beta:0.70.1-beta-SNAPSHOT
google-cloud-translate:1.52.0:1.52.1-SNAPSHOT
google-cloud-util:0.70.0-alpha:0.70.1-alpha-SNAPSHOT