Add cloud tests (#1860)

* Add GCloudTestUtils which has utilities for finding test paths on google
* Add a new cloud_tests github action which runs tests in the "cloud" test group.  These
  tests will be skipped if run from a fork without credentials.

.github/workflows/cloud_tests.yml

@@ -0,0 +1,93 @@
+name: Cloud Tests
+on:
+  push:
+    branches:
+      - 'master'
+  pull_request:
+  workflow_dispatch:
+
+env:
+  PICARD_TEST_INPUTS: gs://hellbender/test/resources/
+  PICARD_TEST_STAGING: gs://hellbender-test-logs/staging/
+  PICARD_TEST_LOGS: /hellbender-test-logs/build_reports/
+  PICARD_TEST_PROJECT: broad-dsde-dev
+
+jobs:
+  ## This workaround is necessary since there is no equivalent to the old TRAVIS_SECURE_ENVIRONMENT variable that indicated
+  ## if a run was privileged and had secrets. Since the GCP credentials are necessary for all tests in order to upload their,
+  ## results that makes them a reasonable proxy for testing the credentials of this entire execution. https://github.com/actions/runner/issues/520
+  check-secrets:
+    name: check if the environment has privileges
+    outputs:
+      google-credentials: ${{ steps.google-credentials.outputs.defined }}
+    runs-on: ubuntu-latest
+    steps:
+      - id: google-credentials
+        env:
+          GCP_CREDENTIALS: ${{ secrets.GCP_CREDENTIALS }}
+        if: "${{ env.GCP_CREDENTIALS != '' }}"
+        run: echo defined=true >> $GITHUB_OUTPUT
+
+  test:
+    runs-on: ubuntu-latest
+    needs: check-secrets
+    strategy:
+      matrix:
+        java: [ 17 ]
+        run_barclay_tests: [true, false]
+        experimental: [ false ]
+      fail-fast: false
+    continue-on-error: ${{ matrix.experimental }}
+    name: Java ${{ matrix.Java }}, Barclay=${{ matrix.run_barclay_tests}} cloud tests
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: 'Set up java ${{ matrix.Java }}'
+        uses: actions/setup-java@v3
+        with:
+          java-version: ${{ matrix.Java }}
+          distribution: 'temurin'
+          cache: gradle
+
+      - name: 'Compile with Gradle'
+        run: |
+          ./gradlew compileJava 
+          ./gradlew installDist
+
+      #Google Cloud stuff
+      - id: 'gcloud-auth'
+        if: needs.check-secrets.outputs.google-credentials == 'true'
+        uses: google-github-actions/auth@v0
+        with:
+          credentials_json: ${{ secrets.GCP_CREDENTIALS }}
+          project_id: ${{ env.PICARD_TEST_PROJECT }}
+          create_credentials_file: true
+
+      - name: 'Set up Cloud SDK'
+        if: needs.check-secrets.outputs.google-credentials == 'true'
+        uses: google-github-actions/setup-gcloud@v0
+
+      - name: compile test code
+        if: needs.check-secrets.outputs.google-credentials == 'true'
+        run: ./gradlew compileTestJava
+
+      - name: Run tests
+        if: needs.check-secrets.outputs.google-credentials == 'true'
+        env:
+          TEST_TYPE: cloud
+        run: |
+          if [[ ${{matrix.run_barclay_tests}} == true ]]; then
+            echo "Running tests using the Barclay command line parser."
+            ./gradlew barclayTest
+          else
+            echo "Running tests using the legacy Picard command line parser."
+            ./gradlew jacocoTestReport
+          fi
+
+      - name: Upload test results
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: cloud-test-results-${{ matrix.Java }}-barclay-${{ matrix.run_barclay_tests}}
+          path: build/reports/tests

build.gradle

@@ -87,6 +87,7 @@ dependencies {
     compileOnly(googleNio)
 
     testImplementation 'org.testng:testng:6.14.3'
+    testImplementation(googleNio)
     implementation 'org.apache.commons:commons-lang3:3.6'
 }
 
@@ -221,10 +222,21 @@ tasks.withType(Test) {
     ]
 
     useTestNG {
-        if (OperatingSystem.current().isUnix()) {
-            excludeGroups "slow", "broken"
+        List excludes = ["slow", "broken"]
+        if( !OperatingSystem.current().isUnix() ) {
+            excludes << "unix"
+        }
+        String TEST_TYPE = "$System.env.TEST_TYPE"
+        if (TEST_TYPE == "cloud") {
+            // run only the cloud tests
+            includeGroups "cloud", "bucket"
+            excludeGroups(*excludes)
+        } else if (TEST_TYPE == "all") {
+            //include everything
+            excludeGroups(*excludes)
         } else {
-            excludeGroups "slow", "broken", "unix"
+            excludes.addAll("cloud", "bucket")
+            excludeGroups(*excludes);
         }
     }
 

src/test/java/picard/util/GCloudTestUtils.java

@@ -0,0 +1,61 @@
+package picard.util;
+
+public final class GCloudTestUtils {
+    /**
+     *  This is a public requester pays bucket owned by the broad-gatk-test project.
+     *  It must be owned by a different project than the service account doing the testing or the test may fail because it can access the
+     *  file directly through alternative permissions.
+     */
+    public static final String REQUESTER_PAYS_BUCKET_DEFAULT = "gs://hellbender-requester-pays-test/";
+
+    public static final String TEST_INPUTS_DEFAULT = "gs://hellbender/test/resources/";
+    public static final String TEST_STAGING_DEFAULT = "gs://hellbender-test-logs/staging/";
+    public static final String TEST_PROJECT_DEFAULT = "broad-dsde-dev";
+
+
+    /**
+     * A publicly readable GCS bucket set as requester pays, this should not be owned by the same project that is set
+     * as {@link #getTestProject()} or the tests for requester pays access may be invalid.
+     *
+     * @return PICARD_REQUESTER_PAYS_BUCKET env. var if defined, {@value GCloudTestUtils#REQUESTER_PAYS_BUCKET_DEFAULT}.
+     */
+    public static String getRequesterPaysBucket() {
+        return getSystemProperty("PICARD_REQUESTER_PAYS_BUCKET", REQUESTER_PAYS_BUCKET_DEFAULT);
+    }
+
+    private static String getSystemProperty(final String variableName, final String defaultValue) {
+        final String valueFromEnvironment = System.getProperty(variableName);
+        return valueFromEnvironment == null || valueFromEnvironment.isEmpty()? defaultValue : valueFromEnvironment;
+    }
+
+    /**
+     * name of the google cloud project that stores the data and will run the code
+     *
+     * @return PICARD_TEST_PROJECT env. var if defined or {@value #TEST_PROJECT_DEFAULT}
+     */
+    public static String getTestProject() {
+        return getSystemProperty("PICARD_TEST_PROJECT", TEST_PROJECT_DEFAULT);
+    }
+
+    /**
+     * A writable GCS path where java files can be cached and temporary test files can be written,
+     * of the form gs://bucket/, or gs://bucket/path/.
+     *
+     * @return PICARD_TEST_STAGING env. var if defined, or {@value #TEST_STAGING_DEFAULT}
+     */
+    public static String getTestStaging() {
+        return getSystemProperty("PICARD_TEST_STAGING", TEST_STAGING_DEFAULT);
+    }
+
+    /**
+     * A GCS path where the test inputs are stored.
+     * <p>
+     * The value of PICARD_TEST_INPUTS should end in a "/" (for example, "gs://hellbender/test/resources/")
+     *
+     * @return PICARD_TEST_INPUTS env. var if defined or {@value #TEST_INPUTS_DEFAULT}.
+     */
+    public static String getTestInputPath() {
+        return getSystemProperty("PICARD_TEST_INPUTS", TEST_INPUTS_DEFAULT);
+    }
+
+}

src/test/java/picard/util/GCloudTestUtilsUnitTest.java

@@ -0,0 +1,37 @@
+package picard.util;
+
+import htsjdk.samtools.util.IOUtil;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.stream.Stream;
+
+public class GCloudTestUtilsUnitTest {
+
+    @Test(groups = "bucket")
+    public void testDownload() throws IOException {
+        final String bigTextPath = GCloudTestUtils.getTestInputPath() + "nio/big.txt";
+        try(final Stream<String> lines = Files.lines(IOUtil.getPath(bigTextPath))){
+            String firstLine = lines.findFirst().orElseThrow();
+            Assert.assertEquals(firstLine, "The Project Gutenberg EBook of The Adventures of Sherlock Holmes");
+        }
+    }
+
+    @Test(groups = "bucket")
+    public void testUpload() throws IOException {
+        final Path uploadDir = Files.createTempDirectory(IOUtil.getPath(GCloudTestUtils.getTestStaging()), "picardTest");
+        final Path txtUpload = uploadDir.resolve("tmp.txt");
+        try {
+            Assert.assertFalse(Files.exists(txtUpload));
+            Files.writeString(txtUpload, "Hello there.");
+            Assert.assertEquals(Files.readString(txtUpload), "Hello there.");
+        } finally {
+            Files.delete(txtUpload);
+        }
+        Assert.assertFalse(Files.exists(txtUpload));
+        Assert.assertFalse(Files.exists(uploadDir));
+    }
+}