diff --git a/.github/workflows/cloud_tests.yml b/.github/workflows/cloud_tests.yml new file mode 100644 index 0000000000..24b32c4cc7 --- /dev/null +++ b/.github/workflows/cloud_tests.yml @@ -0,0 +1,93 @@ +name: Cloud Tests +on: + push: + branches: + - 'master' + pull_request: + workflow_dispatch: + +env: + PICARD_TEST_INPUTS: gs://hellbender/test/resources/ + PICARD_TEST_STAGING: gs://hellbender-test-logs/staging/ + PICARD_TEST_LOGS: /hellbender-test-logs/build_reports/ + PICARD_TEST_PROJECT: broad-dsde-dev + +jobs: + ## This workaround is necessary since there is no equivalent to the old TRAVIS_SECURE_ENVIRONMENT variable that indicated + ## if a run was privileged and had secrets. Since the GCP credentials are necessary for all tests in order to upload their, + ## results that makes them a reasonable proxy for testing the credentials of this entire execution. https://github.com/actions/runner/issues/520 + check-secrets: + name: check if the environment has privileges + outputs: + google-credentials: ${{ steps.google-credentials.outputs.defined }} + runs-on: ubuntu-latest + steps: + - id: google-credentials + env: + GCP_CREDENTIALS: ${{ secrets.GCP_CREDENTIALS }} + if: "${{ env.GCP_CREDENTIALS != '' }}" + run: echo defined=true >> $GITHUB_OUTPUT + + test: + runs-on: ubuntu-latest + needs: check-secrets + strategy: + matrix: + java: [ 17 ] + run_barclay_tests: [true, false] + experimental: [ false ] + fail-fast: false + continue-on-error: ${{ matrix.experimental }} + name: Java ${{ matrix.Java }}, Barclay=${{ matrix.run_barclay_tests}} cloud tests + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: 'Set up java ${{ matrix.Java }}' + uses: actions/setup-java@v3 + with: + java-version: ${{ matrix.Java }} + distribution: 'temurin' + cache: gradle + + - name: 'Compile with Gradle' + run: | + ./gradlew compileJava + ./gradlew installDist + + #Google Cloud stuff + - id: 'gcloud-auth' + if: needs.check-secrets.outputs.google-credentials == 'true' + uses: google-github-actions/auth@v0 + with: + credentials_json: ${{ secrets.GCP_CREDENTIALS }} + project_id: ${{ env.PICARD_TEST_PROJECT }} + create_credentials_file: true + + - name: 'Set up Cloud SDK' + if: needs.check-secrets.outputs.google-credentials == 'true' + uses: google-github-actions/setup-gcloud@v0 + + - name: compile test code + if: needs.check-secrets.outputs.google-credentials == 'true' + run: ./gradlew compileTestJava + + - name: Run tests + if: needs.check-secrets.outputs.google-credentials == 'true' + env: + TEST_TYPE: cloud + run: | + if [[ ${{matrix.run_barclay_tests}} == true ]]; then + echo "Running tests using the Barclay command line parser." + ./gradlew barclayTest + else + echo "Running tests using the legacy Picard command line parser." + ./gradlew jacocoTestReport + fi + + - name: Upload test results + if: always() + uses: actions/upload-artifact@v3 + with: + name: cloud-test-results-${{ matrix.Java }}-barclay-${{ matrix.run_barclay_tests}} + path: build/reports/tests \ No newline at end of file diff --git a/build.gradle b/build.gradle index 1ab49f2400..aa468bdeec 100644 --- a/build.gradle +++ b/build.gradle @@ -87,6 +87,7 @@ dependencies { compileOnly(googleNio) testImplementation 'org.testng:testng:6.14.3' + testImplementation(googleNio) implementation 'org.apache.commons:commons-lang3:3.6' } @@ -221,10 +222,21 @@ tasks.withType(Test) { ] useTestNG { - if (OperatingSystem.current().isUnix()) { - excludeGroups "slow", "broken" + List excludes = ["slow", "broken"] + if( !OperatingSystem.current().isUnix() ) { + excludes << "unix" + } + String TEST_TYPE = "$System.env.TEST_TYPE" + if (TEST_TYPE == "cloud") { + // run only the cloud tests + includeGroups "cloud", "bucket" + excludeGroups(*excludes) + } else if (TEST_TYPE == "all") { + //include everything + excludeGroups(*excludes) } else { - excludeGroups "slow", "broken", "unix" + excludes.addAll("cloud", "bucket") + excludeGroups(*excludes); } } diff --git a/src/test/java/picard/util/GCloudTestUtils.java b/src/test/java/picard/util/GCloudTestUtils.java new file mode 100644 index 0000000000..4e9df0cca0 --- /dev/null +++ b/src/test/java/picard/util/GCloudTestUtils.java @@ -0,0 +1,61 @@ +package picard.util; + +public final class GCloudTestUtils { + /** + * This is a public requester pays bucket owned by the broad-gatk-test project. + * It must be owned by a different project than the service account doing the testing or the test may fail because it can access the + * file directly through alternative permissions. + */ + public static final String REQUESTER_PAYS_BUCKET_DEFAULT = "gs://hellbender-requester-pays-test/"; + + public static final String TEST_INPUTS_DEFAULT = "gs://hellbender/test/resources/"; + public static final String TEST_STAGING_DEFAULT = "gs://hellbender-test-logs/staging/"; + public static final String TEST_PROJECT_DEFAULT = "broad-dsde-dev"; + + + /** + * A publicly readable GCS bucket set as requester pays, this should not be owned by the same project that is set + * as {@link #getTestProject()} or the tests for requester pays access may be invalid. + * + * @return PICARD_REQUESTER_PAYS_BUCKET env. var if defined, {@value GCloudTestUtils#REQUESTER_PAYS_BUCKET_DEFAULT}. + */ + public static String getRequesterPaysBucket() { + return getSystemProperty("PICARD_REQUESTER_PAYS_BUCKET", REQUESTER_PAYS_BUCKET_DEFAULT); + } + + private static String getSystemProperty(final String variableName, final String defaultValue) { + final String valueFromEnvironment = System.getProperty(variableName); + return valueFromEnvironment == null || valueFromEnvironment.isEmpty()? defaultValue : valueFromEnvironment; + } + + /** + * name of the google cloud project that stores the data and will run the code + * + * @return PICARD_TEST_PROJECT env. var if defined or {@value #TEST_PROJECT_DEFAULT} + */ + public static String getTestProject() { + return getSystemProperty("PICARD_TEST_PROJECT", TEST_PROJECT_DEFAULT); + } + + /** + * A writable GCS path where java files can be cached and temporary test files can be written, + * of the form gs://bucket/, or gs://bucket/path/. + * + * @return PICARD_TEST_STAGING env. var if defined, or {@value #TEST_STAGING_DEFAULT} + */ + public static String getTestStaging() { + return getSystemProperty("PICARD_TEST_STAGING", TEST_STAGING_DEFAULT); + } + + /** + * A GCS path where the test inputs are stored. + *
+ * The value of PICARD_TEST_INPUTS should end in a "/" (for example, "gs://hellbender/test/resources/")
+ *
+ * @return PICARD_TEST_INPUTS env. var if defined or {@value #TEST_INPUTS_DEFAULT}.
+ */
+ public static String getTestInputPath() {
+ return getSystemProperty("PICARD_TEST_INPUTS", TEST_INPUTS_DEFAULT);
+ }
+
+}
diff --git a/src/test/java/picard/util/GCloudTestUtilsUnitTest.java b/src/test/java/picard/util/GCloudTestUtilsUnitTest.java
new file mode 100644
index 0000000000..55a64eb4f7
--- /dev/null
+++ b/src/test/java/picard/util/GCloudTestUtilsUnitTest.java
@@ -0,0 +1,37 @@
+package picard.util;
+
+import htsjdk.samtools.util.IOUtil;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.stream.Stream;
+
+public class GCloudTestUtilsUnitTest {
+
+ @Test(groups = "bucket")
+ public void testDownload() throws IOException {
+ final String bigTextPath = GCloudTestUtils.getTestInputPath() + "nio/big.txt";
+ try(final Stream