diff --git a/.github/workflows/cloud_tests.yml b/.github/workflows/cloud_tests.yml new file mode 100644 index 0000000000..24b32c4cc7 --- /dev/null +++ b/.github/workflows/cloud_tests.yml @@ -0,0 +1,93 @@ +name: Cloud Tests +on: + push: + branches: + - 'master' + pull_request: + workflow_dispatch: + +env: + PICARD_TEST_INPUTS: gs://hellbender/test/resources/ + PICARD_TEST_STAGING: gs://hellbender-test-logs/staging/ + PICARD_TEST_LOGS: /hellbender-test-logs/build_reports/ + PICARD_TEST_PROJECT: broad-dsde-dev + +jobs: + ## This workaround is necessary since there is no equivalent to the old TRAVIS_SECURE_ENVIRONMENT variable that indicated + ## if a run was privileged and had secrets. Since the GCP credentials are necessary for all tests in order to upload their, + ## results that makes them a reasonable proxy for testing the credentials of this entire execution. https://github.com/actions/runner/issues/520 + check-secrets: + name: check if the environment has privileges + outputs: + google-credentials: ${{ steps.google-credentials.outputs.defined }} + runs-on: ubuntu-latest + steps: + - id: google-credentials + env: + GCP_CREDENTIALS: ${{ secrets.GCP_CREDENTIALS }} + if: "${{ env.GCP_CREDENTIALS != '' }}" + run: echo defined=true >> $GITHUB_OUTPUT + + test: + runs-on: ubuntu-latest + needs: check-secrets + strategy: + matrix: + java: [ 17 ] + run_barclay_tests: [true, false] + experimental: [ false ] + fail-fast: false + continue-on-error: ${{ matrix.experimental }} + name: Java ${{ matrix.Java }}, Barclay=${{ matrix.run_barclay_tests}} cloud tests + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: 'Set up java ${{ matrix.Java }}' + uses: actions/setup-java@v3 + with: + java-version: ${{ matrix.Java }} + distribution: 'temurin' + cache: gradle + + - name: 'Compile with Gradle' + run: | + ./gradlew compileJava + ./gradlew installDist + + #Google Cloud stuff + - id: 'gcloud-auth' + if: needs.check-secrets.outputs.google-credentials == 'true' + uses: google-github-actions/auth@v0 + with: + credentials_json: ${{ secrets.GCP_CREDENTIALS }} + project_id: ${{ env.PICARD_TEST_PROJECT }} + create_credentials_file: true + + - name: 'Set up Cloud SDK' + if: needs.check-secrets.outputs.google-credentials == 'true' + uses: google-github-actions/setup-gcloud@v0 + + - name: compile test code + if: needs.check-secrets.outputs.google-credentials == 'true' + run: ./gradlew compileTestJava + + - name: Run tests + if: needs.check-secrets.outputs.google-credentials == 'true' + env: + TEST_TYPE: cloud + run: | + if [[ ${{matrix.run_barclay_tests}} == true ]]; then + echo "Running tests using the Barclay command line parser." + ./gradlew barclayTest + else + echo "Running tests using the legacy Picard command line parser." + ./gradlew jacocoTestReport + fi + + - name: Upload test results + if: always() + uses: actions/upload-artifact@v3 + with: + name: cloud-test-results-${{ matrix.Java }}-barclay-${{ matrix.run_barclay_tests}} + path: build/reports/tests \ No newline at end of file diff --git a/build.gradle b/build.gradle index 1ab49f2400..aa468bdeec 100644 --- a/build.gradle +++ b/build.gradle @@ -87,6 +87,7 @@ dependencies { compileOnly(googleNio) testImplementation 'org.testng:testng:6.14.3' + testImplementation(googleNio) implementation 'org.apache.commons:commons-lang3:3.6' } @@ -221,10 +222,21 @@ tasks.withType(Test) { ] useTestNG { - if (OperatingSystem.current().isUnix()) { - excludeGroups "slow", "broken" + List excludes = ["slow", "broken"] + if( !OperatingSystem.current().isUnix() ) { + excludes << "unix" + } + String TEST_TYPE = "$System.env.TEST_TYPE" + if (TEST_TYPE == "cloud") { + // run only the cloud tests + includeGroups "cloud", "bucket" + excludeGroups(*excludes) + } else if (TEST_TYPE == "all") { + //include everything + excludeGroups(*excludes) } else { - excludeGroups "slow", "broken", "unix" + excludes.addAll("cloud", "bucket") + excludeGroups(*excludes); } } diff --git a/src/test/java/picard/util/GCloudTestUtils.java b/src/test/java/picard/util/GCloudTestUtils.java new file mode 100644 index 0000000000..4e9df0cca0 --- /dev/null +++ b/src/test/java/picard/util/GCloudTestUtils.java @@ -0,0 +1,61 @@ +package picard.util; + +public final class GCloudTestUtils { + /** + * This is a public requester pays bucket owned by the broad-gatk-test project. + * It must be owned by a different project than the service account doing the testing or the test may fail because it can access the + * file directly through alternative permissions. + */ + public static final String REQUESTER_PAYS_BUCKET_DEFAULT = "gs://hellbender-requester-pays-test/"; + + public static final String TEST_INPUTS_DEFAULT = "gs://hellbender/test/resources/"; + public static final String TEST_STAGING_DEFAULT = "gs://hellbender-test-logs/staging/"; + public static final String TEST_PROJECT_DEFAULT = "broad-dsde-dev"; + + + /** + * A publicly readable GCS bucket set as requester pays, this should not be owned by the same project that is set + * as {@link #getTestProject()} or the tests for requester pays access may be invalid. + * + * @return PICARD_REQUESTER_PAYS_BUCKET env. var if defined, {@value GCloudTestUtils#REQUESTER_PAYS_BUCKET_DEFAULT}. + */ + public static String getRequesterPaysBucket() { + return getSystemProperty("PICARD_REQUESTER_PAYS_BUCKET", REQUESTER_PAYS_BUCKET_DEFAULT); + } + + private static String getSystemProperty(final String variableName, final String defaultValue) { + final String valueFromEnvironment = System.getProperty(variableName); + return valueFromEnvironment == null || valueFromEnvironment.isEmpty()? defaultValue : valueFromEnvironment; + } + + /** + * name of the google cloud project that stores the data and will run the code + * + * @return PICARD_TEST_PROJECT env. var if defined or {@value #TEST_PROJECT_DEFAULT} + */ + public static String getTestProject() { + return getSystemProperty("PICARD_TEST_PROJECT", TEST_PROJECT_DEFAULT); + } + + /** + * A writable GCS path where java files can be cached and temporary test files can be written, + * of the form gs://bucket/, or gs://bucket/path/. + * + * @return PICARD_TEST_STAGING env. var if defined, or {@value #TEST_STAGING_DEFAULT} + */ + public static String getTestStaging() { + return getSystemProperty("PICARD_TEST_STAGING", TEST_STAGING_DEFAULT); + } + + /** + * A GCS path where the test inputs are stored. + *

+ * The value of PICARD_TEST_INPUTS should end in a "/" (for example, "gs://hellbender/test/resources/") + * + * @return PICARD_TEST_INPUTS env. var if defined or {@value #TEST_INPUTS_DEFAULT}. + */ + public static String getTestInputPath() { + return getSystemProperty("PICARD_TEST_INPUTS", TEST_INPUTS_DEFAULT); + } + +} diff --git a/src/test/java/picard/util/GCloudTestUtilsUnitTest.java b/src/test/java/picard/util/GCloudTestUtilsUnitTest.java new file mode 100644 index 0000000000..55a64eb4f7 --- /dev/null +++ b/src/test/java/picard/util/GCloudTestUtilsUnitTest.java @@ -0,0 +1,37 @@ +package picard.util; + +import htsjdk.samtools.util.IOUtil; +import org.testng.Assert; +import org.testng.annotations.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.stream.Stream; + +public class GCloudTestUtilsUnitTest { + + @Test(groups = "bucket") + public void testDownload() throws IOException { + final String bigTextPath = GCloudTestUtils.getTestInputPath() + "nio/big.txt"; + try(final Stream lines = Files.lines(IOUtil.getPath(bigTextPath))){ + String firstLine = lines.findFirst().orElseThrow(); + Assert.assertEquals(firstLine, "The Project Gutenberg EBook of The Adventures of Sherlock Holmes"); + } + } + + @Test(groups = "bucket") + public void testUpload() throws IOException { + final Path uploadDir = Files.createTempDirectory(IOUtil.getPath(GCloudTestUtils.getTestStaging()), "picardTest"); + final Path txtUpload = uploadDir.resolve("tmp.txt"); + try { + Assert.assertFalse(Files.exists(txtUpload)); + Files.writeString(txtUpload, "Hello there."); + Assert.assertEquals(Files.readString(txtUpload), "Hello there."); + } finally { + Files.delete(txtUpload); + } + Assert.assertFalse(Files.exists(txtUpload)); + Assert.assertFalse(Files.exists(uploadDir)); + } +} \ No newline at end of file