From 8562fde8c34df6f8fa5751cc54ef4d869873b74d Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Wed, 11 Oct 2023 10:22:51 +0200 Subject: [PATCH] Add option for staging user-defined files (#130) * Add stage closure to config * Add `stage` to testsuite * Add missing test file --- docs/docs/configuration.md | 71 +++++++++++++ .../nf/test/commands/RunTestsCommand.java | 2 +- .../com/askimed/nf/test/config/Config.java | 12 +++ .../askimed/nf/test/config/FileStaging.java | 100 ++++++++++++++++++ .../askimed/nf/test/config/StageBuilder.java | 23 ++++ .../askimed/nf/test/core/AbstractTest.java | 47 +++++--- .../nf/test/core/AbstractTestSuite.java | 23 +++- .../java/com/askimed/nf/test/core/ITest.java | 4 +- .../askimed/nf/test/lang/WorkflowTest.java | 98 ++++++++++++----- test-assets/test.txt | 1 + .../workflow/staging/hello-stage.nf.test | 19 ++++ test-data/workflow/staging/hello.nf | 19 ++++ test-data/workflow/staging/hello.nf.test | 15 +++ .../workflow/staging/nf-test.file.config | 6 ++ .../workflow/staging/nf-test.file.mode.config | 7 ++ .../workflow/staging/nf-test.folder.config | 7 ++ 16 files changed, 410 insertions(+), 44 deletions(-) create mode 100644 src/main/java/com/askimed/nf/test/config/FileStaging.java create mode 100644 src/main/java/com/askimed/nf/test/config/StageBuilder.java create mode 100644 test-assets/test.txt create mode 100644 test-data/workflow/staging/hello-stage.nf.test create mode 100644 test-data/workflow/staging/hello.nf create mode 100644 test-data/workflow/staging/hello.nf.test create mode 100644 test-data/workflow/staging/nf-test.file.config create mode 100644 test-data/workflow/staging/nf-test.file.mode.config create mode 100644 test-data/workflow/staging/nf-test.folder.config diff --git a/docs/docs/configuration.md b/docs/docs/configuration.md index b3d7d49f..e5aadb8e 100644 --- a/docs/docs/configuration.md +++ b/docs/docs/configuration.md @@ -99,3 +99,74 @@ Profiles are evaluated in a specific order, ensuring predictable behavior: 3. **Profile Defined on the Command Line (CLI):** Finally, any profiles provided directly through the CLI have the highest priority and override/extends previously defined profiles. By understanding this profile evaluation order, you can effectively configure Nextflow executions for your test cases in a flexible and organized manner. + +## File Staging + +The `stage` section of the `nf-test.config` file is used to define files that are needed by Nextflow in the test environment (`meta` directory). Additionally, the directories `lib`, `bin`, and `assets` are automatically staged. + +### Supported Directives + +#### `symlink` + +This directive is used to create symbolic links (symlinks) in the test environment. Symlinks are pointers to files or directories and can be useful for creating references to data files or directories required for the test. The syntax for the `symlink` directive is as follows: + +``` +symlink "source_path" +``` + +`source_path`: The path to the source file or directory that you want to symlink. + +#### `copy` + +This directive is used to copy files or directories into the test environment. It allows you to duplicate files from a specified source to a location within the test environment. The syntax for the `copy` directive is as follows: + +``` +copy "source_path" +``` + +`source_path`: The path to the source file or directory that you want to copy. + +### Example Usage + +Here's an example of how to use the `stage` section in an `nf-test.config` file: + +```groovy +config { + ... + stage { + symlink "data/original_data.txt" + copy "resources/config.yml" + } + ... +} +``` + +In this example: + +- The `symlink` directive creates a symlink named "original_data.txt" in the `meta` directory pointing to the file located at "data/original_data.txt." +- The `copy` directive copies the "config.yml" file from the "resources" directory to the `meta` directory. + +### Testsuite + +Furthermore, it is also possible to stage files that are specific to a single testsuite: + +``` +nextflow_workflow { + + name "Test workflow HELLO_WORKFLOW" + + script "./hello.nf" + workflow "HELLO_WORKFLOW" + + stage { + symlink "test-assets/test.txt" + } + + test("Should print out test file") { + expect { + assert workflow.success + } + } + +} +``` diff --git a/src/main/java/com/askimed/nf/test/commands/RunTestsCommand.java b/src/main/java/com/askimed/nf/test/commands/RunTestsCommand.java index 915a6982..43705551 100644 --- a/src/main/java/com/askimed/nf/test/commands/RunTestsCommand.java +++ b/src/main/java/com/askimed/nf/test/commands/RunTestsCommand.java @@ -102,7 +102,7 @@ public Integer execute() throws Exception { try { File configFile = new File(configFilename); if (configFile.exists()) { - + log.info("Load config from file {}...", configFile.getAbsolutePath()); Config config = Config.parse(configFile); defaultConfigFile = config.getConfigFile(); defaultWithTrace = config.isWithTrace(); diff --git a/src/main/java/com/askimed/nf/test/config/Config.java b/src/main/java/com/askimed/nf/test/config/Config.java index 931980b3..d8e7d375 100644 --- a/src/main/java/com/askimed/nf/test/config/Config.java +++ b/src/main/java/com/askimed/nf/test/config/Config.java @@ -36,6 +36,8 @@ public class Config { private String configFile = DEFAULT_NEXTFLOW_CONFIG; + private StageBuilder stageBuilder = new StageBuilder(); + public void testsDir(String testsDir) { this.testsDir = testsDir; } @@ -112,6 +114,16 @@ public String getLibDir() { return libDir; } + public void stage(Closure closure) { + closure.setDelegate(stageBuilder); + closure.setResolveStrategy(Closure.DELEGATE_ONLY); + closure.call(); + } + + public StageBuilder getStageBuilder() { + return stageBuilder; + } + public void configFile(String config) { this.configFile = config; } diff --git a/src/main/java/com/askimed/nf/test/config/FileStaging.java b/src/main/java/com/askimed/nf/test/config/FileStaging.java new file mode 100644 index 00000000..9c596c1f --- /dev/null +++ b/src/main/java/com/askimed/nf/test/config/FileStaging.java @@ -0,0 +1,100 @@ +package com.askimed.nf.test.config; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.askimed.nf.test.util.FileUtil; + +public class FileStaging { + + public static String MODE_COPY = "copy"; + + public static String MODE_SYMLINK = "symlink"; + + private String path = ""; + + private String mode = MODE_SYMLINK; + + private static Logger log = LoggerFactory.getLogger(FileStaging.class); + + public FileStaging() { + + } + + public FileStaging(String path) { + this(path, MODE_SYMLINK); + } + + public FileStaging(String path, String mode) { + this.path = path; + this.mode = mode; + } + + public void setPath(String path) { + this.path = path; + } + + public String getPath() { + return path; + } + + public void setMode(String mode) { + this.mode = mode; + } + + public String getMode() { + return mode; + } + + public void stage(String target) throws IOException { + + if (path == null) { + throw new IOException("No path set."); + } + + Path localFile = Path.of(path); + + if (localFile.toFile().exists()) { + + String parent = new File(target).getParentFile().getAbsolutePath(); + if (parent != null) { + FileUtil.createDirectory(parent); + } + + if (localFile.toFile().isDirectory()) { + stageDirectory(target, localFile); + } else { + stageFile(target, localFile); + } + + } else { + log.warn("File '{}' not found. Ignore it.", localFile.toFile().getAbsolutePath()); + } + } + + private void stageFile(String target, Path localFile) throws IOException { + if (mode.equalsIgnoreCase(MODE_COPY)) { + log.info("Copy file '{}' to '{}'", localFile.toFile().getAbsolutePath(), target); + Files.copy(localFile.toFile().toPath(), Path.of(target)); + } else if (mode.equalsIgnoreCase(MODE_SYMLINK)) { + log.info("Create symlink '{}' --> '{}'", target, localFile.toFile().getAbsolutePath()); + Files.createSymbolicLink(Path.of(target), localFile.toAbsolutePath()); + } + } + + private void stageDirectory(String target, Path localFile) throws IOException { + if (mode.equalsIgnoreCase(MODE_COPY)) { + log.info("Copy directory '{}' to '{}'", localFile.toFile().getAbsolutePath(), target); + FileUtil.copyDirectory(localFile.toFile().getAbsolutePath(), target); + } else if (mode.equalsIgnoreCase(MODE_SYMLINK)) { + log.info("Create symlink '{}' --> '{}'", target, localFile.toFile().getAbsolutePath()); + Files.createSymbolicLink(Path.of(target), localFile.toAbsolutePath()); + } + } + +} diff --git a/src/main/java/com/askimed/nf/test/config/StageBuilder.java b/src/main/java/com/askimed/nf/test/config/StageBuilder.java new file mode 100644 index 00000000..7c6dfcb8 --- /dev/null +++ b/src/main/java/com/askimed/nf/test/config/StageBuilder.java @@ -0,0 +1,23 @@ +package com.askimed.nf.test.config; + +import java.util.List; +import java.util.Vector; + +public class StageBuilder { + + private List paths = new Vector(); + + public void copy(String path) { + paths.add(new FileStaging(path, FileStaging.MODE_COPY)); + } + + public void symlink(String path) { + paths.add(new FileStaging(path, FileStaging.MODE_SYMLINK)); + } + + + public List getPaths() { + return paths; + } + +} diff --git a/src/main/java/com/askimed/nf/test/core/AbstractTest.java b/src/main/java/com/askimed/nf/test/core/AbstractTest.java index 0f6484b7..3cc6e71d 100644 --- a/src/main/java/com/askimed/nf/test/core/AbstractTest.java +++ b/src/main/java/com/askimed/nf/test/core/AbstractTest.java @@ -8,6 +8,11 @@ import java.util.List; import java.util.Vector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.askimed.nf.test.config.Config; +import com.askimed.nf.test.config.FileStaging; import com.askimed.nf.test.util.FileUtil; public abstract class AbstractTest implements ITest { @@ -54,7 +59,8 @@ public abstract class AbstractTest implements ITest { public boolean skipped = false; - public static String[] SHARED_DIRECTORIES = { "bin", "lib", "assets" }; + public static FileStaging[] SHARED_DIRECTORIES = { new FileStaging("bin"), new FileStaging("lib"), + new FileStaging("assets") }; protected File config = null; @@ -70,6 +76,8 @@ public abstract class AbstractTest implements ITest { private String options; + private static Logger log = LoggerFactory.getLogger(AbstractTest.class); + public AbstractTest(AbstractTestSuite parent) { this.parent = parent; options = parent.getOptions(); @@ -84,21 +92,26 @@ public File getConfig() { } @Override - public void setup(File testDirectory) throws IOException { + public void setup(Config config, File testDirectory) throws IOException { if (testDirectory == null) { throw new IOException("Testcase setup failed: No home directory set"); } - + launchDir = initDirectory("Launch Directory", testDirectory, DIRECTORY_TESTS, getHash()); metaDir = initDirectory("Meta Directory", launchDir, DIRECTORY_META); outputDir = initDirectory("Output Directory", launchDir, DIRECTORY_OUTPUT); workDir = initDirectory("Working Directory", launchDir, DIRECTORY_WORK); try { - // copy bin and lib to metaDir. TODO: use symlinks and read additional "mapping" - // from config file + // copy bin, assets and lib to metaDir shareDirectories(SHARED_DIRECTORIES, metaDir); + if (config != null) { + // copy user defined staging directories + log.debug("Stage {} user provided files...", config.getStageBuilder().getPaths().size()); + shareDirectories(config.getStageBuilder().getPaths(), metaDir); + } + shareDirectories(parent.getStageBuilder().getPaths(), metaDir); } catch (Exception e) { throw new IOException("Testcase setup failed: Directories could not be shared:\n" + e); } @@ -154,11 +167,11 @@ public String getErrorReport() throws Throwable { @Override public String getHash() { - + if (parent == null || parent.getFilename() == null || getName() == null || getName().isEmpty()) { throw new RuntimeException("Error generating hash"); } - + return hash(parent.getFilename() + getName()); } @@ -236,13 +249,17 @@ public boolean isWithTrace() { return withTrace; } - protected void shareDirectories(String[] directories, File metaDir) throws IOException { - for (String directory : directories) { - File localDirectory = new File(directory); - if (localDirectory.exists()) { - String metaDirectory = FileUtil.path(metaDir.getAbsolutePath(), directory); - FileUtil.copyDirectory(localDirectory.getAbsolutePath(), metaDirectory); - } + protected void shareDirectories(List directories, File stageDir) throws IOException { + for (FileStaging directory : directories) { + String metaDirectory = FileUtil.path(stageDir.getAbsolutePath(), directory.getPath()); + directory.stage(metaDirectory); + } + } + + protected void shareDirectories(FileStaging[] directories, File stageDir) throws IOException { + for (FileStaging directory : directories) { + String metaDirectory = FileUtil.path(stageDir.getAbsolutePath(), directory.getPath()); + directory.stage(metaDirectory); } } @@ -254,7 +271,7 @@ public void setUpdateSnapshot(boolean updateSnapshot) { public boolean isUpdateSnapshot() { return updateSnapshot; } - + @Override public String toString() { return getHash().substring(0, 8) + ": " + getName(); diff --git a/src/main/java/com/askimed/nf/test/core/AbstractTestSuite.java b/src/main/java/com/askimed/nf/test/core/AbstractTestSuite.java index 21341a5d..3b7c7af2 100644 --- a/src/main/java/com/askimed/nf/test/core/AbstractTestSuite.java +++ b/src/main/java/com/askimed/nf/test/core/AbstractTestSuite.java @@ -5,6 +5,7 @@ import java.util.Vector; import com.askimed.nf.test.config.Config; +import com.askimed.nf.test.config.StageBuilder; import com.askimed.nf.test.lang.extensions.SnapshotFile; import groovy.lang.Closure; @@ -41,8 +42,13 @@ public abstract class AbstractTestSuite implements ITestSuite { private List testClosures = new Vector(); + private StageBuilder stageBuilder = new StageBuilder(); + + private Config config; + @Override public void configure(Config config) { + this.config = config; autoSort = config.isAutoSort(); options = config.getOptions(); homeDirectory = new File(config.getWorkDir()); @@ -70,12 +76,13 @@ protected void addTestClosure(String name, Closure closure) { } public void evalualteTestClosures() throws Throwable { + for (NamedClosure namedClosure : testClosures) { String testName = namedClosure.name; Closure closure = namedClosure.closure; - + ITest test = getNewTestInstance(testName); - test.setup(getHomeDirectory()); + test.setup(config, getHomeDirectory()); closure.setDelegate(test); closure.setResolveStrategy(Closure.DELEGATE_ONLY); closure.call(); @@ -133,6 +140,16 @@ public String getOptions() { return options; } + public void stage(Closure closure) { + closure.setDelegate(stageBuilder); + closure.setResolveStrategy(Closure.DELEGATE_ONLY); + closure.call(); + } + + public StageBuilder getStageBuilder() { + return stageBuilder; + } + @Override public void setGlobalConfigFile(File globalConfig) { this.globalConfig = globalConfig; @@ -233,7 +250,7 @@ protected String makeAbsolute(String path) { protected boolean isRelative(String path) { return path.startsWith("../") || path.startsWith("./"); } - + @Override public String toString() { return name; diff --git a/src/main/java/com/askimed/nf/test/core/ITest.java b/src/main/java/com/askimed/nf/test/core/ITest.java index 17d997f1..40c9497d 100644 --- a/src/main/java/com/askimed/nf/test/core/ITest.java +++ b/src/main/java/com/askimed/nf/test/core/ITest.java @@ -2,9 +2,11 @@ import java.io.File; +import com.askimed.nf.test.config.Config; + public interface ITest extends ITaggable { - public void setup(File homeDirectory) throws Throwable; + public void setup(Config config, File homeDirectory) throws Throwable; public void execute() throws Throwable; diff --git a/src/test/java/com/askimed/nf/test/lang/WorkflowTest.java b/src/test/java/com/askimed/nf/test/lang/WorkflowTest.java index 647d974b..bd009076 100644 --- a/src/test/java/com/askimed/nf/test/lang/WorkflowTest.java +++ b/src/test/java/com/askimed/nf/test/lang/WorkflowTest.java @@ -34,7 +34,7 @@ public void testWorkflowSucces() throws Exception { assertEquals(0, exitCode); } - + @Test public void testWorkflowWithRelativePath() throws Exception { @@ -43,15 +43,15 @@ public void testWorkflowWithRelativePath() throws Exception { assertEquals(0, exitCode); } - + @Test - public void testWorkflowUnamedOutputs() throws Exception { + public void testWorkflowUnamedOutputs() throws Exception { App app = new App(); int exitCode = app.run(new String[] { "test", "test-data/workflow/unnamed/trial.unnamed.nf.test" }); assertEquals(0, exitCode); } - + @Test public void testWorkflowAndSnapshot() throws Exception { @@ -60,7 +60,7 @@ public void testWorkflowAndSnapshot() throws Exception { assertEquals(0, exitCode); } - + @Test public void testWorkflowWithNoOutputs() throws Exception { @@ -83,35 +83,37 @@ public void testOverrideWorkflow() throws Exception { public void testLibs() throws Exception { App app = new App(); - int exitCode = app.run(new String[] { "test", "test-data/workflow/libs/hello.nf.test", "--lib", "lib" ,"--debug"}); + int exitCode = app + .run(new String[] { "test", "test-data/workflow/libs/hello.nf.test", "--lib", "lib", "--debug" }); assertEquals(0, exitCode); } - + @Test public void testParamsIssue34() throws Exception { - App app = new App(); - int exitCode = app.run(new String[] { "test", "test-data/workflow/issue34/trial.nf.test"}); - assertEquals(0, exitCode); + App app = new App(); + int exitCode = app.run(new String[] { "test", "test-data/workflow/issue34/trial.nf.test" }); + assertEquals(0, exitCode); } - + @Test public void testParamsIssue34Setup() throws Exception { - App app = new App(); - int exitCode = app.run(new String[] { "test", "test-data/workflow/issue34/trial.setup.nf.test"}); - assertEquals(0, exitCode); + App app = new App(); + int exitCode = app.run(new String[] { "test", "test-data/workflow/issue34/trial.setup.nf.test" }); + assertEquals(0, exitCode); } - + @Test public void testHangingWorkflowIssue57() throws Exception { - App app = new App(); - int exitCode = app.run(new String[] { "test", "test-data/workflow/hanging/meaningless_workflow.nf.test","--debug"}); - assertEquals(0, exitCode); + App app = new App(); + int exitCode = app + .run(new String[] { "test", "test-data/workflow/hanging/meaningless_workflow.nf.test", "--debug" }); + assertEquals(0, exitCode); } @@ -119,18 +121,66 @@ public void testHangingWorkflowIssue57() throws Exception { public void testWorkflowNonUniqueFilenames() throws Exception { App app = new App(); - int exitCode = app.run(new String[] { "test", "test-data/workflow/non-unique-filenames/main.nf.test"}); + int exitCode = app.run(new String[] { "test", "test-data/workflow/non-unique-filenames/main.nf.test" }); assertEquals(0, exitCode); } - - + @Test public void testIssue125() throws Exception { - App app = new App(); - int exitCode = app.run(new String[] { "test", "test-data/workflow/issue125/example_wf.nf.test"}); - assertEquals(0, exitCode); + App app = new App(); + int exitCode = app.run(new String[] { "test", "test-data/workflow/issue125/example_wf.nf.test" }); + assertEquals(0, exitCode); + + } + + @Test + public void testStagingWithoutMapping() throws Exception { + + App app = new App(); + int exitCode = app.run(new String[] { "test", "test-data/workflow/staging/hello.nf.test" }); + assertEquals(1, exitCode); + + } + + @Test + public void testStagingWitMappingFolder() throws Exception { + + App app = new App(); + int exitCode = app.run(new String[] { "test", "test-data/workflow/staging/hello.nf.test", "--config", + "test-data/workflow/staging/nf-test.folder.config", "--debug" }); + assertEquals(0, exitCode); + + } + + @Test + public void testStagingWitMappingFile() throws Exception { + + App app = new App(); + int exitCode = app.run(new String[] { "test", "test-data/workflow/staging/hello.nf.test", "--config", + "test-data/workflow/staging/nf-test.file.config" }); + assertEquals(0, exitCode); + + } + + @Test + public void testStagingWitMappingFileAndMode() throws Exception { + + App app = new App(); + int exitCode = app.run(new String[] { "test", "test-data/workflow/staging/hello.nf.test", "--config", + "test-data/workflow/staging/nf-test.file.mode.config" }); + assertEquals(0, exitCode); } + + @Test + public void testStagingInTestsuite() throws Exception { + + App app = new App(); + int exitCode = app.run(new String[] { "test", "test-data/workflow/staging/hello-stage.nf.test" }); + assertEquals(0, exitCode); + + } + } diff --git a/test-assets/test.txt b/test-assets/test.txt new file mode 100644 index 00000000..82d053fb --- /dev/null +++ b/test-assets/test.txt @@ -0,0 +1 @@ +hello nextflow! \ No newline at end of file diff --git a/test-data/workflow/staging/hello-stage.nf.test b/test-data/workflow/staging/hello-stage.nf.test new file mode 100644 index 00000000..9ebf10b8 --- /dev/null +++ b/test-data/workflow/staging/hello-stage.nf.test @@ -0,0 +1,19 @@ +nextflow_workflow { + + name "Test workflow HELLO_WORKFLOW" + + script "./hello.nf" + workflow "HELLO_WORKFLOW" + + stage { + symlink "test-assets/test.txt" + } + + test("Should print out test file") { + expect { + assert workflow.success + } + + } + +} diff --git a/test-data/workflow/staging/hello.nf b/test-data/workflow/staging/hello.nf new file mode 100644 index 00000000..45ffc215 --- /dev/null +++ b/test-data/workflow/staging/hello.nf @@ -0,0 +1,19 @@ +process HELLO { + + input: + file input + + script: + """ + cat ${input} + """ + +} + +workflow HELLO_WORKFLOW { + + HELLO( + file("${baseDir}/test-assets/test.txt") + ) + +} \ No newline at end of file diff --git a/test-data/workflow/staging/hello.nf.test b/test-data/workflow/staging/hello.nf.test new file mode 100644 index 00000000..50c37914 --- /dev/null +++ b/test-data/workflow/staging/hello.nf.test @@ -0,0 +1,15 @@ +nextflow_workflow { + + name "Test workflow HELLO_WORKFLOW" + + script "./hello.nf" + workflow "HELLO_WORKFLOW" + + test("Should print out test file") { + expect { + assert workflow.success + } + + } + +} diff --git a/test-data/workflow/staging/nf-test.file.config b/test-data/workflow/staging/nf-test.file.config new file mode 100644 index 00000000..c0e60b55 --- /dev/null +++ b/test-data/workflow/staging/nf-test.file.config @@ -0,0 +1,6 @@ +config { + configFile "" + stage { + symlink "test-assets/test.txt" + } +} \ No newline at end of file diff --git a/test-data/workflow/staging/nf-test.file.mode.config b/test-data/workflow/staging/nf-test.file.mode.config new file mode 100644 index 00000000..1ec296ca --- /dev/null +++ b/test-data/workflow/staging/nf-test.file.mode.config @@ -0,0 +1,7 @@ +config { + configFile "" + stage { + symlink "rrr" + copy "test-assets/test.txt" + } +} \ No newline at end of file diff --git a/test-data/workflow/staging/nf-test.folder.config b/test-data/workflow/staging/nf-test.folder.config new file mode 100644 index 00000000..2e0397b4 --- /dev/null +++ b/test-data/workflow/staging/nf-test.folder.config @@ -0,0 +1,7 @@ +config { + configFile "" + stage { + copy 'test-assets' + symlink 'missing' + } +} \ No newline at end of file