Add option for staging user-defined files (#130)

* Add stage closure to config

* Add `stage` to testsuite

* Add missing test file

docs/docs/configuration.md

@@ -99,3 +99,74 @@ Profiles are evaluated in a specific order, ensuring predictable behavior:
 3. **Profile Defined on the Command Line (CLI):** Finally, any profiles provided directly through the CLI have the highest priority and override/extends previously defined profiles.
 
 By understanding this profile evaluation order, you can effectively configure Nextflow executions for your test cases in a flexible and organized manner.
+
+## File Staging
+
+The `stage` section of the `nf-test.config` file is used to define files that are needed by Nextflow in the test environment (`meta` directory). Additionally, the directories `lib`, `bin`, and `assets` are automatically staged.
+
+### Supported Directives
+
+#### `symlink`
+
+This directive is used to create symbolic links (symlinks) in the test environment. Symlinks are pointers to files or directories and can be useful for creating references to data files or directories required for the test. The syntax for the `symlink` directive is as follows:
+
+```
+symlink "source_path"
+```
+
+`source_path`: The path to the source file or directory that you want to symlink.
+
+#### `copy`
+
+This directive is used to copy files or directories into the test environment. It allows you to duplicate files from a specified source to a location within the test environment. The syntax for the `copy` directive is as follows:
+
+```
+copy "source_path"
+```
+
+`source_path`: The path to the source file or directory that you want to copy.
+
+### Example Usage
+
+Here's an example of how to use the `stage` section in an `nf-test.config` file:
+
+```groovy
+config {
+    ...
+    stage {
+        symlink "data/original_data.txt"
+        copy "resources/config.yml"
+    }
+    ...
+}
+```
+
+In this example:
+
+- The `symlink` directive creates a symlink named "original_data.txt" in the `meta` directory pointing to the file located at "data/original_data.txt."
+- The `copy` directive copies the "config.yml" file from the "resources" directory to the `meta` directory.
+
+### Testsuite
+
+Furthermore, it is also possible to stage files that are specific to a single testsuite:
+
+```
+nextflow_workflow {
+
+    name "Test workflow HELLO_WORKFLOW"
+
+    script "./hello.nf"
+    workflow "HELLO_WORKFLOW"
+
+    stage {
+        symlink "test-assets/test.txt"
+    }
+
+    test("Should print out test file") {
+        expect {
+            assert workflow.success
+        }
+    }
+
+}
+```

src/main/java/com/askimed/nf/test/commands/RunTestsCommand.java

@@ -102,7 +102,7 @@ public Integer execute() throws Exception {
 			try {
 				File configFile = new File(configFilename);
 				if (configFile.exists()) {
-
+					log.info("Load config from file {}...", configFile.getAbsolutePath());
 					Config config = Config.parse(configFile);
 					defaultConfigFile = config.getConfigFile();
 					defaultWithTrace = config.isWithTrace();

src/main/java/com/askimed/nf/test/config/Config.java

@@ -36,6 +36,8 @@ public class Config {
 
 	private String configFile = DEFAULT_NEXTFLOW_CONFIG;
 
+	private StageBuilder stageBuilder = new StageBuilder();
+
 	public void testsDir(String testsDir) {
 		this.testsDir = testsDir;
 	}
@@ -112,6 +114,16 @@ public String getLibDir() {
 		return libDir;
 	}
 
+	public void stage(Closure closure) {
+		closure.setDelegate(stageBuilder);
+		closure.setResolveStrategy(Closure.DELEGATE_ONLY);		
+		closure.call();
+	}
+
+	public StageBuilder getStageBuilder() {
+		return stageBuilder;
+	}
+
 	public void configFile(String config) {
 		this.configFile = config;
 	}

src/main/java/com/askimed/nf/test/config/FileStaging.java

@@ -0,0 +1,100 @@
+package com.askimed.nf.test.config;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.askimed.nf.test.util.FileUtil;
+
+public class FileStaging {
+
+	public static String MODE_COPY = "copy";
+
+	public static String MODE_SYMLINK = "symlink";
+
+	private String path = "";
+
+	private String mode = MODE_SYMLINK;
+
+	private static Logger log = LoggerFactory.getLogger(FileStaging.class);
+
+	public FileStaging() {
+
+	}
+
+	public FileStaging(String path) {
+		this(path, MODE_SYMLINK);
+	}
+
+	public FileStaging(String path, String mode) {
+		this.path = path;
+		this.mode = mode;
+	}
+
+	public void setPath(String path) {
+		this.path = path;
+	}
+
+	public String getPath() {
+		return path;
+	}
+
+	public void setMode(String mode) {
+		this.mode = mode;
+	}
+
+	public String getMode() {
+		return mode;
+	}
+
+	public void stage(String target) throws IOException {
+
+		if (path == null) {
+			throw new IOException("No path set.");
+		}
+
+		Path localFile = Path.of(path);
+
+		if (localFile.toFile().exists()) {
+
+			String parent = new File(target).getParentFile().getAbsolutePath();
+			if (parent != null) {
+				FileUtil.createDirectory(parent);
+			}
+
+			if (localFile.toFile().isDirectory()) {
+				stageDirectory(target, localFile);
+			} else {
+				stageFile(target, localFile);
+			}
+
+		} else {
+			log.warn("File '{}' not found. Ignore it.", localFile.toFile().getAbsolutePath());
+		}
+	}
+
+	private void stageFile(String target, Path localFile) throws IOException {
+		if (mode.equalsIgnoreCase(MODE_COPY)) {
+			log.info("Copy file '{}' to '{}'", localFile.toFile().getAbsolutePath(), target);
+			Files.copy(localFile.toFile().toPath(), Path.of(target));
+		} else if (mode.equalsIgnoreCase(MODE_SYMLINK)) {
+			log.info("Create symlink '{}' --> '{}'", target, localFile.toFile().getAbsolutePath());
+			Files.createSymbolicLink(Path.of(target), localFile.toAbsolutePath());
+		}
+	}
+
+	private void stageDirectory(String target, Path localFile) throws IOException {
+		if (mode.equalsIgnoreCase(MODE_COPY)) {
+			log.info("Copy directory '{}' to '{}'", localFile.toFile().getAbsolutePath(), target);
+			FileUtil.copyDirectory(localFile.toFile().getAbsolutePath(), target);
+		} else if (mode.equalsIgnoreCase(MODE_SYMLINK)) {
+			log.info("Create symlink '{}' --> '{}'", target, localFile.toFile().getAbsolutePath());
+			Files.createSymbolicLink(Path.of(target), localFile.toAbsolutePath());
+		}
+	}
+
+}

src/main/java/com/askimed/nf/test/config/StageBuilder.java

@@ -0,0 +1,23 @@
+package com.askimed.nf.test.config;
+
+import java.util.List;
+import java.util.Vector;
+
+public class StageBuilder {
+
+	private List<FileStaging> paths = new Vector<FileStaging>();
+
+	public void copy(String path) {
+		paths.add(new FileStaging(path, FileStaging.MODE_COPY));
+	}
+
+	public void symlink(String path) {
+		paths.add(new FileStaging(path, FileStaging.MODE_SYMLINK));
+	}
+
+
+	public List<FileStaging> getPaths() {
+		return paths;
+	}
+
+}

src/main/java/com/askimed/nf/test/core/AbstractTest.java

@@ -8,6 +8,11 @@
 import java.util.List;
 import java.util.Vector;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.askimed.nf.test.config.Config;
+import com.askimed.nf.test.config.FileStaging;
 import com.askimed.nf.test.util.FileUtil;
 
 public abstract class AbstractTest implements ITest {
@@ -54,7 +59,8 @@ public abstract class AbstractTest implements ITest {
 
 	public boolean skipped = false;
 
-	public static String[] SHARED_DIRECTORIES = { "bin", "lib", "assets" };
+	public static FileStaging[] SHARED_DIRECTORIES = { new FileStaging("bin"), new FileStaging("lib"),
+			new FileStaging("assets") };
 
 	protected File config = null;
 
@@ -70,6 +76,8 @@ public abstract class AbstractTest implements ITest {
 
 	private String options;
 
+	private static Logger log = LoggerFactory.getLogger(AbstractTest.class);
+
 	public AbstractTest(AbstractTestSuite parent) {
 		this.parent = parent;
 		options = parent.getOptions();
@@ -84,21 +92,26 @@ public File getConfig() {
 	}
 
 	@Override
-	public void setup(File testDirectory) throws IOException {
+	public void setup(Config config, File testDirectory) throws IOException {
 
 		if (testDirectory == null) {
 			throw new IOException("Testcase setup failed: No home directory set");
 		}
-				
+
 		launchDir = initDirectory("Launch Directory", testDirectory, DIRECTORY_TESTS, getHash());
 		metaDir = initDirectory("Meta Directory", launchDir, DIRECTORY_META);
 		outputDir = initDirectory("Output Directory", launchDir, DIRECTORY_OUTPUT);
 		workDir = initDirectory("Working Directory", launchDir, DIRECTORY_WORK);
 
 		try {
-			// copy bin and lib to metaDir. TODO: use symlinks and read additional "mapping"
-			// from config file
+			// copy bin, assets and lib to metaDir
 			shareDirectories(SHARED_DIRECTORIES, metaDir);
+			if (config != null) {
+				// copy user defined staging directories
+				log.debug("Stage {} user provided files...", config.getStageBuilder().getPaths().size());
+				shareDirectories(config.getStageBuilder().getPaths(), metaDir);
+			}
+			shareDirectories(parent.getStageBuilder().getPaths(), metaDir);
 		} catch (Exception e) {
 			throw new IOException("Testcase setup failed: Directories could not be shared:\n" + e);
 		}
@@ -154,11 +167,11 @@ public String getErrorReport() throws Throwable {
 
 	@Override
 	public String getHash() {
-		
+
 		if (parent == null || parent.getFilename() == null || getName() == null || getName().isEmpty()) {
 			throw new RuntimeException("Error generating hash");
 		}
-		
+
 		return hash(parent.getFilename() + getName());
 
 	}
@@ -236,13 +249,17 @@ public boolean isWithTrace() {
 		return withTrace;
 	}
 
-	protected void shareDirectories(String[] directories, File metaDir) throws IOException {
-		for (String directory : directories) {
-			File localDirectory = new File(directory);
-			if (localDirectory.exists()) {
-				String metaDirectory = FileUtil.path(metaDir.getAbsolutePath(), directory);
-				FileUtil.copyDirectory(localDirectory.getAbsolutePath(), metaDirectory);
-			}
+	protected void shareDirectories(List<FileStaging> directories, File stageDir) throws IOException {
+		for (FileStaging directory : directories) {
+			String metaDirectory = FileUtil.path(stageDir.getAbsolutePath(), directory.getPath());
+			directory.stage(metaDirectory);
+		}
+	}
+
+	protected void shareDirectories(FileStaging[] directories, File stageDir) throws IOException {
+		for (FileStaging directory : directories) {
+			String metaDirectory = FileUtil.path(stageDir.getAbsolutePath(), directory.getPath());
+			directory.stage(metaDirectory);
 		}
 	}
 
@@ -254,7 +271,7 @@ public void setUpdateSnapshot(boolean updateSnapshot) {
 	public boolean isUpdateSnapshot() {
 		return updateSnapshot;
 	}
-	
+
 	@Override
 	public String toString() {
 		return getHash().substring(0, 8) + ": " + getName();