Add possibility to set and combine multiple profiles (#128)

docs/docs/cli/test.md

@@ -10,45 +10,52 @@ nf-test test [<NEXTFLOW_FILES>|<SCRIPT_FOLDERS>]
 
 #### `--profile <NEXTFLOW_PROFILE>`
 
+To run your test using a specific Nextflow profile, you can use the `--profile` argument. [Learn more](/docs/configuration/#managing-profiles).
+
 #### `--debug`
+
 The debug parameter prints out all available output channels which can be accessed in the `then` clause.
 
 #### `--without-trace`
+
 The Linux tool `procps` is required to run Nextflow tracing. In case your container does not support this tool, you can also run nf-test without tracing. Please note that the `workflow.trace` are not available when running it with this flag.
 
 #### `--tag <tag>`
+
 Execute only tests with the provided tag. Multiple tags can be used and have to be separated by commas (e.g. `tag1,tag2`).
 
 #### `--tap <filename>`
+
 Writes test results in [TAP format](https://testanything.org) to file.
 
 #### `--junitxml <filename>`
+
 Writes test results in [JUnit XML format](https://junit.org/) to file, which conforms to [the standard schema](https://github.com/junit-team/junit5/blob/242f3b3ef84cfd96c9de26992588812a68cdef8b/platform-tests/src/test/resources/jenkins-junit.xsd).
 
 ## Examples
 
-* Run all test scripts that can be found in the `testDir` defined in the `nf-test.config` file in the current working directory:
+- Run all test scripts that can be found in the `testDir` defined in the `nf-test.config` file in the current working directory:
 
-    ```
-    nf-test test
-    ```
+  ```
+  nf-test test
+  ```
 
-* Run all specified test scripts and search specified directories for additional test scripts:
+- Run all specified test scripts and search specified directories for additional test scripts:
 
-    ```
-    nf-test test tests/modules/local/salmon_index.nf.test tests/modules/bwa_index.nf.test
+  ```
+  nf-test test tests/modules/local/salmon_index.nf.test tests/modules/bwa_index.nf.test
 
-    nf-test test tests/modules tests/modules/bwa_index.nf.test
-    ```
+  nf-test test tests/modules tests/modules/bwa_index.nf.test
+  ```
 
-* Run a specific test using its hash:
+- Run a specific test using its hash:
 
-    ```
-    nf-test test tests/main.nf.test@d41119e4
-    ```
+  ```
+  nf-test test tests/main.nf.test@d41119e4
+  ```
 
-* Run all tests and write results to `report.tap`:
+- Run all tests and write results to `report.tap`:
 
-    ```
-    nf-test test --tap report.tap
-    ```
+  ```
+  nf-test test --tap report.tap
+  ```

docs/docs/configuration.md

@@ -1,31 +1,31 @@
 # Configuration
 
-## Setup test profile
+## `nf-test.config`
 
-To run your test using a specific Nextflow profile, you can use the `--profile` argument on the command line or define a default profile in `nf-test.config`.
+The `nf-test.config` file is a configuration file used to customize settings and behavior for `nf-test`. This file must be located in the root of your project, and it is automatically loaded when you run `nf-test test`. Below are the parameters that can be adapted:
 
+| Parameter    | Description                                                                                                                                                        | Default Value             |
+| ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------- |
+| `testsDir`   | Location for storing all nf-test cases (test scripts). If you want all test files to be in the same directory as the script itself, you can set the testDir to `.` | `"tests"`                 |
+| `workDir`    | Directory for storing temporary files and working directories for each test. This directory should be added to `.gitignore`.                                       | `".nf-test"`              |
+| `configFile` | Location of an optional `nextflow.config` file specifically used for executing tests. [Learn more](#testsnextflowconfig).                                          | `"tests/nextflow.config"` |
+| `libDir`     | Location of a library folder that is automatically added to the classpath during testing to include additional libraries or resources needed for test cases.       | `"tests/lib"`             |
+| `profile`    | Default profile to use for running tests defined in the Nextflow configuration. See [Learn more](#managing-profiles).                                              | `"docker"`                |
+| `withTrace`  | Enable or disable tracing options during testing. Disable tracing if your containers don't include the `procps` tool.                                              | `true`                    |
+| `autoSort`   | Enable or disable sorted channels by default when running tests.                                                                                                   | `true`                    |
+| `options`    | Custom Nextflow command-line options to be applied when running tests. For example `"-dump-channels -stub-run"`                                                    |                           |
 
-##  `nf-test.config`
-
-This config file contains settings for nf-test.
+Here's an example of what an `nf-test.config` file could look like:
 
 ```groovy
 config {
-    // location for all nf-tests
     testsDir "tests"
-    // nf-test directory including temporary files for each test
     workDir ".nf-test"
-    // location of an optional nextflow.config file specific for executing tests
     configFile "tests/nextflow.config"
-    // location of library folder that is added automatically to the classpath
-    libDir "tests/lib"    
-    // run all test with the defined docker profile from the main nextflow.config
+    libDir "tests/lib"
     profile "docker"
-    // disable tracing options in case container does not include `procps` Linux tool.
     withTrace false
-    //disable sorted channels
     autoSort false
-    // add Nextflow options
     options "-dump-channels -stub-run"
 }
 ```
@@ -43,7 +43,7 @@ params {
 
 ## Configuration for tests
 
-nf-test allows to set an additional configuration for a testsuite:
+nf-test allows to set and overwrite the `config`, `autoSort` and `options` properties for a specific testsuite:
 
 ```
 nextflow_process {
@@ -59,7 +59,7 @@ nextflow_process {
 }
 ```
 
-It is also possible to overwrite the `config`, `autoSort` or Nextflow properties (e.g. `options "-dump-channels"`) for a specific test. Depending on the used Nextflow option, also add the `--debug` nf-test option on the command-line to see the addtional output.  
+It is also possible to overwrite these properties for specific test. Depending on the used Nextflow option, also add the `--debug` nf-test option on the command-line to see the addtional output.
 
 ```
 nextflow_process {
@@ -75,3 +75,27 @@ nextflow_process {
 
 }
 ```
+
+## Managing Profiles
+
+Profiles in `nf-test` provide a convenient way to configure and customize Nextflow executions for your test cases. To run your test using a specific Nextflow profile, you can use the `--profile` argument on the command line or define a default profile in `nf-test.config`.
+
+### Basic Profile Usage
+
+By default, `nf-test` reads the profile configuration from `nf-test.config`. If you've defined a profile called `A` in `nf-test.config`, running `nf-test --profile B` will start Nextflow with only the `B` profile. It replaces any existing profiles.
+
+### Combining Profiles with "+"
+
+To combine profiles, you can use the `+` prefix. For example, running `nf-test --profile +B` will start Nextflow with both `A` and `B` profiles, resulting in `-profile A,B`. This allows you to extend the existing configuration with additional profiles.
+
+### Profile Priority Order
+
+Profiles are evaluated in a specific order, ensuring predictable behavior:
+
+1. **Profile in nf-test.config:** The first profile considered is the one defined in `nf-test.config`.
+
+2. **Profile Defined in Testcase:** If you specify a profile within a testcase, it takes precedence over the one in `nf-test.config`.
+
+3. **Profile Defined on the Command Line (CLI):** Finally, any profiles provided directly through the CLI have the highest priority and override/extends previously defined profiles.
+
+By understanding this profile evaluation order, you can effectively configure Nextflow executions for your test cases in a flexible and organized manner.

src/main/java/com/askimed/nf/test/commands/RunTestsCommand.java

@@ -68,6 +68,11 @@ public class RunTestsCommand extends AbstractCommand {
 			"--lib" }, description = "Library extension path", required = false, showDefaultValue = Visibility.ALWAYS)
 	private String lib = "";
 
+	@Option(names = { "--config",
+			"-c" }, description = "nf-test.config filename", required = false, showDefaultValue = Visibility.ALWAYS)
+
+	private String configFilename = Config.FILENAME;
+
 	@Option(names = {
 			"--plugins" }, description = "Library extension path", required = false, showDefaultValue = Visibility.ALWAYS)
 	private String plugins = null;
@@ -86,17 +91,14 @@ public Integer execute() throws Exception {
 
 		try {
 
-			String defaultProfile = null;
 			File defaultConfigFile = null;
 			String libDir = lib;
 			boolean defaultWithTrace = true;
 			try {
-
-				File configFile = new File(Config.FILENAME);
+				File configFile = new File(configFilename);
 				if (configFile.exists()) {
 
 					Config config = Config.parse(configFile);
-					defaultProfile = config.getProfile();
 					defaultConfigFile = config.getConfigFile();
 					defaultWithTrace = config.isWithTrace();
 					if (!libDir.isEmpty()) {
@@ -164,12 +166,7 @@ public Integer execute() throws Exception {
 			engine.setCleanSnapshot(cleanSnapshot);
 			engine.setLibDir(libDir);
 			engine.setPluginManager(manager);
-
-			if (profile != null) {
-				engine.setProfile(profile);
-			} else {
-				engine.setProfile(defaultProfile);
-			}
+			engine.addProfile(profile);
 			if (withoutTrace) {
 				engine.setWithTrace(false);
 			} else {

src/main/java/com/askimed/nf/test/core/AbstractTestSuite.java

@@ -15,7 +15,7 @@ public abstract class AbstractTestSuite implements ITestSuite {
 
 	private String name;
 
-	private String profile = null;
+	private List<String> profiles = new Vector<String>();;
 
 	private File globalConfig = null;
 
@@ -46,6 +46,9 @@ public void configure(Config config) {
 		autoSort = config.isAutoSort();
 		options = config.getOptions();
 		homeDirectory = new File(config.getWorkDir());
+		if (config.getProfile() != null) {
+			addProfile(config.getProfile());
+		}
 	}
 
 	public void script(String script) {
@@ -91,7 +94,7 @@ public void name(String name) {
 	}
 
 	public void profile(String profile) {
-		this.profile = profile;
+		this.profiles.add(profile);
 	}
 
 	public void config(String config) {
@@ -106,12 +109,12 @@ public String getName() {
 		return name;
 	}
 
-	public void setProfile(String profile) {
-		this.profile = profile;
+	public void addProfile(String profile) {
+		this.profiles.add(profile);
 	}
 
-	public String getProfile() {
-		return profile;
+	public List<String> getProfiles() {
+		return profiles;
 	}
 
 	public void autoSort(boolean autoSort) {

src/main/java/com/askimed/nf/test/core/ITestSuite.java

@@ -12,7 +12,7 @@ public interface ITestSuite extends ITaggable {
 
 	public String getName();
 
-	public void setProfile(String profile);
+	public void addProfile(String profile);
 
 	public void setGlobalConfigFile(File config);
 

src/main/java/com/askimed/nf/test/core/TestExecutionEngine.java

@@ -28,7 +28,7 @@ public class TestExecutionEngine {
 
 	private boolean debug = false;
 
-	private String profile = null;
+	private List<String> profiles = new Vector<String>();
 
 	private File configFile = null;
 
@@ -47,7 +47,7 @@ public class TestExecutionEngine {
 	private TagQuery tagQuery = new TagQuery();
 
 	private static Logger log = LoggerFactory.getLogger(TestExecutionEngine.class);
-	
+
 	public void setScripts(List<File> scripts) {
 		this.scripts = scripts;
 	}
@@ -56,8 +56,11 @@ public void setDebug(boolean debug) {
 		this.debug = debug;
 	}
 
-	public void setProfile(String profile) {
-		this.profile = profile;
+	public void addProfile(String profile) {
+		if (profile == null) {
+			return;
+		}
+		this.profiles.add(profile);
 	}
 
 	public void setConfigFile(File configFile) {
@@ -165,37 +168,37 @@ public int execute() throws Throwable {
 
 		int totalTests = 0;
 		int failedTests = 0;
-		
+
 		log.info("Started test plan");
-		
+
 		listener.testPlanExecutionStarted();
-		
+
 		boolean failed = false;
 		for (ITestSuite testSuite : testSuits) {
 
-			// override profile from CLI
-			if (profile != null) {
-				testSuite.setProfile(profile);
+			for (String profile : profiles) {
+				testSuite.addProfile(profile);
 			}
 
 			if (configFile != null) {
+				// TODO: addConfig as list
 				testSuite.setGlobalConfigFile(configFile);
 			}
-			
+
 			log.info("Running testsuite '{}' from file '{}'.", testSuite, testSuite.getFilename());
 
 			listener.testSuiteExecutionStarted(testSuite);
 
 			for (ITest test : testSuite.getTests()) {
 				if (test.isSkipped()) {
-					log.info("Test '{}' skipped.", test);				
+					log.info("Test '{}' skipped.", test);
 					listener.executionSkipped(test, "");
 					continue;
 				}
-				
+
 				log.info("Run test '{}'. type: {}", test, test.getClass().getName());
 				totalTests++;
-				
+
 				listener.executionStarted(test);
 				TestExecutionResult result = new TestExecutionResult(test);
 				test.setWithTrace(withTrace);
@@ -223,13 +226,13 @@ public int execute() throws Throwable {
 				}
 				test.cleanup();
 				result.setEndTime(System.currentTimeMillis());
-				
+
 				log.info("Test '{}' finished. status: {}", result.getTest(), result.getStatus(), result.getThrowable());
-				
+
 				listener.executionFinished(test, result);
 
 			}
-			
+
 			// Remove obsolete snapshots when no test was skipped and no test failed.
 			if (cleanSnapshot && !testSuite.hasSkippedTests() && !testSuite.hasFailedTests()
 					&& testSuite.hasSnapshotLoaded()) {
@@ -239,16 +242,15 @@ public int execute() throws Throwable {
 				snapshot.save();
 			}
 
-			log.info("Testsuite '{}' finished. snapshot file: {}, skipped tests: {}, failed tests: {}",
-					testSuite, testSuite.hasSnapshotLoaded(), testSuite.hasSkippedTests(), testSuite.hasFailedTests());
+			log.info("Testsuite '{}' finished. snapshot file: {}, skipped tests: {}, failed tests: {}", testSuite,
+					testSuite.hasSnapshotLoaded(), testSuite.hasSkippedTests(), testSuite.hasFailedTests());
 
-
 			listener.testSuiteExecutionFinished(testSuite);
 
 		}
 
 		log.info("Executed {} tests. {} tests failed. Done!", totalTests, failedTests);
-		
+
 		listener.testPlanExecutionFinished();
 
 		return (failed) ? 1 : 0;

src/main/java/com/askimed/nf/test/lang/function/FunctionTest.java

@@ -123,7 +123,9 @@ public void execute() throws Throwable {
 		NextflowCommand nextflow = new NextflowCommand();
 		nextflow.setScript(workflow.getAbsolutePath());
 		nextflow.setParams(context.getParams());
-		nextflow.setProfile(parent.getProfile());
+		for (String profile: parent.getProfiles()) {
+			nextflow.addProfile(profile);
+		}
 		File projectConfig = new File("nextflow.config");
 		if (projectConfig.exists()) {
 			nextflow.addConfig(projectConfig);	

src/main/java/com/askimed/nf/test/lang/pipeline/PipelineTest.java

@@ -103,7 +103,9 @@ public void execute() throws Throwable {
 		NextflowCommand nextflow = new NextflowCommand();
 		nextflow.setScript(script);
 		nextflow.setParams(context.getParams());
-		nextflow.setProfile(parent.getProfile());
+		for (String profile: parent.getProfiles()) {
+			nextflow.addProfile(profile);
+		}
 		File projectConfig = new File("nextflow.config");
 		if (projectConfig.exists()) {
 			nextflow.addConfig(projectConfig);