feat: [7.0.0] template_engine: basic support #497

1) highlight template_engine as known section 2) Only one argument expected after `template_engine` 3) Keywords arguments not supported after `template_engine` 4) Do not warn about unused log section if rule wiht `template_engine` 5) `template_engine` is execution section, so could not be with run/shell/notebook/script 6) `template_engine` should be last in the rule 7) no smksl template injection in `template_engine` Resolves: #497
JetBrains-Research · Aug 3, 2022 · 197f189 · 197f189
1 parent 3505d87
commit 197f189
Show file tree

Hide file tree

Showing 17 changed files with 142 additions and 116 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -15,6 +15,7 @@ Released on ...
 ### Added:
 - [7.10.0] Support conda environment definitions to be passed as function pointers, similar to input, params, and resources #494  (see [#494](https://github.com/JetBrains-Research/snakecharm/issues/494)) 
 - [7.9.0] Syntax supprt for `exclude` keyword in use rules (see [#495](https://github.com/JetBrains-Research/snakecharm/issues/495), [#496](https://github.com/JetBrains-Research/snakecharm/issues/496)) 
+- [ 7.0.0] template_engine: basic support (see [#497](https://github.com/JetBrains-Research/snakecharm/issues/497)) 
 - TODO  (see [#NNN](https://github.com/JetBrains-Research/snakecharm/issues/NNN)) 
 
 ## [2022.1.749]

diff --git a/src/main/kotlin/com/jetbrains/snakecharm/codeInsight/SnakemakeAPI.kt b/src/main/kotlin/com/jetbrains/snakecharm/codeInsight/SnakemakeAPI.kt
@@ -32,6 +32,7 @@ import com.jetbrains.snakecharm.lang.SnakemakeNames.SECTION_SCRIPT
 import com.jetbrains.snakecharm.lang.SnakemakeNames.SECTION_SHADOW
 import com.jetbrains.snakecharm.lang.SnakemakeNames.SECTION_SHELL
 import com.jetbrains.snakecharm.lang.SnakemakeNames.SECTION_SINGULARITY
+import com.jetbrains.snakecharm.lang.SnakemakeNames.SECTION_TEMPLATE_ENGINE
 import com.jetbrains.snakecharm.lang.SnakemakeNames.SECTION_THREADS
 import com.jetbrains.snakecharm.lang.SnakemakeNames.SECTION_VERSION
 import com.jetbrains.snakecharm.lang.SnakemakeNames.SECTION_WILDCARD_CONSTRAINTS
@@ -104,10 +105,13 @@ object SnakemakeAPI {
     const val SNAKEMAKE_MODULE_NAME_IO_PY = "io.py"
 
     /**
-     * Sections that gets 'snakemake' object with 'snakemake.input', 'snakemake.params' etc settings
+     * Sections that execute external script with access to 'snakemake' object, i.e to 'snakemake.input',
+     * 'snakemake.params' etc settings. So we cannot verify that log section mentioned in rule is
+     * unused.
      */
     val EXECUTION_SECTIONS_THAT_ACCEPTS_SNAKEMAKE_PARAMS_OBJ_FROM_RULE = setOf(
-        SECTION_WRAPPER, SECTION_NOTEBOOK, SECTION_SCRIPT, SECTION_CWL
+        SECTION_WRAPPER, SECTION_NOTEBOOK, SECTION_SCRIPT, SECTION_CWL,
+        SECTION_TEMPLATE_ENGINE
     )
 
     val EXECUTION_SECTIONS_KEYWORDS = setOf(
@@ -123,7 +127,7 @@ object SnakemakeAPI {
         SECTION_CWL, SECTION_BENCHMARK, SECTION_VERSION,
         SECTION_MESSAGE, SECTION_THREADS, SECTION_SINGULARITY,
         SECTION_PRIORITY, SECTION_CONDA, SECTION_GROUP,
-        SECTION_SHADOW, SECTION_CACHE, SECTION_NOTEBOOK, SECTION_CONTAINER,
+        SECTION_SHADOW, SECTION_CACHE, SECTION_NOTEBOOK, SECTION_CONTAINER, SECTION_TEMPLATE_ENGINE,
         SECTION_HANDOVER, SECTION_CONTAINERIZED,
         SECTION_DEFAULT_TARGET,
         SECTION_RETRIES
@@ -160,7 +164,7 @@ object SnakemakeAPI {
         SECTION_BENCHMARK, SECTION_VERSION, SECTION_MESSAGE, SECTION_SHELL, SECTION_THREADS, SECTION_SINGULARITY,
         SECTION_PRIORITY, SECTION_WILDCARD_CONSTRAINTS, SECTION_GROUP, SECTION_SHADOW,
         SECTION_CONDA,
-        SECTION_SCRIPT, SECTION_WRAPPER, SECTION_CWL, SECTION_NOTEBOOK,
+        SECTION_SCRIPT, SECTION_WRAPPER, SECTION_CWL, SECTION_NOTEBOOK, SECTION_TEMPLATE_ENGINE,
         SECTION_CACHE,
         SECTION_CONTAINER,
         SECTION_CONTAINERIZED,
@@ -247,7 +251,8 @@ object SnakemakeAPI {
         SECTION_PRIORITY, SECTION_SINGULARITY, SECTION_CACHE,
         SECTION_CONTAINER, SECTION_CONTAINERIZED, SECTION_NOTEBOOK,
         SECTION_ENVMODULES, SECTION_HANDOVER, SECTION_DEFAULT_TARGET,
-        SECTION_RETRIES
+        SECTION_RETRIES,
+        SECTION_TEMPLATE_ENGINE
     )
 
     /**
@@ -312,7 +317,8 @@ object SnakemakeAPI {
         SECTION_BENCHMARK, SECTION_VERSION, SECTION_MESSAGE, SECTION_SHELL, SECTION_THREADS, SECTION_SINGULARITY,
         SECTION_PRIORITY, SECTION_GROUP, SECTION_SHADOW, SECTION_CONDA, SECTION_SCRIPT, SECTION_WRAPPER,
         SECTION_CWL, SECTION_NOTEBOOK, SECTION_CACHE, SECTION_CONTAINER, SECTION_CONTAINERIZED, SECTION_ENVMODULES,
-        SECTION_NAME, SECTION_HANDOVER, SECTION_DEFAULT_TARGET, SECTION_RETRIES
+        SECTION_NAME, SECTION_HANDOVER, SECTION_DEFAULT_TARGET, SECTION_RETRIES,
+        SECTION_TEMPLATE_ENGINE
     )
 
     /**

diff --git a/src/main/kotlin/com/jetbrains/snakecharm/lang/SnakemakeNames.kt b/src/main/kotlin/com/jetbrains/snakecharm/lang/SnakemakeNames.kt
@@ -71,6 +71,7 @@ object SnakemakeNames {
     const val SECTION_NAME = "name" // >= 5.31
     const val SECTION_HANDOVER = "handover" // >= 6.2
     const val SECTION_DEFAULT_TARGET = "default_target" // >= 5.12.0
+    const val SECTION_TEMPLATE_ENGINE = "template_engine" // >= 7.0.0
     const val SECTION_RETRIES = "retries" // >= 7.7.0
 
     const val RUN_SECTION_VARIABLE_RULE = "rule"

diff --git a/src/main/resources/inspectionDescriptions/SmkRuleSectionAfterExecutionInspection.html b/src/main/resources/inspectionDescriptions/SmkRuleSectionAfterExecutionInspection.html
@@ -1,5 +1,5 @@
 <html>
 <body>
-Rule sections such as input/output/params/..., cannot be used after run/shell/script/wrapper/cwl/notebook sections.
+Rule sections such as input/output/params/..., cannot be used after run/shell/script/wrapper/cwl/notebook/template_engine sections.
 </body>
 </html>
diff --git a/src/test/kotlin/com/jetbrains/snakecharm/lang/parser/SnakemakeParsingTest.kt b/src/test/kotlin/com/jetbrains/snakecharm/lang/parser/SnakemakeParsingTest.kt
@@ -337,6 +337,10 @@ class SnakemakeParsingTest : ParsingTestCase(
         doTest()
     }
 
+    fun testRuleSectionTemplateEngine() {
+        doTest()
+    }
+
     fun testRuleSectionContainer() {
         doTest()
     }

diff --git a/src/test/resources/features/completion/keywords_completion.feature b/src/test/resources/features/completion/keywords_completion.feature
@@ -207,44 +207,45 @@ Feature: Completion for snakemake keyword-like things
         <result>: 
       """
     Examples:
-      | rule_like  | str  | result         |
-      | rule       | inp  | input          |
-      | checkpoint | inp  | input          |
-      | rule       | out  | output         |
-      | checkpoint | out  | output         |
-      | rule       | par  | params         |
-      | checkpoint | par  | params         |
-      | rule       | lo   | log            |
-      | checkpoint | lo   | log            |
-      | rule       | be   | benchmark      |
-      | checkpoint | be   | benchmark      |
-      | rule       | vers | version        |
-      | checkpoint | vers | version        |
-      | rule       | cac  | cache          |
-      | checkpoint | cac  | cache          |
-      | rule       | mes  | message        |
-      | checkpoint | mes  | message        |
-      | rule       | pr   | priority       |
-      | checkpoint | pr   | priority       |
-      | rule       | gr   | group          |
-      | checkpoint | gr   | group          |
-      | rule       | envm | envmodules     |
-      | checkpoint | envm | envmodules     |
-      | rule       | cw   | cwl            |
-      | checkpoint | cw   | cwl            |
-      | rule       | sc   | script         |
-      | checkpoint | sc   | script         |
-      | rule       | run  | run            |
-      | checkpoint | run  | run            |
-      | rule       | wr   | wrapper        |
-      | checkpoint | wr   | wrapper        |
-      | rule       | na   | name           |
-      | checkpoint | na   | name           |
-      | rule       | han  | handover       |
-      | checkpoint | han  | handover       |
-      | rule       | defa | default_target |
-      | checkpoint | defa | default_target |
-      | rule       | retr | retries        |
+      | rule_like  | str  | result          |
+      | rule       | inp  | input           |
+      | checkpoint | inp  | input           |
+      | rule       | out  | output          |
+      | checkpoint | out  | output          |
+      | rule       | par  | params          |
+      | checkpoint | par  | params          |
+      | rule       | lo   | log             |
+      | checkpoint | lo   | log             |
+      | rule       | be   | benchmark       |
+      | checkpoint | be   | benchmark       |
+      | rule       | vers | version         |
+      | checkpoint | vers | version         |
+      | rule       | cac  | cache           |
+      | checkpoint | cac  | cache           |
+      | rule       | mes  | message         |
+      | checkpoint | mes  | message         |
+      | rule       | pr   | priority        |
+      | checkpoint | pr   | priority        |
+      | rule       | gr   | group           |
+      | checkpoint | gr   | group           |
+      | rule       | envm | envmodules      |
+      | checkpoint | envm | envmodules      |
+      | rule       | cw   | cwl             |
+      | checkpoint | cw   | cwl             |
+      | rule       | sc   | script          |
+      | checkpoint | sc   | script          |
+      | rule       | run  | run             |
+      | checkpoint | run  | run             |
+      | rule       | wr   | wrapper         |
+      | checkpoint | wr   | wrapper         |
+      | rule       | na   | name            |
+      | checkpoint | na   | name            |
+      | rule       | han  | handover        |
+      | checkpoint | han  | handover        |
+      | rule       | defa | default_target  |
+      | checkpoint | defa | default_target  |
+      | rule       | retr | retries         |
+      | rule       | temp | template_engine |
 
   Scenario Outline: Complete at rule/checkpoint/module level
     Given a snakemake project

diff --git a/src/test/resources/features/completion/smkl_in_section_completion.feature b/src/test/resources/features/completion/smkl_in_section_completion.feature
@@ -102,6 +102,7 @@ Feature: Completion for sections/variables in SmkSL injections
       | handover             |
       | default_target       |
       | retries              |
+      | template_engine      |
     Examples:
       | rule_like  | signature     |
       | rule       | shell: "{     |

diff --git a/.../features/highlighting/inspections/execution_subsection_in_use_section_inspection.feature b/.../features/highlighting/inspections/execution_subsection_in_use_section_inspection.feature
@@ -1,42 +1,26 @@
 Feature: Inspection if subsection is unexpected for section but it i appropriate for another section
 
-  Scenario: When 'use' section contains execution subsections
+  Scenario Outline: When 'use' section contains execution subsections
     Given a snakemake project
     Given I open a file "foo.smk" with text
       """
       use rule RULE as NEW_RULE with:
-          run: ""
-          shell: ""
-          notebook: ""
           input: "file1"
-          script: ""
-          cwl: ""
-          wrapper: ""
           output: "file2"
+          <section>: "foo"
       """
     And SmkExecutionSubsectionInUseSectionInspection inspection is enabled
-    Then I expect inspection error on <run> with message
+    Then I expect inspection error on <<section>> with message
       """
       Execution sections can't be overridden in 'use rule'
       """
-    Then I expect inspection error on <shell> with message
-      """
-      Execution sections can't be overridden in 'use rule'
-      """
-    Then I expect inspection error on <notebook> with message
-      """
-      Execution sections can't be overridden in 'use rule'
-      """
-    Then I expect inspection error on <script> with message
-      """
-      Execution sections can't be overridden in 'use rule'
-      """
-    Then I expect inspection error on <cwl> with message
-      """
-      Execution sections can't be overridden in 'use rule'
-      """
-    Then I expect inspection error on <wrapper> with message
-      """
-      Execution sections can't be overridden in 'use rule'
-      """
-    When I check highlighting errors
+    When I check highlighting errors
+    Examples:
+      | section         |
+      | run             |
+      | shell           |
+      | notebook        |
+      | script          |
+      | cwl             |
+      | wrapper         |
+      | template_engine |
diff --git a/src/test/resources/features/highlighting/inspections/multiple_args_inspection.feature b/src/test/resources/features/highlighting/inspections/multiple_args_inspection.feature
@@ -1,5 +1,5 @@
 Feature: Inspection for multiple arguments in various sections
-  Scenario Outline: Multiple arguments in module/subworkflow section
+  Scenario Outline: module/subworkflow sections with only one argument
     Given a snakemake project
     Given I open a file "foo.smk" with text
     """
@@ -26,7 +26,7 @@ Feature: Inspection for multiple arguments in various sections
       | module      | skip_validation |
       | module      | meta_wrapper    |
 
-  Scenario Outline: Multiple arguments in execution sections
+  Scenario Outline: rule/checkpoint sections with only one argument
     Given a snakemake project
     Given I open a file "foo.smk" with text
     """
@@ -44,27 +44,28 @@ Feature: Inspection for multiple arguments in various sections
     """
     When I check highlighting errors
     Examples:
-      | section        |
-      | shell          |
-      | script         |
-      | wrapper        |
-      | cwl            |
-      | conda          |
-      | singularity    |
-      | priority       |
-      | version        |
-      | cache          |
-      | group          |
-      | message        |
-      | benchmark      |
-      | threads        |
-      | shadow         |
-      | notebook       |
-      | container      |
-      | containerized  |
-      | handover       |
-      | default_target |
-      | retries        |
+      | section         |
+      | shell           |
+      | script          |
+      | wrapper         |
+      | cwl             |
+      | conda           |
+      | singularity     |
+      | priority        |
+      | version         |
+      | cache           |
+      | group           |
+      | message         |
+      | benchmark       |
+      | threads         |
+      | shadow          |
+      | notebook        |
+      | container       |
+      | containerized   |
+      | handover        |
+      | default_target  |
+      | retries         |
+      | template_engine |
 
   Scenario Outline: Multiple arguments in workflow section
     Given a snakemake project

diff --git a/...resources/features/highlighting/inspections/rule_keywords_after_execution_section.feature b/...resources/features/highlighting/inspections/rule_keywords_after_execution_section.feature
@@ -18,13 +18,14 @@ Feature: Rule sections after execution sections inspection.
     """
     When I check highlighting errors
     Examples:
-      | rule_like  | sect1    | sect1_text           | sect2     | sect2_text |
-      | rule       | script   | "s.py"               | log       | "l.log"    |
-      | checkpoint | script   | "s.py"               | log       | "l.log"    |
-      | rule       | cwl      | "https://f.cwl" | resources | mem_mb=100 |
-      | rule       | wrapper  | "dir/wrapper"        | threads   | 8          |
-      | rule       | shell    | "cmd"                | params    | a="value"  |
-      | rule       | notebook | "n.r"                | params    | a="value"  |
+      | rule_like  | sect1           | sect1_text      | sect2     | sect2_text |
+      | rule       | script          | "s.py"          | log       | "l.log"    |
+      | checkpoint | script          | "s.py"          | log       | "l.log"    |
+      | rule       | cwl             | "https://f.cwl" | resources | mem_mb=100 |
+      | rule       | wrapper         | "dir/wrapper"   | threads   | 8          |
+      | rule       | shell           | "cmd"           | params    | a="value"  |
+      | rule       | notebook        | "n.r"           | params    | a="value"  |
+      | rule       | template_engine | "n.r"           | params    | a="value"  |
 
   Scenario Outline: Move execution section to the end of the rule fix test
     Given a snakemake project

diff --git a/...st/resources/features/highlighting/inspections/unexpected_keyword_args_inspection.feature b/...st/resources/features/highlighting/inspections/unexpected_keyword_args_inspection.feature
@@ -48,6 +48,7 @@ Feature: Inspection for unexpected keyword arguments in section
       | rule       | handover        |
       | rule       | default_target  |
       | rule       | retries         |
+      | rule       | template_engine |
       | checkpoint | message         |
       | checkpoint | notebook        |
       | checkpoint | priority        |

diff --git a/src/test/resources/features/highlighting/inspections/unused_log_section_inspection.feature b/src/test/resources/features/highlighting/inspections/unused_log_section_inspection.feature
@@ -130,16 +130,20 @@ Feature: Rule SmkUnusedLogFileInspection inspection
     """
     <rule_like> NAME:
         log: <log_variation>
-        wrapper: "0.60.1/bio/samtools/merge"
+        <section>
     """
     And SmkUnusedLogFileInspection inspection is enabled
     And I expect no inspection weak warnings
     When I check highlighting weak warnings
     Examples:
-      | rule_like  | log_variation |
-      | rule       | "log.log"     |
-      | checkpoint | "my_log.log"  |
-      | rule       | l1="foo.log"  |
+      | rule_like  | log_variation | section                              |
+      | rule       | "log.log"     | wrapper: "0.60.1/bio/samtools/merge" |
+      | rule       | l1="foo.log"  | wrapper: "0.60.1/bio/samtools/merge" |
+      | rule       | "log.log"     | notebook: ""                         |
+      | rule       | "log.log"     | script: ""                           |
+      | rule       | "log.log"     | cwl: ""                              |
+      | rule       | "log.log"     | template_engine: ""                                 |
+      | checkpoint | "my_log.log"  | wrapper: "0.60.1/bio/samtools/merge" |
 
   Scenario Outline: Unused in 'run' section 'log' section
     Given a snakemake project

diff --git a/src/test/resources/features/highlighting/smk_syntax_annotator.feature b/src/test/resources/features/highlighting/smk_syntax_annotator.feature
@@ -99,6 +99,7 @@ Feature: Annotate additional syntax
       | rule       | handover             | ""         | SMK_DECORATOR            |
       | rule       | default_target       | ""         | SMK_DECORATOR            |
       | rule       | retries              | ""         | SMK_DECORATOR            |
+      | rule       | template_engine      | ""         | SMK_DECORATOR            |
       | checkpoint | output               | "file.txt" | SMK_DECORATOR            |
       | checkpoint | run                  | ""         | PY.PREDEFINED_DEFINITION |