perform validation on root workflow

.github/workflows/fuzzy_compile_weekly.yml

@@ -75,6 +75,10 @@ jobs:
         if: always()
         run: cd workflow-inference-compiler/ && pip install ".[all]"
 
+      - name: Generate WIC Validation Jsonschema
+        if: always()
+        run: cd workflow-inference-compiler/ && wic --generate_schemas_only
+
       - name: Install Molecular Modeling Workflows
         if: always()
         # Also run mm-workflows command to generate

.github/workflows/lint_and_test.yml

@@ -154,6 +154,10 @@ jobs:
         if: always()
         run: cd workflow-inference-compiler/ && pip install ".[all]"
 
+      - name: Generate WIC Validation Jsonschema
+        if: always()
+        run: cd workflow-inference-compiler/ && wic --generate_schemas_only
+
       - name: Install Molecular Modeling Workflows
         if: always()
         # Also run mm-workflows command to generate

.github/workflows/lint_and_test_macos.yml

@@ -80,6 +80,10 @@ jobs:
         if: always()
         run: cd workflow-inference-compiler/ && pip install ".[all]"
 
+      - name: Generate WIC Validation Jsonschema
+        if: always()
+        run: cd workflow-inference-compiler/ && wic --generate_schemas_only
+
       - name: Install Molecular Modeling Workflows
         if: always()
         # Also run mm-workflows command to generate

.github/workflows/run_workflows.yml

@@ -133,6 +133,10 @@ jobs:
       if: always()
       run: cd workflow-inference-compiler/ && pip install ".[all]"
 
+    - name: Generate WIC Validation Jsonschema
+      if: always()
+      run: cd workflow-inference-compiler/ && wic --generate_schemas_only
+
     - name: Install Molecular Modeling Workflows
       if: always()
       # Also run mm-workflows command to generate

.github/workflows/run_workflows_weekly.yml

@@ -97,6 +97,10 @@ jobs:
       if: always()
       run: cd workflow-inference-compiler/ && pip install ".[all]"
 
+    - name: Generate WIC Validation Jsonschema
+      if: always()
+      run: cd workflow-inference-compiler/ && wic --generate_schemas_only
+
     - name: Install Molecular Modeling Workflows
       if: always()
       # Also run mm-workflows command to generate

README.md

@@ -26,6 +26,7 @@ conda activate wic
 cd ..
 
 pip install -e ".[all]"
+wic --generate_schemas_only
 pre-commit install  # Required for developers
 
 cd install

docs/installguide.md

@@ -18,6 +18,7 @@ conda activate wic
 cd ..
 
 pip install -e ".[all]"
+wic --generate_schemas_only
 pre-commit install  # Required for developers
 
 cd install && ./install_mm-workflows.sh && cd ..

src/wic/ast.py

@@ -41,6 +41,21 @@ def read_ast_from_disk(homedir: str,
     """
     (step_id, yaml_tree) = yaml_tree_tuple
 
+    try:
+        validator.validate(yaml_tree)
+    except Exception as e:
+        yaml_path = Path(step_id.stem)
+        print('Failed to validate', yaml_path)
+        print(f'See validation_{yaml_path.stem}.txt for detailed technical information.')
+        # Do not display a nasty stack trace to the user; hide it in a file.
+        with open(f'validation_{yaml_path.stem}.txt', mode='w', encoding='utf-8') as f:
+            # https://mypy.readthedocs.io/en/stable/common_issues.html#python-version-and-system-platform-checks
+            if sys.version_info >= (3, 10):
+                traceback.print_exception(type(e), value=e, tb=None, file=f)
+            else:
+                traceback.print_exception(etype=type(e), value=e, tb=None, file=f)
+        sys.exit(1)
+
     wic = {'wic': yaml_tree.get('wic', {})}
     if 'backends' in wic['wic']:
         # Recursively expand each backend, but do NOT choose a specific backend.
@@ -93,20 +108,6 @@ def read_ast_from_disk(homedir: str,
             with open(yaml_path, mode='r', encoding='utf-8') as y:
                 sub_yaml_tree_raw: Yaml = yaml.safe_load(y.read())
 
-            try:
-                validator.validate(sub_yaml_tree_raw)
-            except Exception as e:
-                print('Failed to validate', yaml_path)
-                print(f'See validation_{yaml_path.stem}.txt for detailed technical information.')
-                # Do not display a nasty stack trace to the user; hide it in a file.
-                with open(f'validation_{yaml_path.stem}.txt', mode='w', encoding='utf-8') as f:
-                    # https://mypy.readthedocs.io/en/stable/common_issues.html#python-version-and-system-platform-checks
-                    if sys.version_info >= (3, 10):
-                        traceback.print_exception(type(e), value=e, tb=None, file=f)
-                    else:
-                        traceback.print_exception(etype=type(e), value=e, tb=None, file=f)
-                sys.exit(1)
-
             y_t = YamlTree(StepId(step_key, plugin_ns), sub_yaml_tree_raw)
             (step_id_, sub_yml_tree) = read_ast_from_disk(homedir, y_t, yml_paths, tools, validator)
 

src/wic/main.py

@@ -31,7 +31,7 @@ def main() -> None:
     # Generate schemas for validation and vscode IntelliSense code completion
     yaml_stems = utils.flatten([list(p) for p in yml_paths.values()])
     schema_store: Dict[str, Json] = {}
-    validator = wic_schema.get_validator(tools_cwl, yaml_stems, schema_store, write_to_disk=True)
+    validator = wic_schema.get_validator(tools_cwl, yaml_stems, schema_store, write_to_disk=False)
 
     # Generating yml schemas every time takes ~20 seconds and guarantees the
     # subworkflow schemas are always up to date. However, since it compiles all

src/wic/schemas/wic_schema.py

@@ -579,6 +579,13 @@ def get_validator(tools_cwl: Tools, yml_stems: List[str], schema_store: Dict[str
         with open('autogenerated/schemas/wic.json', mode='w', encoding='utf-8') as f:
             f.write(json.dumps(schema, indent=2))
 
+    # Load cached schema from disk
+    # NOTE: This may or may not be the correct solution. We should double check
+    # all of the call sites of get_validator and the write_to_disk parameters.
+    if Path('autogenerated/schemas/wic.json').exists():
+        with open('autogenerated/schemas/wic.json', mode='r', encoding='utf-8') as r:
+            schema = json.loads(r.read())
+
     # See https://stackoverflow.com/questions/53968770/how-to-set-up-local-file-references-in-python-jsonschema-document
     # The $ref tag refers to URIs defined in $id tags, NOT relative paths on
     # the local filesystem! We need to create a global mapping between ids and schemas