Add workflow visualization (#5)

.github/workflows/tox.yml

@@ -22,6 +22,9 @@ jobs:
 
     - uses: actions/checkout@v4
 
+    - name: Install graphviz for viz extras
+      run: sudo apt-get update && sudo apt-get install -y graphviz
+
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v4
       with:

doc/source/conf.py

@@ -57,10 +57,26 @@
 # html_static_path = ['_static']
 
 html_theme_options = {
+    "repo_url": "https://github.com/BlueBrain/blue-cwl",
+    "repo_name": "BlueBrain/blue-cwl",
     "metadata_distribution": "blue_cwl",
 }
 
 html_title = "blue-cwl"
 
 # If true, links to the reST sources are added to the pages.
 html_show_sourcelink = False
+
+import os
+from pathlib import Path
+from blue_cwl.variant import iter_registered_variants
+from blue_cwl.core import cwl
+from blue_cwl.utils import create_dir
+
+graph_dir = create_dir("./generated")
+
+for variant in iter_registered_variants():
+    tool = variant.tool_definition
+    name = f"{variant.generator_name}__{variant.variant_name}__{variant.version}.svg"
+    if isinstance(tool, cwl.Workflow):
+        tool.write_image(filepath=graph_dir / name)

doc/source/index.rst

@@ -6,4 +6,5 @@
    :maxdepth: 2
 
    Home <self>
+   registry
    changelog

doc/source/registry.rst

@@ -0,0 +1,10 @@
+
+.. _registry:
+
+Registry
+========
+
+connectome_filtering|synapses|v2
+********************************
+
+.. image:: generated/connectome_filtering__synapses__v2.svg

pyproject.toml

@@ -28,6 +28,8 @@ dependencies = [
   "morph-tool",
   "jsonschema",
   "luigi",
+  "pydot",
+  "matplotlib",
 ]
 classifiers = [
   "Development Status :: 2 - Pre-Alpha",

src/blue_cwl/core/cwl.py

@@ -279,3 +279,15 @@ def make_workflow_step(self, step_name, input_values, sources):
         return blue_cwl.core.process.build_workflow_step_process(
             self, step_name, input_values, sources
         )
+
+    def show_image(self):
+        """Show workflow graph image."""
+        from blue_cwl.core import viz
+
+        viz.show_workflow_graph_image(self)
+
+    def write_image(self, filepath):
+        """Save workflow graph as an image."""
+        from blue_cwl.core import viz
+
+        viz.write_workflow_graph_image(self, filepath)

src/blue_cwl/core/viz.py

@@ -0,0 +1,101 @@
+"""CWL visualization."""
+
+import io
+from pathlib import Path
+
+import matplotlib.image as mpimg
+import matplotlib.pyplot as plt
+import pydot
+
+
+def show_workflow_graph_image(cwl_workflow):
+    """Show Workflow graph."""
+    graph = _build_workflow_graph(cwl_workflow)
+
+    png_str = graph.create_png()  # pylint: disable=no-member
+    # treat the DOT output as an image file
+    sio = io.BytesIO()
+    sio.write(png_str)
+    sio.seek(0)
+    img = mpimg.imread(sio)
+
+    # plot the image
+    plt.imshow(img, aspect="equal")
+    plt.axis("off")
+    plt.show()
+
+
+def write_workflow_graph_image(cwl_workflow, filepath):
+    """Write workflow graph image."""
+    graph = _build_workflow_graph(cwl_workflow)
+    fmt = Path(filepath).suffix[1:]
+    graph.write(filepath, format=fmt)
+
+
+def _build_workflow_graph(cwl_workflow):
+    def file_node(name, label):
+        return pydot.Node(
+            name,
+            label=label,
+            fontsize=20,
+            shape="note",
+            color="gold",
+            fillcolor="gold",
+            style="filled",
+        )
+
+    def step_node(name, label):
+        return pydot.Node(
+            name,
+            label=label,
+            fontsize=30,
+            shape="oval",
+            color="aquamarine",
+            fillcolor="aquamarine",
+            style="filled",
+        )
+
+    graph = pydot.Dot(repr(cwl_workflow), graph_type="digraph")
+
+    # add inputs as indepdenent nodes
+    for name in cwl_workflow.inputs:
+        node_name = f"inputs__{name}"
+        graph.add_node(file_node(node_name, name))
+
+    # add workflow steps as independent nodes
+    for step in cwl_workflow.iter_steps():
+        graph.add_node(step_node(step.id, step.id))
+        for name in step.outputs:
+            node_name = f"{step.id}__{name}"
+            graph.add_node(file_node(node_name, name))
+            graph.add_edge(pydot.Edge(step.id, node_name))
+
+    for step in cwl_workflow.iter_steps():
+        # use a set to avoid creating multiple arrows from
+        # the same source. We want a rough idea of the workflow.
+        visited = set()
+        for inp in step.inputs.values():
+            res = inp.split_source_output()
+
+            if res is None:
+                continue
+
+            for source_name, output_name in res:
+                visited.add(source_name)
+
+                if source_name is None:
+                    source_name = f"inputs__{output_name}"
+                else:
+                    source_name = f"{source_name}__{output_name}"
+
+                if source_name not in visited:
+                    visited.add(source_name)
+
+                    graph.add_edge(
+                        pydot.Edge(
+                            source_name,
+                            step.id,
+                        )
+                    )
+
+    return graph

src/blue_cwl/variant.py

@@ -406,6 +406,17 @@ def _get_variant_directory(generator_name: str, variant_name: str, version: str)
     return _get_variant_dir(package_path, generator_name, variant_name, version)
 
 
+def iter_registered_variants():
+    """Iterate over all variants in the registry."""
+    package_path = importlib.resources.files("blue_cwl")
+    root_dir = _check_directory_exists(package_path / "generators")
+
+    for generator_dir in filter(lambda o: o.is_dir(), root_dir.iterdir()):
+        for variant_dir in filter(lambda o: o.is_dir(), generator_dir.iterdir()):
+            for version_dir in filter(lambda o: o.is_dir(), variant_dir.iterdir()):
+                yield Variant.from_registry(generator_dir.name, variant_dir.name, version_dir.name)
+
+
 def _get_variant_file(generator_name: str, variant_name: str, version: str) -> Path:
     variant_dir = _get_variant_directory(generator_name, variant_name, version)
 

tests/unit/core/test_viz.py

@@ -0,0 +1,23 @@
+import pytest
+from pathlib import Path
+from unittest.mock import patch
+from blue_cwl.core import parse_cwl_file
+
+
+DATA_DIR = Path(__file__).parent / "data/cat-echo"
+
+@pytest.fixture
+def workflow():
+    return parse_cwl_file(DATA_DIR / "workflow-cat-echo.cwl")
+
+
+def test_show_workflow_image(workflow):
+    """Test that image generation works."""
+    with patch("blue_cwl.core.viz.plt.show"):
+        workflow.show_image()
+
+
+def test_write_workflow_image(tmp_path, workflow):
+    """Test that image writing works."""
+    out_file = tmp_path / "image.png"
+    workflow.write_image(filepath=out_file)

tox.ini

@@ -40,7 +40,7 @@ passenv =
     MODULEPATH
     LOADEDMODULES
     _LMFILES_
-setenv = 
+setenv =
     NEXUS_BASE = {env:NEXUS_BASE:https://staging.nise.bbp.epfl.ch/nexus/v1}
     NEXUS_ORG = {env:NEXUS_ORG:bbp_test}
     NEXUS_PROJ =  {env:NEXUS_PROJ:studio_data_11}
@@ -127,11 +127,15 @@ omit = */blue_cwl/app/*
 [testenv:docs]
 changedir = doc
 extras = docs
-# set warnings as errors using the -W sphinx option
-commands = make html SPHINXOPTS=-W
+commands =
+    sphinx-build -b html \
+        {toxinidir}/doc/source \
+        {toxinidir}/doc/build/html \
+        -d {toxinidir}/doc/build/doctrees \
+        -W -T
 allowlist_externals =
-    {[testenv]allowlist_externals}
-    make
+    /bin/mkdir
+
 
 [pycodestyle]
 # E731: do not assign a lambda expression, use a def