feat: implement multi-file analysis. (#110)

Cargo.toml

@@ -1,6 +1,7 @@
 [workspace]
 members = [
     "wdl",
+    "wdl-analysis",
     "wdl-ast",
     "wdl-gauntlet",
     "wdl-grammar",
@@ -40,3 +41,10 @@ git2 = "0.18.3"
 temp-dir = "0.1.13"
 url = "2.5.2"
 urlencoding = "2.1.3"
+parking_lot = "0.12.3"
+reqwest = "0.12.5"
+petgraph = "0.6.5"
+futures = "0.3.30"
+glob = "0.3.1"
+path-clean = "1.0.1"
+indicatif = "0.17.8"

Gauntlet.toml

@@ -376,201 +376,11 @@ document = "biowdl/tasks:/umi.wdl"
 message = 'umi.wdl:39:60: error: unknown escape sequence `\.`'
 permalink = "https://github.com/biowdl/tasks/blob/2bf875300d90a3c9c8d670b3d99026452d5dbae2/umi.wdl/#L39"
 
-[[diagnostics]]
-document = "broadinstitute/gatk-tool-wdls:/wdls/ApplyBQSR.wdl"
-message = "ApplyBQSR.wdl:104:6: error: conflicting task name `ApplyBQSR`"
-permalink = "https://github.com/broadinstitute/gatk-tool-wdls/blob/4536dd2c6719b86f4c50348f0ce354818260cadb/wdls/ApplyBQSR.wdl/#L104"
-
-[[diagnostics]]
-document = "broadinstitute/gatk-tool-wdls:/wdls/ApplyBQSRAllArgs.wdl"
-message = "ApplyBQSRAllArgs.wdl:291:6: error: conflicting task name `ApplyBQSR`"
-permalink = "https://github.com/broadinstitute/gatk-tool-wdls/blob/4536dd2c6719b86f4c50348f0ce354818260cadb/wdls/ApplyBQSRAllArgs.wdl/#L291"
-
-[[diagnostics]]
-document = "broadinstitute/gatk-tool-wdls:/wdls/BaseRecalibrator.wdl"
-message = "BaseRecalibrator.wdl:111:6: error: conflicting task name `BaseRecalibrator`"
-permalink = "https://github.com/broadinstitute/gatk-tool-wdls/blob/4536dd2c6719b86f4c50348f0ce354818260cadb/wdls/BaseRecalibrator.wdl/#L111"
-
-[[diagnostics]]
-document = "broadinstitute/gatk-tool-wdls:/wdls/BaseRecalibratorAllArgs.wdl"
-message = "BaseRecalibratorAllArgs.wdl:318:6: error: conflicting task name `BaseRecalibrator`"
-permalink = "https://github.com/broadinstitute/gatk-tool-wdls/blob/4536dd2c6719b86f4c50348f0ce354818260cadb/wdls/BaseRecalibratorAllArgs.wdl/#L318"
-
-[[diagnostics]]
-document = "broadinstitute/gatk-tool-wdls:/wdls/ClipReads.wdl"
-message = "ClipReads.wdl:100:6: error: conflicting task name `ClipReads`"
-permalink = "https://github.com/broadinstitute/gatk-tool-wdls/blob/4536dd2c6719b86f4c50348f0ce354818260cadb/wdls/ClipReads.wdl/#L100"
-
-[[diagnostics]]
-document = "broadinstitute/gatk-tool-wdls:/wdls/ClipReadsAllArgs.wdl"
-message = "ClipReadsAllArgs.wdl:304:6: error: conflicting task name `ClipReads`"
-permalink = "https://github.com/broadinstitute/gatk-tool-wdls/blob/4536dd2c6719b86f4c50348f0ce354818260cadb/wdls/ClipReadsAllArgs.wdl/#L304"
-
-[[diagnostics]]
-document = "broadinstitute/gatk-tool-wdls:/wdls/CollectReadCounts.wdl"
-message = "CollectReadCounts.wdl:99:6: error: conflicting task name `CollectReadCounts`"
-permalink = "https://github.com/broadinstitute/gatk-tool-wdls/blob/4536dd2c6719b86f4c50348f0ce354818260cadb/wdls/CollectReadCounts.wdl/#L99"
-
-[[diagnostics]]
-document = "broadinstitute/gatk-tool-wdls:/wdls/CollectReadCountsAllArgs.wdl"
-message = "CollectReadCountsAllArgs.wdl:266:6: error: conflicting task name `CollectReadCounts`"
-permalink = "https://github.com/broadinstitute/gatk-tool-wdls/blob/4536dd2c6719b86f4c50348f0ce354818260cadb/wdls/CollectReadCountsAllArgs.wdl/#L266"
-
-[[diagnostics]]
-document = "broadinstitute/gatk-tool-wdls:/wdls/CombineGVCFs.wdl"
-message = "CombineGVCFs.wdl:112:6: error: conflicting task name `CombineGVCFs`"
-permalink = "https://github.com/broadinstitute/gatk-tool-wdls/blob/4536dd2c6719b86f4c50348f0ce354818260cadb/wdls/CombineGVCFs.wdl/#L112"
-
-[[diagnostics]]
-document = "broadinstitute/gatk-tool-wdls:/wdls/CombineGVCFsAllArgs.wdl"
-message = "CombineGVCFsAllArgs.wdl:323:6: error: conflicting task name `CombineGVCFs`"
-permalink = "https://github.com/broadinstitute/gatk-tool-wdls/blob/4536dd2c6719b86f4c50348f0ce354818260cadb/wdls/CombineGVCFsAllArgs.wdl/#L323"
-
-[[diagnostics]]
-document = "broadinstitute/gatk-tool-wdls:/wdls/CountBases.wdl"
-message = "CountBases.wdl:91:6: error: conflicting task name `CountBases`"
-permalink = "https://github.com/broadinstitute/gatk-tool-wdls/blob/4536dd2c6719b86f4c50348f0ce354818260cadb/wdls/CountBases.wdl/#L91"
-
-[[diagnostics]]
-document = "broadinstitute/gatk-tool-wdls:/wdls/CountBasesAllArgs.wdl"
-message = "CountBasesAllArgs.wdl:262:6: error: conflicting task name `CountBases`"
-permalink = "https://github.com/broadinstitute/gatk-tool-wdls/blob/4536dd2c6719b86f4c50348f0ce354818260cadb/wdls/CountBasesAllArgs.wdl/#L262"
-
-[[diagnostics]]
-document = "broadinstitute/gatk-tool-wdls:/wdls/CountReads.wdl"
-message = "CountReads.wdl:91:6: error: conflicting task name `CountReads`"
-permalink = "https://github.com/broadinstitute/gatk-tool-wdls/blob/4536dd2c6719b86f4c50348f0ce354818260cadb/wdls/CountReads.wdl/#L91"
-
-[[diagnostics]]
-document = "broadinstitute/gatk-tool-wdls:/wdls/CountReadsAllArgs.wdl"
-message = "CountReadsAllArgs.wdl:262:6: error: conflicting task name `CountReads`"
-permalink = "https://github.com/broadinstitute/gatk-tool-wdls/blob/4536dd2c6719b86f4c50348f0ce354818260cadb/wdls/CountReadsAllArgs.wdl/#L262"
-
-[[diagnostics]]
-document = "broadinstitute/gatk-tool-wdls:/wdls/FixMisencodedBaseQualityReads.wdl"
-message = "FixMisencodedBaseQualityReads.wdl:100:6: error: conflicting task name `FixMisencodedBaseQualityReads`"
-permalink = "https://github.com/broadinstitute/gatk-tool-wdls/blob/4536dd2c6719b86f4c50348f0ce354818260cadb/wdls/FixMisencodedBaseQualityReads.wdl/#L100"
-
-[[diagnostics]]
-document = "broadinstitute/gatk-tool-wdls:/wdls/FixMisencodedBaseQualityReadsAllArgs.wdl"
-message = "FixMisencodedBaseQualityReadsAllArgs.wdl:267:6: error: conflicting task name `FixMisencodedBaseQualityReads`"
-permalink = "https://github.com/broadinstitute/gatk-tool-wdls/blob/4536dd2c6719b86f4c50348f0ce354818260cadb/wdls/FixMisencodedBaseQualityReadsAllArgs.wdl/#L267"
-
-[[diagnostics]]
-document = "broadinstitute/gatk-tool-wdls:/wdls/FlagStat.wdl"
-message = "FlagStat.wdl:91:6: error: conflicting task name `FlagStat`"
-permalink = "https://github.com/broadinstitute/gatk-tool-wdls/blob/4536dd2c6719b86f4c50348f0ce354818260cadb/wdls/FlagStat.wdl/#L91"
-
-[[diagnostics]]
-document = "broadinstitute/gatk-tool-wdls:/wdls/FlagStatAllArgs.wdl"
-message = "FlagStatAllArgs.wdl:262:6: error: conflicting task name `FlagStat`"
-permalink = "https://github.com/broadinstitute/gatk-tool-wdls/blob/4536dd2c6719b86f4c50348f0ce354818260cadb/wdls/FlagStatAllArgs.wdl/#L262"
-
-[[diagnostics]]
-document = "broadinstitute/gatk-tool-wdls:/wdls/GatherTranches.wdl"
-message = "GatherTranches.wdl:95:6: error: conflicting task name `GatherTranches`"
-permalink = "https://github.com/broadinstitute/gatk-tool-wdls/blob/4536dd2c6719b86f4c50348f0ce354818260cadb/wdls/GatherTranches.wdl/#L95"
-
-[[diagnostics]]
-document = "broadinstitute/gatk-tool-wdls:/wdls/GatherTranchesAllArgs.wdl"
-message = "GatherTranchesAllArgs.wdl:146:6: error: conflicting task name `GatherTranches`"
-permalink = "https://github.com/broadinstitute/gatk-tool-wdls/blob/4536dd2c6719b86f4c50348f0ce354818260cadb/wdls/GatherTranchesAllArgs.wdl/#L146"
-
-[[diagnostics]]
-document = "broadinstitute/gatk-tool-wdls:/wdls/LeftAlignIndels.wdl"
-message = "LeftAlignIndels.wdl:112:6: error: conflicting task name `LeftAlignIndels`"
-permalink = "https://github.com/broadinstitute/gatk-tool-wdls/blob/4536dd2c6719b86f4c50348f0ce354818260cadb/wdls/LeftAlignIndels.wdl/#L112"
-
-[[diagnostics]]
-document = "broadinstitute/gatk-tool-wdls:/wdls/LeftAlignIndelsAllArgs.wdl"
-message = "LeftAlignIndelsAllArgs.wdl:267:6: error: conflicting task name `LeftAlignIndels`"
-permalink = "https://github.com/broadinstitute/gatk-tool-wdls/blob/4536dd2c6719b86f4c50348f0ce354818260cadb/wdls/LeftAlignIndelsAllArgs.wdl/#L267"
-
-[[diagnostics]]
-document = "broadinstitute/gatk-tool-wdls:/wdls/PrintReads.wdl"
-message = "PrintReads.wdl:100:6: error: conflicting task name `PrintReads`"
-permalink = "https://github.com/broadinstitute/gatk-tool-wdls/blob/4536dd2c6719b86f4c50348f0ce354818260cadb/wdls/PrintReads.wdl/#L100"
-
-[[diagnostics]]
-document = "broadinstitute/gatk-tool-wdls:/wdls/PrintReadsAllArgs.wdl"
-message = "PrintReadsAllArgs.wdl:267:6: error: conflicting task name `PrintReads`"
-permalink = "https://github.com/broadinstitute/gatk-tool-wdls/blob/4536dd2c6719b86f4c50348f0ce354818260cadb/wdls/PrintReadsAllArgs.wdl/#L267"
-
-[[diagnostics]]
-document = "broadinstitute/gatk-tool-wdls:/wdls/ReadAnonymizer.wdl"
-message = "ReadAnonymizer.wdl:112:6: error: conflicting task name `ReadAnonymizer`"
-permalink = "https://github.com/broadinstitute/gatk-tool-wdls/blob/4536dd2c6719b86f4c50348f0ce354818260cadb/wdls/ReadAnonymizer.wdl/#L112"
-
-[[diagnostics]]
-document = "broadinstitute/gatk-tool-wdls:/wdls/ReadAnonymizerAllArgs.wdl"
-message = "ReadAnonymizerAllArgs.wdl:275:6: error: conflicting task name `ReadAnonymizer`"
-permalink = "https://github.com/broadinstitute/gatk-tool-wdls/blob/4536dd2c6719b86f4c50348f0ce354818260cadb/wdls/ReadAnonymizerAllArgs.wdl/#L275"
-
-[[diagnostics]]
-document = "broadinstitute/gatk-tool-wdls:/wdls/RevertBaseQualityScores.wdl"
-message = "RevertBaseQualityScores.wdl:100:6: error: conflicting task name `RevertBaseQualityScores`"
-permalink = "https://github.com/broadinstitute/gatk-tool-wdls/blob/4536dd2c6719b86f4c50348f0ce354818260cadb/wdls/RevertBaseQualityScores.wdl/#L100"
-
-[[diagnostics]]
-document = "broadinstitute/gatk-tool-wdls:/wdls/RevertBaseQualityScoresAllArgs.wdl"
-message = "RevertBaseQualityScoresAllArgs.wdl:267:6: error: conflicting task name `RevertBaseQualityScores`"
-permalink = "https://github.com/broadinstitute/gatk-tool-wdls/blob/4536dd2c6719b86f4c50348f0ce354818260cadb/wdls/RevertBaseQualityScoresAllArgs.wdl/#L267"
-
-[[diagnostics]]
-document = "broadinstitute/gatk-tool-wdls:/wdls/SplitNCigarReads.wdl"
-message = "SplitNCigarReads.wdl:112:6: error: conflicting task name `SplitNCigarReads`"
-permalink = "https://github.com/broadinstitute/gatk-tool-wdls/blob/4536dd2c6719b86f4c50348f0ce354818260cadb/wdls/SplitNCigarReads.wdl/#L112"
-
-[[diagnostics]]
-document = "broadinstitute/gatk-tool-wdls:/wdls/SplitNCigarReadsAllArgs.wdl"
-message = "SplitNCigarReadsAllArgs.wdl:291:6: error: conflicting task name `SplitNCigarReads`"
-permalink = "https://github.com/broadinstitute/gatk-tool-wdls/blob/4536dd2c6719b86f4c50348f0ce354818260cadb/wdls/SplitNCigarReadsAllArgs.wdl/#L291"
-
-[[diagnostics]]
-document = "broadinstitute/gatk-tool-wdls:/wdls/SplitReads.wdl"
-message = "SplitReads.wdl:95:6: error: conflicting task name `SplitReads`"
-permalink = "https://github.com/broadinstitute/gatk-tool-wdls/blob/4536dd2c6719b86f4c50348f0ce354818260cadb/wdls/SplitReads.wdl/#L95"
-
-[[diagnostics]]
-document = "broadinstitute/gatk-tool-wdls:/wdls/SplitReadsAllArgs.wdl"
-message = "SplitReadsAllArgs.wdl:274:6: error: conflicting task name `SplitReads`"
-permalink = "https://github.com/broadinstitute/gatk-tool-wdls/blob/4536dd2c6719b86f4c50348f0ce354818260cadb/wdls/SplitReadsAllArgs.wdl/#L274"
-
-[[diagnostics]]
-document = "broadinstitute/gatk-tool-wdls:/wdls/UnmarkDuplicates.wdl"
-message = "UnmarkDuplicates.wdl:100:6: error: conflicting task name `UnmarkDuplicates`"
-permalink = "https://github.com/broadinstitute/gatk-tool-wdls/blob/4536dd2c6719b86f4c50348f0ce354818260cadb/wdls/UnmarkDuplicates.wdl/#L100"
-
-[[diagnostics]]
-document = "broadinstitute/gatk-tool-wdls:/wdls/UnmarkDuplicatesAllArgs.wdl"
-message = "UnmarkDuplicatesAllArgs.wdl:267:6: error: conflicting task name `UnmarkDuplicates`"
-permalink = "https://github.com/broadinstitute/gatk-tool-wdls/blob/4536dd2c6719b86f4c50348f0ce354818260cadb/wdls/UnmarkDuplicatesAllArgs.wdl/#L267"
-
-[[diagnostics]]
-document = "broadinstitute/palantir-workflows:/BenchmarkSVs/WittyerTasks.wdl"
-message = "WittyerTasks.wdl:74:14: error: conflicting declaration name `wittyer_config`"
-permalink = "https://github.com/broadinstitute/palantir-workflows/blob/1e32078d3b57dcb2291534d0caa30f600d40967e/BenchmarkSVs/WittyerTasks.wdl/#L74"
-
-[[diagnostics]]
-document = "broadinstitute/palantir-workflows:/BenchmarkVCFs/BenchmarkAndCompareVCFs.wdl"
-message = "BenchmarkAndCompareVCFs.wdl:43:14: error: conflicting declaration name `preemptible`"
-permalink = "https://github.com/broadinstitute/palantir-workflows/blob/1e32078d3b57dcb2291534d0caa30f600d40967e/BenchmarkVCFs/BenchmarkAndCompareVCFs.wdl/#L43"
-
 [[diagnostics]]
 document = "broadinstitute/palantir-workflows:/HaplotypeMap/BuildHaplotypeMap.wdl"
 message = "BuildHaplotypeMap.wdl:17:1: error: a WDL document must start with a version statement"
 permalink = "https://github.com/broadinstitute/palantir-workflows/blob/1e32078d3b57dcb2291534d0caa30f600d40967e/HaplotypeMap/BuildHaplotypeMap.wdl/#L17"
 
-[[diagnostics]]
-document = "broadinstitute/palantir-workflows:/LongReadRNABenchmark/IsoformDiscoveryBenchmark.wdl"
-message = "IsoformDiscoveryBenchmark.wdl:147:13: error: conflicting scatter variable name `gtf`"
-permalink = "https://github.com/broadinstitute/palantir-workflows/blob/1e32078d3b57dcb2291534d0caa30f600d40967e/LongReadRNABenchmark/IsoformDiscoveryBenchmark.wdl/#L147"
-
-[[diagnostics]]
-document = "broadinstitute/warp:/pipelines/broad/dna_seq/somatic/single_sample/wgs/gdc_genome/GDCWholeGenomeSomaticSingleSample.wdl"
-message = "GDCWholeGenomeSomaticSingleSample.wdl:672:14: error: conflicting scatter variable name `ubam`"
-permalink = "https://github.com/broadinstitute/warp/blob/e49ffbcd72b138ad4dc6ec62c847943908e4abf9/pipelines/broad/dna_seq/somatic/single_sample/wgs/gdc_genome/GDCWholeGenomeSomaticSingleSample.wdl/#L672"
-
 [[diagnostics]]
 document = "broadinstitute/warp:/pipelines/skylab/scATAC/scATAC.wdl"
 message = "scATAC.wdl:203:9: error: duplicate key `cpu` in runtime section"

README.md

@@ -90,21 +90,27 @@ the `wdl` family of crates.
 The `wdl` CLI tool can be run with the following command:
 
 ```bash
-cargo run --bin wdl --features binaries -- $ARGS
+cargo run --bin wdl --features cli -- $ARGS
 ```
 
 Where `$ARGS` are the command line arguments to the `wdl` CLI tool.
 
 The `wdl` CLI tool currently supports three subcommands:
 
-* `parse` - Parses a WDL source file and prints both the parse diagnostics and
-  the resulting Concrete Syntax Tree (CST).
-* `check` - Parses and validates a WDL source file. Exits with a status code of
-  `0` if the file is syntactically valid; otherwise, prints the validation
-  diagnostics and exits with a status code of `1`.
-* `lint` - Parses, validates, and runs the linting rules on a WDL
-  source file. Exits with a status code of `0` if the file passes all lints;
-  otherwise, prints the linting diagnostics and exits with a status code of `1`.
+* `parse` - Parses a WDL document and prints both the parse diagnostics and the
+  resulting Concrete Syntax Tree (CST).
+* `check` - Parses, validates, and analyzes a WDL document or a directory
+  containing WDL documents. Exits with a status code of `0` if the documents
+  are valid; otherwise, prints the validation diagnostics and exits with a
+  status code of `1`.
+* `lint` - Parses, validates, and runs the linting rules on a WDL document.
+  Exits with a status code of `0` if the file passes all lints; otherwise,
+  prints the linting diagnostics and exits with a status code of `1`.
+* `analyze` - Parses, validates, and analyzes a single WDL document or a
+  directory containing WDL documents. Prints a debug representation of the
+  document scopes and exits with a status code of `0` if the documents are
+  valid; otherwise, prints the validation diagnostics and exits with a status
+  code of `1`.
 
 Each of the subcommands supports passing `-` as the file path to denote reading
 from STDIN instead of a file on disk.

wdl-analysis/CHANGELOG.md

@@ -0,0 +1,12 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## Unreleased
+
+### Added
+
+* Added the `wdl-analysis` crate for analyzing WDL documents ([#110](https://github.com/stjude-rust-labs/wdl/pull/110)).

wdl-analysis/Cargo.toml

@@ -0,0 +1,41 @@
+[package]
+name = "wdl-analysis"
+version = "0.1.0"
+license.workspace = true
+edition.workspace = true
+authors.workspace = true
+homepage.workspace = true
+repository.workspace = true
+description = "Analysis of Workflow Description Language (WDL) documents."
+documentation = "https://docs.rs/wdl-analysis"
+
+[dependencies]
+wdl-ast = { path = "../wdl-ast", version = "0.4.0" }
+wdl-lint = { path = "../wdl-lint", version = "0.3.0", optional = true }
+anyhow = { workspace = true }
+rowan = { workspace = true }
+url = { workspace = true }
+tokio = { workspace = true }
+parking_lot = { workspace = true }
+log = { workspace = true }
+rayon = { workspace = true }
+reqwest = { workspace = true }
+petgraph = { workspace = true }
+futures = { workspace = true }
+glob = { workspace = true }
+path-clean = { workspace = true }
+indexmap = { workspace = true }
+
+[dev-dependencies]
+pretty_assertions = { workspace = true }
+colored = { workspace = true }
+codespan-reporting = { workspace = true }
+
+[features]
+default = []
+codespan = ["wdl-ast/codespan"]
+
+[[test]]
+name = "analysis"
+required-features = ["codespan"]
+harness = false