feat(dataflow): Implement and thoroughly test live variable analysis

.github/workflows/ci.yaml

@@ -122,8 +122,19 @@ jobs:
       - name: Install Turnt part 2
         run: cd turnt ; flit install --symlink
 
-      - name: Test dataflow
+      - name: Test reaching definitions analysis
         run: |
           cd lesson4/dataflow
           python3 check.py def ../../bril/benchmarks/**/*.bril ../../bril/examples/test/df/*.bril
+      - name: Test live variables analysis
+        run: |
+          cd lesson4/dataflow
+          python3 match_outputs.py \
+            "python3 ../../bril/examples/df.py live | grep '  in:'" \
+            "cargo run --package dataflow --quiet -- --analysis live | grep in:" \
+            ../../bril/benchmarks/core/*.bril ../../bril/examples/test/df/*.bril \
+            --exclude is-decreasing.bril --exclude recfact.bril --exclude relative-primes.bril # differ on definition of basic block
+      - name: Snapshot test analyses on examples/test/df/ programs
+        run: |
+          cd lesson4/dataflow
           cd turnt && turnt df_copied_from_bril/*.bril

lesson2/build-cfg/src/lib.rs

@@ -175,7 +175,7 @@ impl FunctionCfgBuilder {
         }
     }
 
-    pub fn finish(mut self) -> Result<FunctionCfg, Whatever> {
+    pub fn finish(mut self, prune: bool) -> Result<FunctionCfg, Whatever> {
         for (block_idx, block) in &self.cfg.vertices {
             match &block.exit {
                 LabeledExit::Fallthrough => {
@@ -265,6 +265,27 @@ impl FunctionCfgBuilder {
             }
         }
 
+        if prune {
+            self.cfg.vertices.retain(|idx, _| {
+                if idx == self.cfg.entry {
+                    true
+                } else if let Some(rev_edges) = self.cfg.rev_edges.get(idx) {
+                    !rev_edges.is_empty()
+                } else {
+                    false
+                }
+            });
+            self.cfg
+                .edges
+                .retain(|idx, _| self.cfg.vertices.contains_key(idx));
+            self.cfg
+                .rev_edges
+                .retain(|idx, _| self.cfg.vertices.contains_key(idx));
+            for (_, rev_edges) in self.cfg.rev_edges.iter_mut() {
+                rev_edges.retain(|idx| self.cfg.vertices.contains_key(*idx));
+            }
+        }
+
         Ok(self.cfg)
     }
 }
@@ -274,7 +295,10 @@ fn pos_to_string(pos: Option<&Position>) -> String {
         .unwrap_or("<unknown>".into())
 }
 
-pub fn build_cfg(function: &Function) -> Result<FunctionCfg, Whatever> {
+pub fn build_cfg(
+    function: &Function,
+    prune: bool,
+) -> Result<FunctionCfg, Whatever> {
     let mut builder = FunctionCfgBuilder::new(
         function.name.clone(),
         function.args.clone(),
@@ -286,7 +310,11 @@ pub fn build_cfg(function: &Function) -> Result<FunctionCfg, Whatever> {
     for instruction in &function.instrs {
         match instruction {
             Code::Label { label, .. } => {
-                builder.finish_current_and_start_new_block();
+                if !builder.current_block.instructions.is_empty()
+                    || builder.current_block.label.is_some()
+                {
+                    builder.finish_current_and_start_new_block();
+                }
                 builder.set_current_label(label.clone());
             }
             Code::Instruction(instruction) => match instruction {
@@ -384,5 +412,5 @@ pub fn build_cfg(function: &Function) -> Result<FunctionCfg, Whatever> {
     builder.finish_current_and_start_new_block();
     //}
 
-    builder.finish()
+    builder.finish(prune)
 }

lesson2/build-cfg/src/main.rs

@@ -51,7 +51,7 @@ fn print_reconstructed(program: Program) -> Result<(), Whatever> {
     }
 
     for function in &program.functions {
-        let cfg = build_cfg(function).whatever_context(format!(
+        let cfg = build_cfg(function, false).whatever_context(format!(
             "Failed to build control-flow graph for function `{}`",
             function.name
         ))?;
@@ -66,7 +66,7 @@ fn print_pretty(program: Program) -> Result<(), Whatever> {
     let mut f = IndentWriter::new(&mut stdout, 4);
 
     for function in &program.functions {
-        let cfg = build_cfg(function).whatever_context(format!(
+        let cfg = build_cfg(function, true).whatever_context(format!(
             "Failed to build control-flow graph for function `{}`",
             function.name
         ))?;
@@ -227,7 +227,7 @@ fn print_pretty(program: Program) -> Result<(), Whatever> {
         }
 
         f.decrease_indent();
-        writeln!(f, "}}\n").whatever_context("Writing to stdout failed")?;
+        writeln!(f, "\n}}\n").whatever_context("Writing to stdout failed")?;
     }
 
     Ok(())

lesson3/lvn/src/main.rs

@@ -411,7 +411,7 @@ fn main() -> Result<(), Whatever> {
         println!("{}", import);
     }
     for function in program.functions {
-        let mut cfg = build_cfg::build_cfg(&function)
+        let mut cfg = build_cfg::build_cfg(&function, false)
             .whatever_context("Failed to build cfg")?;
 
         for block in cfg.vertices.values_mut() {

lesson3/tdce/src/main.rs

@@ -113,7 +113,7 @@ fn main() -> Result<(), Whatever> {
         println!("{}", import);
     }
     for function in program.functions {
-        let mut cfg = build_cfg::build_cfg(&function)
+        let mut cfg = build_cfg::build_cfg(&function, false)
             .whatever_context("Failed to build cfg")?;
 
         //trivial_dead_code_elimination(&mut cfg.vertices);

lesson4/bril-util/src/lib.rs

@@ -11,6 +11,8 @@ pub enum InstructionValue {
 pub trait InstructionExt {
     fn kill(&self) -> Option<&String>;
 
+    fn gen(&self) -> &[String];
+
     fn value(&self) -> Option<InstructionValue>;
 }
 
@@ -23,6 +25,14 @@ impl InstructionExt for Instruction {
         }
     }
 
+    fn gen(&self) -> &[String] {
+        match self {
+            Instruction::Value { args, .. }
+            | Instruction::Effect { args, .. } => args,
+            Instruction::Constant { .. } => &[],
+        }
+    }
+
     fn value(&self) -> Option<InstructionValue> {
         match self {
             Instruction::Constant { value, .. } => {

lesson4/dataflow/match_outputs.py

@@ -0,0 +1,101 @@
+import sys, subprocess, json
+import multiprocessing
+import difflib
+
+
+def parse_args():
+    if len(sys.argv) < 4:
+        print(
+            "usage: python3 match_outputs.py <oracle> <tested> <file>... [--exclude <file>]..."
+        )
+    oracle = sys.argv[1]
+    tested = sys.argv[2]
+    args = sys.argv[3:]
+    filenames = []
+    exclude = []
+    all_are_args = False
+    next_is_exclude = False
+    for arg in args:
+        if all_are_args:
+            filenames.append(arg)
+        elif arg == "--":
+            all_are_args = True
+        elif arg == "--exclude":
+            next_is_exclude = True
+        else:
+            if next_is_exclude:
+                exclude.append(arg)
+                next_is_exclude = False
+            else:
+                filenames.append(arg)
+    return (
+        oracle,
+        tested,
+        [
+            filename
+            for filename in filenames
+            if not any(isinstance(e, str) and filename.endswith(e) for e in exclude)
+        ],
+    )
+
+
+def init_worker(shared_failure_event, shared_oracle, shared_tested):
+    global failure_event
+    global oracle
+    global tested
+    failure_event = shared_failure_event
+    oracle = shared_oracle
+    tested = shared_tested
+
+
+def check_file(file):
+    oracle_output = subprocess.getoutput(f"bril2json <{file} | {oracle}")
+    my_output = subprocess.getoutput(f"bril2json <{file} | {tested}")
+    if oracle_output == my_output:
+        print(f"\x1b[32m{file} OK\x1b[m")
+    else:
+        print(f"\x1b[31m{file} ERROR\x1b[m")
+        failure_event.set()
+
+        red = lambda text: f"\033[38;2;255;0;0m{text}\033[m"
+        green = lambda text: f"\033[38;2;0;255;0m{text}\033[m"
+        blue = lambda text: f"\033[38;2;0;0;255m{text}\033[m"
+        white = lambda text: f"\033[38;2;255;255;255m{text}\033[m"
+        gray = lambda text: f"\033[38;2;128;128;128m{text}\033[m"
+
+        diff = difflib.ndiff(oracle_output.splitlines(), my_output.splitlines())
+        print("--- DIFF ---")
+        for line in diff:
+            if line.startswith("+"):
+                print(green(line))
+            elif line.startswith("-"):
+                print(red(line))
+            elif line.startswith("^"):
+                print(blue(line))
+            elif line.startswith("?"):
+                print(gray(line))
+            else:
+                print(white(line))
+
+
+if __name__ == "__main__":
+    if len(sys.argv) < 3:
+        print("usage: python3 test.py <oracle> <tested> <file>...")
+        sys.exit(1)
+    (oracle, tested, files) = parse_args()
+
+    with multiprocessing.Manager() as manager:
+        failure_event = manager.Event()
+
+        with multiprocessing.Pool(
+            multiprocessing.cpu_count(),
+            initializer=init_worker,
+            initargs=(failure_event, oracle, tested),
+        ) as pool:
+            pool.imap_unordered(check_file, files)
+            pool.close()
+            pool.join()
+            if failure_event.is_set():
+                print("Exiting due to errors")
+                pool.terminate()
+                sys.exit(1)

lesson4/dataflow/src/live_variables.rs

@@ -0,0 +1,61 @@
+use std::collections::HashSet;
+
+use bril_util::InstructionExt;
+use build_cfg::{BasicBlock, BasicBlockIdx, FunctionCfg};
+
+use crate::{solve_dataflow, Direction};
+
+#[derive(Debug, PartialEq, Eq, Hash, Clone)]
+pub struct Variable(String);
+
+fn transfer(
+    block: &BasicBlock,
+    _block_idx: BasicBlockIdx,
+    mut outputs: HashSet<Variable>,
+) -> HashSet<Variable> {
+    let mut kill_set = HashSet::new();
+    let mut gen_set = HashSet::new();
+    for instruction in &block.instructions {
+        gen_set.extend(
+            instruction
+                .gen()
+                .iter()
+                .filter(|variable| !kill_set.contains(variable))
+                .map(|variable| Variable(variable.to_string())),
+        );
+        if let Some(kill) = instruction.kill() {
+            kill_set.insert(kill);
+            outputs.remove(&Variable(kill.clone()));
+        }
+    }
+    outputs.extend(gen_set);
+    outputs
+}
+
+pub fn live_variables(cfg: &FunctionCfg) {
+    println!("@{} {{", cfg.signature.name);
+    for (block, solution) in solve_dataflow(
+        cfg,
+        Direction::Backward,
+        HashSet::new(),
+        |lhs, rhs| lhs.union(rhs).cloned().collect(),
+        transfer,
+    ) {
+        if let Some(label) = &cfg.vertices[block].label {
+            println!("  .{}", label.name);
+        }
+        let mut variables = solution
+            .into_iter()
+            .map(|variable| variable.0)
+            .collect::<Vec<_>>();
+        variables.sort();
+        println!(
+            "  in:  {}",
+            if variables.is_empty() {
+                "∅".to_string()
+            } else {
+                variables.join(", ")
+            }
+        );
+    }
+}

lesson4/dataflow/src/main.rs

@@ -12,13 +12,16 @@ use bril_rs::Program;
 use build_cfg::{
     slotmap::SecondaryMap, BasicBlock, BasicBlockIdx, FunctionCfg,
 };
+use live_variables::live_variables;
 use reaching_definitions::reaching_definitions;
 use snafu::{whatever, ResultExt, Whatever};
 
+pub mod live_variables;
 pub mod reaching_definitions;
 
 enum Analysis {
     ReachingDefinitions,
+    LiveVariables,
 }
 
 impl FromStr for Analysis {
@@ -27,6 +30,7 @@ impl FromStr for Analysis {
     fn from_str(s: &str) -> Result<Self, Self::Err> {
         Ok(match s {
             "def" => Self::ReachingDefinitions,
+            "live" => Self::LiveVariables,
             _ => whatever!("Unknown analysis '{}'", s),
         })
     }
@@ -143,11 +147,12 @@ fn main() -> Result<(), Whatever> {
     };
 
     for function in program.functions {
-        let cfg = build_cfg::build_cfg(&function)
+        let cfg = build_cfg::build_cfg(&function, true)
             .whatever_context("Failed to build cfg")?;
 
         match opts.analysis {
             Analysis::ReachingDefinitions => reaching_definitions(&cfg),
+            Analysis::LiveVariables => live_variables(&cfg),
         }
     }
 

lesson4/dataflow/turnt/df_copied_from_bril/cond-args.live.out

@@ -0,0 +1,8 @@
+@main {
+  in:  cond
+  .left
+  in:  a
+  .right
+  in:  ∅
+  .end
+  in:  a, c

lesson4/dataflow/turnt/df_copied_from_bril/cond.live.out

@@ -0,0 +1,8 @@
+@main {
+  in:  ∅
+  .left
+  in:  a
+  .right
+  in:  ∅
+  .end
+  in:  a, c

lesson4/dataflow/turnt/df_copied_from_bril/fact.live.out

@@ -0,0 +1,8 @@
+@main {
+  in:  ∅
+  .header
+  in:  i, result
+  .body
+  in:  i, result
+  .end
+  in:  result