Improvements to DynamicPPLBenchmarks

.github/workflows/Benchmarking.yml

@@ -0,0 +1,32 @@
+name: Benchmarking
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  benchmark:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Set up Julia
+        uses: julia-actions/setup-julia@v2
+        with:
+          version: '1'
+
+      - name: Install Dependencies
+        run: julia --project=benchmarks/ -e 'using Pkg; Pkg.instantiate()'
+
+      - name: Run Benchmarks and Generate Reports
+        run: julia --project=benchmarks/ -e 'using DynamicPPLBenchmarks; weave_benchmarks()'
+
+      - name: Deploy to GitHub Pages
+        uses: peaceiris/actions-gh-pages@v3
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./benchmarks/results
+          publish_branch: gh-pages

benchmarks/Project.toml

@@ -4,10 +4,7 @@ version = "0.1.0"
 
 [deps]
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
-DiffUtils = "8294860b-85a6-42f8-8c35-d911f667b5f6"
 Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
 DynamicPPL = "366bfd00-2699-11ea-058f-f148b4cae6d8"
-LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433"
-Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a"
-Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
-Weave = "44d3d7a6-8a23-5bf8-98c5-b353f8df5ec9"
+PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d"
+TuringBenchmarking = "0db1332d-5c25-4deb-809f-459bc696f94f"

benchmarks/benchmarks.jl

@@ -0,0 +1,87 @@
+using DynamicPPL
+using DynamicPPLBenchmarks
+using BenchmarkTools
+using TuringBenchmarking
+using Distributions
+using PrettyTables
+
+# Define models
+@model function demo1(x)
+    m ~ Normal()
+    x ~ Normal(m, 1)
+    return (m = m, x = x)
+end
+
+@model function demo2(y)
+    p ~ Beta(1, 1)
+    N = length(y)
+    for n in 1:N
+        y[n] ~ Bernoulli(p)
+    end
+    return (; p)
+end
+
+demo1_data = randn()
+demo2_data = rand(Bool, 10)
+
+# Create model instances with the data
+demo1_instance = demo1(demo1_data)
+demo2_instance = demo2(demo2_data)
+
+# Define available AD backends
+available_ad_backends = Dict(
+    :forwarddiff => :forwarddiff,
+    :reversediff => :reversediff,
+    :zygote      => :zygote
+)
+
+# Define available VarInfo types.
+# Each entry is (Name, function to produce the VarInfo)
+available_varinfo_types = Dict(
+    :untyped           => ("UntypedVarInfo", VarInfo),
+    :typed             => ("TypedVarInfo", m -> VarInfo(m)),
+    :simple_namedtuple => ("SimpleVarInfo (NamedTuple)", m -> SimpleVarInfo{Float64}(m())),
+    :simple_dict       => ("SimpleVarInfo (Dict)", m -> begin
+        retvals = m()
+        varnames = map(keys(retvals)) do k
+            VarName{k}()
+        end
+        SimpleVarInfo{Float64}(Dict(zip(varnames, values(retvals))))
+    end)
+)
+
+# Specify the combinations to test:
+# (Model Name, model instance, VarInfo choice, AD backend)
+chosen_combinations = [
+    ("Demo1", demo1_instance, :typed,           :forwarddiff),
+    ("Demo1", demo1_instance, :simple_namedtuple, :zygote),
+    ("Demo2", demo2_instance, :untyped,           :reversediff),
+    ("Demo2", demo2_instance, :simple_dict,       :forwarddiff)
+]
+
+# Store results as tuples: (Model, AD Backend, VarInfo Type, Eval Time, AD Eval Time)
+results_table = Tuple{String, String, String, Float64, Float64}[]
+
+for (model_name, model, varinfo_choice, adbackend) in chosen_combinations
+    suite = make_suite(model, varinfo_choice, adbackend)
+    results = run(suite)
+    eval_time    = median(results["evaluation"]).time
+    ad_eval_time = median(results["AD_Benchmarking"]["evaluation"]["standard"]).time
+    push!(results_table, (model_name, string(adbackend), string(varinfo_choice), eval_time, ad_eval_time))
+end
+
+# Convert results to a 2D array for PrettyTables
+function to_matrix(tuples::Vector{<:NTuple{5,Any}})
+    n = length(tuples)
+    data = Array{Any}(undef, n, 5)
+    for i in 1:n
+        for j in 1:5
+            data[i, j] = tuples[i][j]
+        end
+    end
+    return data
+end
+
+table_matrix = to_matrix(results_table)
+header = ["Model", "AD Backend", "VarInfo Type", "Evaluation Time (ns)", "AD Eval Time (ns)"]
+pretty_table(table_matrix; header=header, tf=PrettyTables.tf_markdown)