Merge pull request #138 from biaslab/dev-examples

Reorganisation of examples

.gitignore

@@ -10,6 +10,7 @@ examples/*incomplete.ipynb
 examples/*.jl
 examples/*WIP*
 examples/*gif
+!examples/.meta.jl
 
 _output
 test/_output
@@ -36,3 +37,5 @@ examples/*Compiled
 
 **/.benchmarkci
 benchmark/*.json
+
+.venv

docs/make.jl

@@ -5,23 +5,32 @@ using Documenter
 ## https://gr-framework.org/workstations.html#no-output
 ENV["GKSwstype"] = "100"
 
-DocMeta.setdocmeta!(RxInfer, :DocTestSetup, :(using RxInfer); recursive=true)
+DocMeta.setdocmeta!(RxInfer, :DocTestSetup, :(using RxInfer); recursive = true)
 
 # This must be auto-generated with `make examples`
 ExamplesPath = joinpath(@__DIR__, "src", "examples")
 ExamplesOverviewPath = joinpath(ExamplesPath, "overview.md")
 ExamplesMeta = include(joinpath(@__DIR__, "..", "examples", ".meta.jl"))
-Examples = map(filter(example -> !example.hidden, ExamplesMeta)) do examplemeta
-    path = examplemeta.path
-    mdpath = replace(path, ".ipynb" => ".md")
+
+ExamplesCategories = ExamplesMeta[:categories]
+ExamplesCategories = ExamplesCategories[setdiff(keys(ExamplesCategories), [:hidden_examples])]
+
+ExamplesCategoriesOverviewPaths = map(collect(pairs(ExamplesCategories))) do (label, _)
+    return joinpath(ExamplesPath, string(label), "overview.md")
+end
+
+Examples = map(filter(example -> !isequal(example[:category], :hidden_examples), ExamplesMeta[:examples])) do examplemeta
+    filename = examplemeta.filename
+    category = examplemeta.category
+    mdpath = replace(filename, ".ipynb" => ".md")
     title = examplemeta.title
 
-    fullpath = joinpath(ExamplesPath, mdpath)
-    shortpath = joinpath("examples", mdpath)
+    fullpath = joinpath(ExamplesPath, string(category), mdpath)
+    shortpath = joinpath("examples", string(category), mdpath)
 
     # We use `fullpath` to check if the file exists
     # We use `shortpath` to make a page reference in the left panel in the documentation
-    return title => (fullpath, shortpath)
+    return title => (fullpath, shortpath, category)
 end
 
 if !isdir(ExamplesPath)
@@ -30,33 +39,47 @@ end
 
 # Check if some examples are missing from the build
 # The `isfile` check needs only the full path, so we ignore the short path
-ExistingExamples = filter(Examples) do (title, paths)
-    fullpath, _ = paths
+ExistingExamples = filter(Examples) do (title, info)
+    fullpath, _, _ = info
     exists = isfile(fullpath)
     if !exists
         @warn "Example at path $(fullpath) does not exist. Skipping."
     end
     return exists
 end
 
+# Create an array of pages for each category
+ExamplesCategoriesPages = map(collect(pairs(ExamplesCategories))) do (label, category)
+    return label => (title = category.title, pages = [
+        "Overview" => joinpath("examples", string(label), "overview.md")
+    ])
+end |> NamedTuple
+
 # The `pages` argument in the `makedocs` needs only a short path, so we ignore the full path
-ExistingExamplesPages = map(ExistingExamples) do (title, paths)
-    _, shortpath = paths
-    return title => shortpath
+foreach(ExistingExamples) do (title, info)
+    _, shortpath, category = info
+    push!(ExamplesCategoriesPages[category].pages, title => shortpath)
 end
 
 if length(Examples) !== length(ExistingExamples)
     @warn "Some examples were not found. Use the `make examples` command to generate all examples."
 end
 
-# Check if the `overview.md` file exists
-if !isfile(ExamplesOverviewPath)
-    @warn "`$(ExamplesOverviewPath)` does not exist. Generating an empty overview. Use the `make examples` command to generate the overview and all examples."
-    open(ExamplesOverviewPath, "w") do overview
-        write(overview, "The overview is missing. Use the `make examples` command to generate the overview and all examples.")
+# Check if the main `overview.md` file exists + sub categories `overview.md` files
+foreach(vcat(ExamplesOverviewPath, ExamplesCategoriesOverviewPaths)) do path
+    if !isfile(path)
+        @warn "`$(path)` does not exist. Generating an empty overview. Use the `make examples` command to generate the overview and all examples."
+        open(path, "w") do f
+            write(f, "The overview is missing. Use the `make examples` command to generate the overview and all examples.")
+        end
     end
 end
 
+# Generate the final list of examples for each sub-category
+ExamplesPages = map(collect(pairs(ExamplesCategoriesPages))) do (label, info)
+    return info.title => info.pages
+end
+
 # WIP: Keep it as a nice starting approach for adding a header, currently we are using `assets/header.js`
 # struct DocumentationWriter <: Documenter.Writer
 #     base :: Documenter.HTML
@@ -72,37 +95,29 @@ end
 # end
 
 makedocs(;
-    draft=false,
-    strict=[:doctest, :eval_block, :example_block, :meta_block, :parse_error, :setup_block],
-    modules=[RxInfer],
-    authors="Bagaev Dmitry <d.v.bagaev@tue.nl> and contributors",
-    repo="https://github.com/biaslab/RxInfer.jl/blob/{commit}{path}#{line}",
-    sitename="RxInfer.jl",
-    format=Documenter.HTML(;
-        prettyurls=get(ENV, "CI", "false") == "true",
-        canonical="https://biaslab.github.io/RxInfer.jl",
-        edit_link="main",
-        assets=String["assets/theme.css", "assets/header.css", "assets/header.js"]
+    draft = false,
+    strict = [:doctest, :eval_block, :example_block, :meta_block, :parse_error, :setup_block],
+    modules = [RxInfer],
+    authors = "Bagaev Dmitry <d.v.bagaev@tue.nl> and contributors",
+    repo = "https://github.com/biaslab/RxInfer.jl/blob/{commit}{path}#{line}",
+    sitename = "RxInfer.jl",
+    format = Documenter.HTML(;
+        prettyurls = get(ENV, "CI", "false") == "true",
+        canonical = "https://biaslab.github.io/RxInfer.jl",
+        edit_link = "main",
+        assets = String["assets/theme.css", "assets/header.css", "assets/header.js"]
     ),
-    pages=[
+    pages = [
         "Home" => "index.md",
         "User guide" => [
             # "Background: variational inference" => "manuals/background.md",
-            "Getting started" => "manuals/getting-started.md",
-            "Model specification" => "manuals/model-specification.md",
+            "Getting started"           => "manuals/getting-started.md",
+            "Model specification"       => "manuals/model-specification.md",
             "Constraints specification" => "manuals/constraints-specification.md",
-            "Meta specification" => "manuals/meta-specification.md",
-            "Inference specification" => [
-                "Overview" => "manuals/inference/overview.md",
-                "Static dataset" => "manuals/inference/inference.md",
-                "Real-time dataset / reactive inference" => "manuals/inference/rxinference.md",
-                "Inference results postprocessing" => "manuals/inference/postprocess.md",
-                "Manual inference specification" => "manuals/inference/manual.md"
-            ],
-            "Inference customization" => [
-                "Defining a custom node and rules" => "manuals/custom-node.md",
-            ],
-            "Debugging"                 => "manuals/debugging.md",
+            "Meta specification"        => "manuals/meta-specification.md",
+            "Inference specification"   => ["Overview" => "manuals/inference/overview.md", "Static dataset" => "manuals/inference/inference.md", "Real-time dataset / reactive inference" => "manuals/inference/rxinference.md", "Inference results postprocessing" => "manuals/inference/postprocess.md", "Manual inference specification" => "manuals/inference/manual.md"],
+            "Inference customization"   => ["Defining a custom node and rules" => "manuals/custom-node.md"],
+            "Debugging"                 => "manuals/debugging.md"
         ],
         "Library" => [
             "Built-in functional form constraints" => "library/functional-forms.md",
@@ -112,17 +127,11 @@ makedocs(;
         ],
         "Examples" => [
             "Overview" => "examples/overview.md", # This must be auto-generated with `make examples`
-            ExistingExamplesPages...
+            ExamplesPages...
         ],
-        "Contributing" => [
-            "Overview" => "contributing/overview.md",
-            "Adding a new example" => "contributing/new-example.md",
-            "Publishing a new release" => "contributing/new-release.md"
-        ]
+        "Contributing" =>
+            ["Overview" => "contributing/overview.md", "Adding a new example" => "contributing/new-example.md", "Publishing a new release" => "contributing/new-release.md"]
     ]
 )
 
-deploydocs(;
-    repo="github.com/biaslab/RxInfer.jl",
-    devbranch="main"
-)
+deploydocs(; repo = "github.com/biaslab/RxInfer.jl", devbranch = "main")

docs/src/contributing/new-example.md

@@ -2,14 +2,14 @@
 
 We welcome all possible contributors. This page details some of the guidelines that should be followed when adding a new example (in the `examples/` folder) to this package.
 
-In order to add a new example simply create a new Jupyter notebook with your experiments in the `examples/` folder. When creating a new example add a descriptive explanation of your experiments, model specification, inference constraints decisions and add appropriate results analysis. We expect examples to be readable for the general public and therefore highly value descriptive comments. If a submitted example only contains code, we will kindly request some changes to improve the readability.
+In order to add a new example simply create a new Jupyter notebook with your experiments in the `examples/"subcategory"` folder. When creating a new example add a descriptive explanation of your experiments, model specification, inference constraints decisions and add appropriate results analysis. We expect examples to be readable for the general public and therefore highly value descriptive comments. If a submitted example only contains code, we will kindly request some changes to improve the readability.
 
-After preparing the new example it is necessary to modify the `examples/.meta.jl` file. See the comments in this file for more information.
+After preparing the new example it is necessary to modify the `examples/.meta.jl` file.
 
 1. Make sure that the very first cell of the notebook contains ONLY `# <title>` in it and has the markdown cell type. This is important for generating links in our documentation.
-2. The `path` option must be set to a local path and cannot contain subfolders.
+2. The `path` option must be set to a local path in a category sub-folder.
 3. The text in the `description` option will be used on the `Examples` page in the documentation.
-4. Set `hidden = true` option to hide a certain example in the documentation (the example will be executed to ensure it runs without errors).
+4. Set `category = :hidden_examples` to hide a certain example in the documentation (the example will be executed to ensure it runs without errors).
 5. Please do no use `Overview` as a name for the new example, the title `Overview` is reserved.
 6. Use the following template for equations, note that `$$` and both `\begin` and `\end` commands are on the same line (check other examples if you are not sure). This is important, because otherwise formulas may not render correctly. Inline equations may use `$...$` template.
 ```
@@ -18,10 +18,15 @@ $$\begin{aligned}
 \end{aligned}$$
 ``` 
 7. When using equations, make sure not to follow the left-hand `$$` or `$` with a space, but instead directly start the equation, e.g. not `$$ a + b $$`, but `$$a + b$$`. For equations that are supposed to be on a separate line, make sure `$$...$$` is preceded and followed by an empty line.
-8. Notebooks and plain Julia have different scoping rules for global variables. It may happen that the generation of your example fails due to an `UndefVarError` or other scoping issues. In these cases we recommend using `let ... end` blocks to enforce local scoping (see `Gaussian Mixtures Multivariate.ipynb` as an example)
-
+8. Notebooks and plain Julia have different scoping rules for global variables. It may happen that the generation of your example fails due to an `UndefVarError` or other scoping issues. In these cases we recommend using `let ... end` blocks to enforce local scoping or use the `global` keyword to disambiguate the scoping rules, e.g.
+```julia
+variable = 0
+for i in 1:10
+    global variable = variable + i
+end
+```
 9. All examples must use and activate the local environment specified by `Project.toml` in the second cell (see `1.`). Please have a look at the existing notebooks for an example on how to activate this local environment. If you need additional packages, you can add then to the `(examples)` project.
-10. All plots should be displayed automatically. In special cases, if needed, save figures in the `./pics/figure-name.ext` format. Might be useful for saving gifs.
+10. All plots should be displayed automatically. In special cases, if needed, save figures in the `../pics/figure-name.ext` format. Might be useful for saving gifs. Use `![](../pics/figure-name.ext)` to display a static image.
 
 !!! note
     Please avoid adding `PyPlot` in the `(examples)` project. Installing and building `PyPlot` dependencies takes several minutes on every CI run. Use `Plots` instead.