Merge branch 'main' of https://github.com/robin-janssen/CODES-Benchmark

README.md

@@ -4,7 +4,7 @@
 
 This repo aims to provide a way of benchmarking surrogate models for coupled ODE systems, as can be found in the context of chemical reaction networks. 
 
-Full documentation can be found on the new [website](https://immi000.github.io/CODES-Benchmark-Docs/).
+Full documentation can be found on the new [website](https://robin-janssen.github.io/CODES-Benchmark/).
 
 ## Motivation
 
@@ -140,6 +140,6 @@ Then, install the required packages with
 pip install -r requirements.txt
 ```
 
-The installation is now complete. To be able to run and evaluate the benchmark, you need to first set up a configuration YAML file. There is one provided, but it should be configured. For more information, check the [configuration page](https://immi000.github.io/CODES-Benchmark-Docs/documentation.html#config). There, we also offer an interactive Config-Generator tool with some explanations to help you set up your benchmark.
+The installation is now complete. To be able to run and evaluate the benchmark, you need to first set up a configuration YAML file. There is one provided, but it should be configured. For more information, check the [configuration page](https://robin-janssen.github.io/CODES-Benchmark/documentation.html#config). There, we also offer an interactive Config-Generator tool with some explanations to help you set up your benchmark.
 
-You can also add your own datasets and models to the benchmark to evaluate them against each other or some of our baseline models. For more information on how to do this, please refer to the [documentation](https://immi000.github.io/CODES-Benchmark-Docs/documentation.html).
+You can also add your own datasets and models to the benchmark to evaluate them against each other or some of our baseline models. For more information on how to do this, please refer to the [documentation](https://robin-janssen.github.io/CODES-Benchmark/documentation.html).

codes/benchmark/bench_fcts.py

@@ -1194,7 +1194,7 @@ def tabular_comparison(all_metrics: dict, config: dict) -> None:
         for i, value in enumerate(mae_values)
     ]
     mre_row = ["MRE"] + [
-        f"{value*100:.2f} %" if i != best_mre_index else f"* {value*100:.2f} % *"
+        f"{value * 100:.2f} %" if i != best_mre_index else f"* {value * 100:.2f} % *"
         for i, value in enumerate(mre_values)
     ]
     epochs_row = ["Epochs"] + [value for i, value in enumerate(epochs)]

codes/surrogates/surrogates.py

@@ -19,7 +19,7 @@ class AbstractSurrogateModel(ABC, nn.Module):
     structure of a surrogate model and defines the methods that need to be
     implemented by the subclasses for it to be compatible with the benchmarking
     framework. For more information, see
-    https://immi000.github.io/CODES-Benchmark-Docs/documentation.html#add_model.
+    https://robin-janssen.github.io/CODES-Benchmark/documentation.html#add_model.
 
     Args:
         device (str, optional): The device to run the model on. Defaults to None.
@@ -105,15 +105,15 @@ def __init__(
 
     @abstractmethod
     def forward(self, inputs: Any) -> tuple[Tensor, Tensor]:
-        """
-        Forward pass of the model.
+        # """
+        # Forward pass of the model.
 
-        Args:
-            inputs (Any): The input data as recieved from the dataloader.
+        # Args:
+        #     inputs (Any): The input data as recieved from the dataloader.
 
-        Returns:
-            tuple[Tensor, Tensor]: The model predictions and the targets.
-        """
+        # Returns:
+        #     tuple[Tensor, Tensor]: The model predictions and the targets.
+        # """
         pass
 
     @abstractmethod

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = source
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/_build/html/.buildinfo

@@ -0,0 +1,4 @@
+# Sphinx build info version 1
+# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
+config: 7303878f072328508a731aec95c1900e
+tags: 645f666f9bcd5a90fca523b33c5a78b7

docs/_build/html/_sources/codes.benchmark.rst

@@ -0,0 +1,37 @@
+codes.benchmark package
+=======================
+
+Submodules
+----------
+
+codes.benchmark.bench\_fcts module
+----------------------------------
+
+.. automodule:: codes.benchmark.bench_fcts
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+codes.benchmark.bench\_plots module
+-----------------------------------
+
+.. automodule:: codes.benchmark.bench_plots
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+codes.benchmark.bench\_utils module
+-----------------------------------
+
+.. automodule:: codes.benchmark.bench_utils
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Module contents
+---------------
+
+.. automodule:: codes.benchmark
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/_build/html/_sources/codes.rst

@@ -0,0 +1,21 @@
+codes package
+=============
+
+Subpackages
+-----------
+
+.. toctree::
+   :maxdepth: 4
+
+   codes.benchmark
+   codes.surrogates
+   codes.train
+   codes.utils
+
+Module contents
+---------------
+
+.. automodule:: codes
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/_build/html/_sources/codes.surrogates.rst

@@ -0,0 +1,29 @@
+codes.surrogates package
+========================
+
+Submodules
+----------
+
+codes.surrogates.surrogate\_classes module
+------------------------------------------
+
+.. automodule:: codes.surrogates.surrogate_classes
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+codes.surrogates.surrogates module
+----------------------------------
+
+.. automodule:: codes.surrogates.surrogates
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Module contents
+---------------
+
+.. automodule:: codes.surrogates
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/_build/html/_sources/codes.train.rst

@@ -0,0 +1,21 @@
+codes.train package
+===================
+
+Submodules
+----------
+
+codes.train.train\_fcts module
+------------------------------
+
+.. automodule:: codes.train.train_fcts
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Module contents
+---------------
+
+.. automodule:: codes.train
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/_build/html/_sources/codes.utils.rst

@@ -0,0 +1,29 @@
+codes.utils package
+===================
+
+Submodules
+----------
+
+codes.utils.data\_utils module
+------------------------------
+
+.. automodule:: codes.utils.data_utils
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+codes.utils.utils module
+------------------------
+
+.. automodule:: codes.utils.utils
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Module contents
+---------------
+
+.. automodule:: codes.utils
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/_build/html/_sources/docs/source/codes.benchmark.bench_fcts.rst

@@ -0,0 +1,33 @@
+codes.benchmark.bench\_fcts
+===========================
+
+.. automodule:: codes.benchmark.bench_fcts
+
+
+   .. rubric:: Functions
+
+   .. autosummary::
+
+      compare_MAE
+      compare_UQ
+      compare_batchsize
+      compare_dynamic_accuracy
+      compare_extrapolation
+      compare_inference_time
+      compare_interpolation
+      compare_main_losses
+      compare_models
+      compare_relative_errors
+      compare_sparse
+      evaluate_UQ
+      evaluate_accuracy
+      evaluate_batchsize
+      evaluate_compute
+      evaluate_dynamic_accuracy
+      evaluate_extrapolation
+      evaluate_interpolation
+      evaluate_sparse
+      run_benchmark
+      tabular_comparison
+      time_inference
+

docs/_build/html/_sources/docs/source/codes.benchmark.bench_plots.rst

@@ -0,0 +1,38 @@
+codes.benchmark.bench\_plots
+============================
+
+.. automodule:: codes.benchmark.bench_plots
+
+
+   .. rubric:: Functions
+
+   .. autosummary::
+
+      get_custom_palette
+      inference_time_bar_plot
+      int_ext_sparse
+      plot_MAE_comparison
+      plot_MAE_comparison_train_duration
+      plot_average_errors_over_time
+      plot_average_uncertainty_over_time
+      plot_comparative_dynamic_correlation_heatmaps
+      plot_comparative_error_correlation_heatmaps
+      plot_dynamic_correlation
+      plot_dynamic_correlation_heatmap
+      plot_error_correlation_heatmap
+      plot_error_distribution_comparative
+      plot_error_distribution_per_chemical
+      plot_example_predictions_with_uncertainty
+      plot_generalization_error_comparison
+      plot_generalization_errors
+      plot_loss_comparison
+      plot_losses
+      plot_relative_errors
+      plot_relative_errors_over_time
+      plot_surr_losses
+      plot_uncertainty_over_time_comparison
+      plot_uncertainty_vs_errors
+      rel_errors_and_uq
+      save_plot
+      save_plot_counter
+

docs/_build/html/_sources/docs/source/codes.benchmark.bench_utils.rst

@@ -0,0 +1,29 @@
+codes.benchmark.bench\_utils
+============================
+
+.. automodule:: codes.benchmark.bench_utils
+
+
+   .. rubric:: Functions
+
+   .. autosummary::
+
+      check_benchmark
+      check_surrogate
+      clean_metrics
+      convert_dict_to_scientific_notation
+      convert_to_standard_types
+      count_trainable_parameters
+      discard_numpy_entries
+      flatten_dict
+      format_seconds
+      format_time
+      get_model_config
+      get_required_models_list
+      get_surrogate
+      load_model
+      make_comparison_csv
+      measure_memory_footprint
+      read_yaml_config
+      write_metrics_to_yaml
+

docs/_build/html/_sources/docs/source/codes.benchmark.rst

@@ -0,0 +1,15 @@
+codes.benchmark
+===============
+
+.. automodule:: codes.benchmark
+
+
+.. rubric:: Modules
+
+.. autosummary::
+   :toctree:
+   :recursive:
+
+   bench_fcts
+   bench_plots
+   bench_utils

docs/_build/html/_sources/docs/source/codes.rst

@@ -0,0 +1,16 @@
+codes
+=====
+
+.. automodule:: codes
+
+
+.. rubric:: Modules
+
+.. autosummary::
+   :toctree:
+   :recursive:
+
+   benchmark
+   surrogates
+   train
+   utils

docs/_build/html/_sources/docs/source/codes.surrogates.rst

@@ -0,0 +1,14 @@
+codes.surrogates
+================
+
+.. automodule:: codes.surrogates
+
+
+.. rubric:: Modules
+
+.. autosummary::
+   :toctree:
+   :recursive:
+
+   surrogate_classes
+   surrogates

docs/_build/html/_sources/docs/source/codes.surrogates.surrogate_classes.rst

@@ -0,0 +1,6 @@
+codes.surrogates.surrogate\_classes
+===================================
+
+.. currentmodule:: codes.surrogates
+
+.. autodata:: surrogate_classes