From 09ad9b074b44404705f18cf0bb24ffa3660c358d Mon Sep 17 00:00:00 2001 From: Brady Planden Date: Sat, 27 Apr 2024 17:39:16 +0100 Subject: [PATCH 01/11] Adds initial AdamW implementation --- pybop/__init__.py | 1 + pybop/_optimisation.py | 4 +- pybop/optimisers/AdamW.py | 261 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 265 insertions(+), 1 deletion(-) create mode 100644 pybop/optimisers/AdamW.py diff --git a/pybop/__init__.py b/pybop/__init__.py index c5d1abf0..7afd428c 100644 --- a/pybop/__init__.py +++ b/pybop/__init__.py @@ -95,6 +95,7 @@ SNES, XNES, ) +from .optimisers.AdamW import AdamW # # Parameter classes diff --git a/pybop/_optimisation.py b/pybop/_optimisation.py index 2a0b8ed9..68835158 100644 --- a/pybop/_optimisation.py +++ b/pybop/_optimisation.py @@ -101,7 +101,9 @@ def __init__( raise ValueError("Unknown optimiser type") if self.pints: - self.optimiser = self.optimiser(self.x0, self.sigma0, self.bounds) + self.optimiser = self.optimiser( + self.x0, sigma0=self.sigma0, boundaries=self.bounds + ) # Check if sensitivities are required self._needs_sensitivities = self.optimiser.needs_sensitivities() diff --git a/pybop/optimisers/AdamW.py b/pybop/optimisers/AdamW.py new file mode 100644 index 00000000..54d8aa5f --- /dev/null +++ b/pybop/optimisers/AdamW.py @@ -0,0 +1,261 @@ +# +# Reimplements the Pints' Adam Class with a Weight Decay addition +# + +import numpy as np +from pints import Optimiser as PintsOptimiser + + +class AdamW(PintsOptimiser): + """ + AdamW optimiser (adaptive moment estimation with weight decay), as described in [1]_. + + This method is an extension of the Adam optimizer that introduces weight decay, + which helps to regularize the weights and prevent overfitting. + + This class reimplements the Pints' Adam Optimiser, but with the weight decay + functionality mentioned above. Original creation and credit is attributed to Pints. + + Pseudo-code is given below. Here the value of the j-th parameter at + iteration i is given as ``p_j[i]`` and the corresponding derivative is + denoted ``g_j[i]``:: + + m_j[i] = beta1 * m_j[i - 1] + (1 - beta1) * g_j[i] + v_j[i] = beta2 * v_j[i - 1] + (1 - beta2) * g_j[i]**2 + + m_j' = m_j[i] / (1 - beta1**(1 + i)) + v_j' = v_j[i] / (1 - beta2**(1 + i)) + + p_j[i] = p_j[i - 1] - alpha * (m_j' / (sqrt(v_j') + eps) + lambda * p_j[i - 1]) + + The initial values of the moments are ``m_j[0] = v_j[0] = 0``, after which + they decay with rates ``beta1`` and ``beta2``. The default values for these are, + ``beta1 = 0.9`` and ``beta2 = 0.999``. + + The terms ``m_j'`` and ``v_j'`` are "initialisation bias corrected" + versions of ``m_j`` and ``v_j`` (see section 3 of the paper). + + The parameter ``alpha`` is a step size, which is set as ``min(sigma0)`` in + this implementation. + + The parameter ``lambda`` is the weight decay rate, which is set to ``0.01`` + by default in this implementation. + + Finally, ``eps`` is a small constant used to avoid division by zero, set to + ``eps = `np.finfo(float).eps` in this implementation. + + This is an unbounded method: Any ``boundaries`` will be ignored. + + References + ---------- + .. [1] Decoupled Weight Decay Regularization + Loshchilov and Hutter, 2019, arxiv (version v3) + https://doi.org/10.48550/arXiv.1711.05101 + """ + + def __init__(self, x0, sigma0=0.015, boundaries=None): + super().__init__(x0, sigma0, boundaries) + + # Set optimiser state + self._running = False + self._ready_for_tell = False + + # Best solution found + self._x_best = self._x0 + self._f_best = np.inf + + # Current point, score, and gradient + self._current = self._x0 + self._current_f = np.inf + self._current_df = None + + # Proposed next point (read-only, so can be passed to user) + self._proposed = self._x0 + self._proposed.setflags(write=False) + + # Moment vectors + self._m = np.zeros(self._x0.shape) + self._v = np.zeros(self._x0.shape) + + # Exponential decay rates for the moment estimates + self._b1 = 0.9 + self._b2 = 0.999 + + # Step size + self._alpha = np.min(self._sigma0) + + # Weight decay rate + self.set_lambda() + + # Small number added to avoid divide-by-zero + self._eps = np.finfo(float).eps + + # Powers of decay rates + self._b1t = 1 + self._b2t = 1 + + def ask(self): + """ + Returns a list of next points in the parameter-space + to evaluate from the optimiser. + """ + + # Running, and ready for tell now + self._ready_for_tell = True + self._running = True + + # Return proposed points (just the one) + return [self._proposed] + + def f_best(self): + """ + Returns the best score found so far. + """ + return self._f_best + + def f_guessed(self): + """ + Returns the score of the last guessed point. + """ + return self._current_f + + def _log_init(self, logger): + """See :meth:`Loggable._log_init()`.""" + logger.add_float("b1") + logger.add_float("b2") + logger.add_float("lambda") + + def _log_write(self, logger): + """See :meth:`Loggable._log_write()`.""" + logger.log(self._b1t) + logger.log(self._b2t) + logger.log(self._lambda) + + def name(self): + """ + Returns the name of the optimiser. + """ + return "AdamW" + + def needs_sensitivities(self): + """ + Returns ``False`` if this optimiser does not require gradient, + and ``True`` otherwise. + """ + return True + + def n_hyper_parameters(self): + """ + The number of hyper-parameters used by this optimiser. + """ + return 1 + + def running(self): + """ + Returns ``True`` if the optimisation is in progress. + """ + return self._running + + def tell(self, reply): + """ + Receives a list of function values from the cost function from points + previously specified by `self.ask()`, and updates the optimiser state + accordingly. + """ + + # Check ask-tell pattern + if not self._ready_for_tell: + raise Exception("ask() not called before tell()") + self._ready_for_tell = False + + # Unpack reply + fx, dfx = reply[0] + + # Update current point + self._current = self._proposed + self._current_f = fx + self._current_df = dfx + + # Update bx^t + self._b1t *= self._b1 + self._b2t *= self._b2 + + # "Update biased first moment estimate" + self._m = self._b1 * self._m + (1 - self._b1) * dfx + + # "Update biased second raw moment estimate" + self._v = self._b2 * self._v + (1 - self._b2) * dfx**2 + + # "Compute bias-corrected first moment estimate" + m = self._m / (1 - self._b1t) + + # "Compute bias-corrected second raw moment estimate" + v = self._v / (1 - self._b2t) + + # Take step with weight decay + self._proposed = self._current - self._alpha * ( + m / (np.sqrt(v) + self._eps) + self._lambda * self._current + ) + + # Update x_best and f_best + if self._f_best > fx: + self._f_best = fx + self._x_best = self._current + + def x_best(self): + """ + Returns the best parameter values found so far. + """ + return self._x_best + + def x_guessed(self): + """ + Returns the last guessed parameter values. + """ + return self._current + + def set_lambda(self, lambda_=0.01): + """ + Sets the lambda_ decay constant. This is the weight decay rate + that helps in finding the optimal solution. + """ + try: + lambda_ = float(lambda_) # Ensure b1 is a floatable + except Exception: + raise ("lambda_ must be numeric, floatable value.") + + if not 0 < lambda_ <= 1: + print("lambda_ must a positive value between 0 and 1") + + self._lambda = lambda_ + return + + def set_b1(self, b1): + """ + Sets the b1 momentum decay constant. + """ + try: + b1 = float(b1) # Ensure b1 is a floatable + except Exception: + raise ("b1 must be numeric, floatable value.") + + if not 0 < b1 <= 1: + print("b1 must a positive value between 0 and 1") + + self._b1 = b1 + return + + def set_b2(self, b2): + """ + Sets the b2 momentum decay constant. + """ + try: + b2 = float(b2) # Ensure b2 is a floatable + except Exception: + raise ("b2 must be numeric, floatable value.") + + if not 0 < b2 <= 1: + print("b2 must a positive value between 0 and 1") + + self._b2 = b2 + return From a454c12b50b3bf9f24dfba299baa4315f5247350 Mon Sep 17 00:00:00 2001 From: Brady Planden Date: Wed, 8 May 2024 09:16:36 +0100 Subject: [PATCH 02/11] Fix Pints' optimiser bound arg, add AdamW tests, update Adam notebook to AdamW --- examples/notebooks/spm_Adam.ipynb | 855 -------------------- examples/notebooks/spm_AdamW.ipynb | 826 +++++++++++++++++++ pybop/_optimisation.py | 2 +- pybop/optimisers/AdamW.py | 8 +- tests/integration/test_parameterisations.py | 1 + tests/unit/test_optimisation.py | 1 + 6 files changed, 835 insertions(+), 858 deletions(-) delete mode 100644 examples/notebooks/spm_Adam.ipynb create mode 100644 examples/notebooks/spm_AdamW.ipynb diff --git a/examples/notebooks/spm_Adam.ipynb b/examples/notebooks/spm_Adam.ipynb deleted file mode 100644 index 7083c002..00000000 --- a/examples/notebooks/spm_Adam.ipynb +++ /dev/null @@ -1,855 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "expmkveO04pw" - }, - "source": [ - "## Parameter Estimation with Adam in PyBOP\n", - "\n", - "In this notebook, we demonstrate an example of parameter estimation for a single-particle model using the Adam optimiser [1]. The ADAM optimiser is an algorithm for gradient-based optimisation, combining the advantages of the Adaptive Gradient Algorithm (AdaGrad) and Root Mean Square Propagation (RMSProp).\n", - "\n", - "[[1]: Adam: A Method for Stochastic Optimization](https://arxiv.org/abs/1412.6980)\n", - "\n", - "### Setting up the Environment\n", - "\n", - "Before we begin, we need to ensure that we have all the necessary tools. We will install PyBOP from its development branch and upgrade some dependencies:" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "execution": { - "iopub.execute_input": "2024-04-04T13:51:40.337833Z", - "iopub.status.busy": "2024-04-04T13:51:40.337689Z", - "iopub.status.idle": "2024-04-04T13:51:41.935008Z", - "shell.execute_reply": "2024-04-04T13:51:41.934618Z" - }, - "id": "X87NUGPW04py", - "outputId": "0d785b07-7cff-4aeb-e60a-4ff5a669afbf" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: pip in /home/engs2510/.pyenv/versions/pybop/lib/python3.11/site-packages (24.0)\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: ipywidgets in /home/engs2510/.pyenv/versions/pybop/lib/python3.11/site-packages (8.1.2)\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: comm>=0.1.3 in /home/engs2510/.pyenv/versions/pybop/lib/python3.11/site-packages (from ipywidgets) (0.2.2)\r\n", - "Requirement already satisfied: ipython>=6.1.0 in /home/engs2510/.pyenv/versions/pybop/lib/python3.11/site-packages (from ipywidgets) (8.23.0)\r\n", - "Requirement already satisfied: traitlets>=4.3.1 in /home/engs2510/.pyenv/versions/pybop/lib/python3.11/site-packages (from ipywidgets) (5.14.2)\r\n", - "Requirement already satisfied: widgetsnbextension~=4.0.10 in /home/engs2510/.pyenv/versions/pybop/lib/python3.11/site-packages (from ipywidgets) (4.0.10)\r\n", - "Requirement already satisfied: jupyterlab-widgets~=3.0.10 in /home/engs2510/.pyenv/versions/pybop/lib/python3.11/site-packages (from ipywidgets) (3.0.10)\r\n", - "Requirement already satisfied: decorator in /home/engs2510/.pyenv/versions/pybop/lib/python3.11/site-packages (from ipython>=6.1.0->ipywidgets) (5.1.1)\r\n", - "Requirement already satisfied: jedi>=0.16 in /home/engs2510/.pyenv/versions/pybop/lib/python3.11/site-packages (from ipython>=6.1.0->ipywidgets) (0.19.1)\r\n", - "Requirement already satisfied: matplotlib-inline in /home/engs2510/.pyenv/versions/pybop/lib/python3.11/site-packages (from ipython>=6.1.0->ipywidgets) (0.1.6)\r\n", - "Requirement already satisfied: prompt-toolkit<3.1.0,>=3.0.41 in /home/engs2510/.pyenv/versions/pybop/lib/python3.11/site-packages (from ipython>=6.1.0->ipywidgets) (3.0.43)\r\n", - "Requirement already satisfied: pygments>=2.4.0 in /home/engs2510/.pyenv/versions/pybop/lib/python3.11/site-packages (from ipython>=6.1.0->ipywidgets) (2.17.2)\r\n", - "Requirement already satisfied: stack-data in /home/engs2510/.pyenv/versions/pybop/lib/python3.11/site-packages (from ipython>=6.1.0->ipywidgets) (0.6.3)\r\n", - "Requirement already satisfied: typing-extensions in /home/engs2510/.pyenv/versions/pybop/lib/python3.11/site-packages (from ipython>=6.1.0->ipywidgets) (4.10.0)\r\n", - "Requirement already satisfied: pexpect>4.3 in /home/engs2510/.pyenv/versions/pybop/lib/python3.11/site-packages (from ipython>=6.1.0->ipywidgets) (4.9.0)\r\n", - "Requirement already satisfied: parso<0.9.0,>=0.8.3 in /home/engs2510/.pyenv/versions/pybop/lib/python3.11/site-packages (from jedi>=0.16->ipython>=6.1.0->ipywidgets) (0.8.3)\r\n", - "Requirement already satisfied: ptyprocess>=0.5 in /home/engs2510/.pyenv/versions/pybop/lib/python3.11/site-packages (from pexpect>4.3->ipython>=6.1.0->ipywidgets) (0.7.0)\r\n", - "Requirement already satisfied: wcwidth in /home/engs2510/.pyenv/versions/pybop/lib/python3.11/site-packages (from prompt-toolkit<3.1.0,>=3.0.41->ipython>=6.1.0->ipywidgets) (0.2.13)\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: executing>=1.2.0 in /home/engs2510/.pyenv/versions/pybop/lib/python3.11/site-packages (from stack-data->ipython>=6.1.0->ipywidgets) (2.0.1)\r\n", - "Requirement already satisfied: asttokens>=2.1.0 in /home/engs2510/.pyenv/versions/pybop/lib/python3.11/site-packages (from stack-data->ipython>=6.1.0->ipywidgets) (2.4.1)\r\n", - "Requirement already satisfied: pure-eval in /home/engs2510/.pyenv/versions/pybop/lib/python3.11/site-packages (from stack-data->ipython>=6.1.0->ipywidgets) (0.2.2)\r\n", - "Requirement already satisfied: six>=1.12.0 in /home/engs2510/.pyenv/versions/pybop/lib/python3.11/site-packages (from asttokens>=2.1.0->stack-data->ipython>=6.1.0->ipywidgets) (1.16.0)\r\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Note: you may need to restart the kernel to use updated packages.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Note: you may need to restart the kernel to use updated packages.\n" - ] - } - ], - "source": [ - "%pip install --upgrade pip ipywidgets\n", - "%pip install pybop -q" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "jAvD5fk104p0" - }, - "source": [ - "### Importing Libraries\n", - "\n", - "With the environment set up, we can now import PyBOP alongside other libraries we will need:" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "execution": { - "iopub.execute_input": "2024-04-04T13:51:41.936561Z", - "iopub.status.busy": "2024-04-04T13:51:41.936439Z", - "iopub.status.idle": "2024-04-04T13:51:42.508083Z", - "shell.execute_reply": "2024-04-04T13:51:42.507654Z" - }, - "id": "SQdt4brD04p1" - }, - "outputs": [], - "source": [ - "import numpy as np\n", - "\n", - "import pybop" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "5XU-dMtU04p2" - }, - "source": [ - "### Generate Synthetic Data\n", - "\n", - "To demonstrate parameter estimation, we first need some data. We will generate synthetic data using the PyBOP forward model, which requires defining a parameter set and the model itself.\n", - "\n", - "#### Defining Parameters and Model\n", - "\n", - "We start by creating an example parameter set and then instantiate the single-particle model (SPM):" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "execution": { - "iopub.execute_input": "2024-04-04T13:51:42.509591Z", - "iopub.status.busy": "2024-04-04T13:51:42.509437Z", - "iopub.status.idle": "2024-04-04T13:51:42.534794Z", - "shell.execute_reply": "2024-04-04T13:51:42.534452Z" - } - }, - "outputs": [], - "source": [ - "parameter_set = pybop.ParameterSet.pybamm(\"Chen2020\")\n", - "model = pybop.lithium_ion.SPM(parameter_set=parameter_set)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Simulating Forward Model\n", - "\n", - "We can then simulate the model using the `predict` method, with a default constant current to generate voltage data." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "execution": { - "iopub.execute_input": "2024-04-04T13:51:42.536154Z", - "iopub.status.busy": "2024-04-04T13:51:42.536069Z", - "iopub.status.idle": "2024-04-04T13:51:42.610305Z", - "shell.execute_reply": "2024-04-04T13:51:42.609892Z" - }, - "id": "sBasxv8U04p3" - }, - "outputs": [], - "source": [ - "t_eval = np.arange(0, 900, 2)\n", - "values = model.predict(t_eval=t_eval)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Adding Noise to Voltage Data\n", - "\n", - "To make the parameter estimation more realistic, we add Gaussian noise to the data." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "execution": { - "iopub.execute_input": "2024-04-04T13:51:42.611946Z", - "iopub.status.busy": "2024-04-04T13:51:42.611728Z", - "iopub.status.idle": "2024-04-04T13:51:42.621525Z", - "shell.execute_reply": "2024-04-04T13:51:42.621156Z" - } - }, - "outputs": [], - "source": [ - "sigma = 0.001\n", - "corrupt_values = values[\"Voltage [V]\"].data + np.random.normal(0, sigma, len(t_eval))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "X8-tubYY04p_" - }, - "source": [ - "## Identify the Parameters" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "PQqhvSZN04p_" - }, - "source": [ - "We will now set up the parameter estimation process by defining the datasets for optimisation and selecting the model parameters we wish to estimate." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Creating Optimisation Dataset\n", - "\n", - "The dataset for optimisation is composed of time, current, and the noisy voltage data:" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "execution": { - "iopub.execute_input": "2024-04-04T13:51:42.622671Z", - "iopub.status.busy": "2024-04-04T13:51:42.622478Z", - "iopub.status.idle": "2024-04-04T13:51:42.628864Z", - "shell.execute_reply": "2024-04-04T13:51:42.628519Z" - }, - "id": "zuvGHWID04p_" - }, - "outputs": [], - "source": [ - "dataset = pybop.Dataset(\n", - " {\n", - " \"Time [s]\": t_eval,\n", - " \"Current function [A]\": values[\"Current [A]\"].data,\n", - " \"Voltage [V]\": corrupt_values,\n", - " }\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ffS3CF_704qA" - }, - "source": [ - "### Defining Parameters to Estimate\n", - "\n", - "We select the parameters for estimation and set up their prior distributions and bounds:" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "execution": { - "iopub.execute_input": "2024-04-04T13:51:42.629987Z", - "iopub.status.busy": "2024-04-04T13:51:42.629809Z", - "iopub.status.idle": "2024-04-04T13:51:42.631895Z", - "shell.execute_reply": "2024-04-04T13:51:42.631621Z" - }, - "id": "WPCybXIJ04qA" - }, - "outputs": [], - "source": [ - "parameters = [\n", - " pybop.Parameter(\n", - " \"Negative electrode active material volume fraction\",\n", - " prior=pybop.Gaussian(0.6, 0.02),\n", - " bounds=[0.5, 0.8],\n", - " ),\n", - " pybop.Parameter(\n", - " \"Positive electrode active material volume fraction\",\n", - " prior=pybop.Gaussian(0.48, 0.02),\n", - " bounds=[0.4, 0.7],\n", - " ),\n", - "]" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "n4OHa-aF04qA" - }, - "source": [ - "### Setting up the Optimisation Problem\n", - "\n", - "With the datasets and parameters defined, we can set up the optimisation problem, its cost function, and the optimiser." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "execution": { - "iopub.execute_input": "2024-04-04T13:51:42.632931Z", - "iopub.status.busy": "2024-04-04T13:51:42.632782Z", - "iopub.status.idle": "2024-04-04T13:51:42.705454Z", - "shell.execute_reply": "2024-04-04T13:51:42.705066Z" - }, - "id": "etMzRtx404qA" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "NOTE: Boundaries ignored by Adam\n" - ] - } - ], - "source": [ - "problem = pybop.FittingProblem(model, parameters, dataset)\n", - "cost = pybop.SumSquaredError(problem)\n", - "optim = pybop.Optimisation(cost, optimiser=pybop.Adam)\n", - "optim.set_max_unchanged_iterations(40)\n", - "optim.set_max_iterations(150)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "caprp-bV04qB" - }, - "source": [ - "### Running the Optimisation\n", - "\n", - "We proceed to run the Adam optimisation algorithm to estimate the parameters:" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "execution": { - "iopub.execute_input": "2024-04-04T13:51:42.706564Z", - "iopub.status.busy": "2024-04-04T13:51:42.706469Z", - "iopub.status.idle": "2024-04-04T13:51:50.537424Z", - "shell.execute_reply": "2024-04-04T13:51:50.537032Z" - }, - "id": "-9OVt0EQ04qB" - }, - "outputs": [], - "source": [ - "x, final_cost = optim.run()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "-4pZsDmS04qC" - }, - "source": [ - "### Viewing the Estimated Parameters\n", - "\n", - "After the optimisation, we can examine the estimated parameter values:" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "execution": { - "iopub.execute_input": "2024-04-04T13:51:50.538815Z", - "iopub.status.busy": "2024-04-04T13:51:50.538619Z", - "iopub.status.idle": "2024-04-04T13:51:50.541683Z", - "shell.execute_reply": "2024-04-04T13:51:50.541465Z" - }, - "id": "Hgz8SV4i04qC", - "outputId": "e1e42ae7-5075-4c47-dd68-1b22ecc170f6" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "array([0.76496615, 0.66254367])" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "x # This will output the estimated parameters" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "KxKURtH704qC" - }, - "source": [ - "## Plotting and Visualisation\n", - "\n", - "PyBOP provides various plotting utilities to visualise the results of the optimisation." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "-cWCOiqR04qC" - }, - "source": [ - "### Comparing System Response\n", - "\n", - "We can quickly plot the system's response using the estimated parameters compared to the target:" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 467 - }, - "execution": { - "iopub.execute_input": "2024-04-04T13:51:50.542618Z", - "iopub.status.busy": "2024-04-04T13:51:50.542472Z", - "iopub.status.idle": "2024-04-04T13:51:50.986055Z", - "shell.execute_reply": "2024-04-04T13:51:50.985844Z" - }, - "id": "tJUJ80Ve04qD", - "outputId": "855fbaa2-1e09-4935-eb1a-8caf7f99eb75" - }, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "02004006008003.83.853.93.9544.05ReferenceModelOptimised ComparisonTime / sVoltage / V" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "pybop.quick_plot(problem, parameter_values=x, title=\"Optimised Comparison\");" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Convergence and Parameter Trajectories\n", - "\n", - "To assess the optimisation process, we can plot the convergence of the cost function and the trajectories of the parameters:" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": { - "execution": { - "iopub.execute_input": "2024-04-04T13:51:50.987237Z", - "iopub.status.busy": "2024-04-04T13:51:50.986963Z", - "iopub.status.idle": "2024-04-04T13:51:52.766386Z", - "shell.execute_reply": "2024-04-04T13:51:52.766178Z" - }, - "id": "N5XYkevi04qD" - }, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "2040608010012014000.511.522.533.54ConvergenceIterationCost" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "image/svg+xml": [ - "0501000.60.650.70.750.80.850501000.50.550.60.650.7Negative electrode active material volume fractionPositive electrode active material volume fractionParameter ConvergenceFunction CallFunction CallNegative electrode active material volume fractionPositive electrode active material volume fraction" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "pybop.plot_convergence(optim)\n", - "pybop.plot_parameters(optim);" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Cost Landscape\n", - "\n", - "Finally, we can visualise the cost landscape and the path taken by the optimiser:" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": { - "execution": { - "iopub.execute_input": "2024-04-04T13:51:52.767346Z", - "iopub.status.busy": "2024-04-04T13:51:52.767261Z", - "iopub.status.idle": "2024-04-04T13:51:57.666000Z", - "shell.execute_reply": "2024-04-04T13:51:57.665745Z" - } - }, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "0.50.550.60.650.70.750.80.40.450.50.550.60.650.70246810Cost LandscapeNegative electrode active material volume fractionPositive electrode active material volume fraction" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "image/svg+xml": [ - "0.60.650.70.750.80.850.90.50.550.60.650.70.750.80.40.81.21.622.4Cost LandscapeNegative electrode active material volume fractionPositive electrode active material volume fraction" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "# Plot the cost landscape\n", - "pybop.plot2d(cost, steps=15)\n", - "# Plot the cost landscape with optimisation path and updated bounds\n", - "bounds = np.array([[0.6, 0.9], [0.5, 0.8]])\n", - "pybop.plot2d(optim, bounds=bounds, steps=15);" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Conclusion\n", - "\n", - "This notebook illustrates how to perform parameter estimation using Adam in PyBOP, providing insights into the optimisation process through various visualisations." - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.8" - }, - "widgets": { - "application/vnd.jupyter.widget-state+json": { - "06f2374f91c8455bb63252092512f2ed": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "2.0.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "2.0.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "2.0.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border_bottom": null, - "border_left": null, - "border_right": null, - "border_top": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "423bffea3a1c42b49a9ad71218e5811b": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "2.0.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "2.0.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "2.0.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border_bottom": null, - "border_left": null, - "border_right": null, - "border_top": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "56ff19291e464d63b23e63b8e2ac9ea3": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "2.0.0", - "model_name": "SliderStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "2.0.0", - "_model_name": "SliderStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "2.0.0", - "_view_name": "StyleView", - "description_width": "", - "handle_color": null - } - }, - "646a8670cb204a31bb56bc2380898093": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "2.0.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "2.0.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "2.0.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border_bottom": null, - "border_left": null, - "border_right": null, - "border_top": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "7d46516469314b88be3500e2afcafcf6": { - "model_module": "@jupyter-widgets/output", - "model_module_version": "1.0.0", - "model_name": "OutputModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/output", - "_model_module_version": "1.0.0", - "_model_name": "OutputModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/output", - "_view_module_version": "1.0.0", - "_view_name": "OutputView", - "layout": "IPY_MODEL_646a8670cb204a31bb56bc2380898093", - "msg_id": "", - "outputs": [], - "tabbable": null, - "tooltip": null - } - }, - "8d003c14da5f4fa68284b28c15cee6e6": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "2.0.0", - "model_name": "VBoxModel", - "state": { - "_dom_classes": [ - "widget-interact" - ], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "2.0.0", - "_model_name": "VBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "2.0.0", - "_view_name": "VBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_aef2fa7adcc14ad0854b73d5910ae3b4", - "IPY_MODEL_7d46516469314b88be3500e2afcafcf6" - ], - "layout": "IPY_MODEL_423bffea3a1c42b49a9ad71218e5811b", - "tabbable": null, - "tooltip": null - } - }, - "aef2fa7adcc14ad0854b73d5910ae3b4": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "2.0.0", - "model_name": "FloatSliderModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "2.0.0", - "_model_name": "FloatSliderModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "2.0.0", - "_view_name": "FloatSliderView", - "behavior": "drag-tap", - "continuous_update": true, - "description": "t", - "description_allow_html": false, - "disabled": false, - "layout": "IPY_MODEL_06f2374f91c8455bb63252092512f2ed", - "max": 1.1333333333333333, - "min": 0, - "orientation": "horizontal", - "readout": true, - "readout_format": ".2f", - "step": 0.011333333333333332, - "style": "IPY_MODEL_56ff19291e464d63b23e63b8e2ac9ea3", - "tabbable": null, - "tooltip": null, - "value": 0 - } - } - } - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/examples/notebooks/spm_AdamW.ipynb b/examples/notebooks/spm_AdamW.ipynb new file mode 100644 index 00000000..20b73330 --- /dev/null +++ b/examples/notebooks/spm_AdamW.ipynb @@ -0,0 +1,826 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "expmkveO04pw" + }, + "source": [ + "## Parameter Estimation with AdamW in PyBOP\n", + "\n", + "In this notebook, we demonstrate an example of parameter estimation for a single-particle model using the AdamW optimiser [1][2]. The AdamW optimiser is an algorithm for gradient-based optimisation, combining the advantages of the Adaptive Gradient Algorithm (AdaGrad) and Root Mean Square Propagation (RMSProp).\n", + "\n", + "[[1]: Adam: A Method for Stochastic Optimization](https://arxiv.org/abs/1412.6980) \n", + "\n", + "[[2]: Decoupled Weight Decay Regularization](https://doi.org/10.48550/arXiv.1711.05101)\n", + "\n", + "### Setting up the Environment\n", + "\n", + "Before we begin, we need to ensure that we have all the necessary tools. We will install PyBOP from its development branch and upgrade some dependencies:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-04-04T13:51:40.337833Z", + "iopub.status.busy": "2024-04-04T13:51:40.337689Z", + "iopub.status.idle": "2024-04-04T13:51:41.935008Z", + "shell.execute_reply": "2024-04-04T13:51:41.934618Z" + }, + "id": "X87NUGPW04py", + "outputId": "0d785b07-7cff-4aeb-e60a-4ff5a669afbf" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: pip in /Users/engs2510/.pyenv/versions/3.11.7/envs/pybop/lib/python3.11/site-packages (24.0)\n", + "Requirement already satisfied: ipywidgets in /Users/engs2510/.pyenv/versions/3.11.7/envs/pybop/lib/python3.11/site-packages (8.1.2)\n", + "Requirement already satisfied: comm>=0.1.3 in /Users/engs2510/.pyenv/versions/3.11.7/envs/pybop/lib/python3.11/site-packages (from ipywidgets) (0.2.1)\n", + "Requirement already satisfied: ipython>=6.1.0 in /Users/engs2510/.pyenv/versions/3.11.7/envs/pybop/lib/python3.11/site-packages (from ipywidgets) (8.22.1)\n", + "Requirement already satisfied: traitlets>=4.3.1 in /Users/engs2510/.pyenv/versions/3.11.7/envs/pybop/lib/python3.11/site-packages (from ipywidgets) (5.14.1)\n", + "Requirement already satisfied: widgetsnbextension~=4.0.10 in /Users/engs2510/.pyenv/versions/3.11.7/envs/pybop/lib/python3.11/site-packages (from ipywidgets) (4.0.10)\n", + "Requirement already satisfied: jupyterlab-widgets~=3.0.10 in /Users/engs2510/.pyenv/versions/3.11.7/envs/pybop/lib/python3.11/site-packages (from ipywidgets) (3.0.10)\n", + "Requirement already satisfied: decorator in /Users/engs2510/.pyenv/versions/3.11.7/envs/pybop/lib/python3.11/site-packages (from ipython>=6.1.0->ipywidgets) (5.1.1)\n", + "Requirement already satisfied: jedi>=0.16 in /Users/engs2510/.pyenv/versions/3.11.7/envs/pybop/lib/python3.11/site-packages (from ipython>=6.1.0->ipywidgets) (0.19.1)\n", + "Requirement already satisfied: matplotlib-inline in /Users/engs2510/.pyenv/versions/3.11.7/envs/pybop/lib/python3.11/site-packages (from ipython>=6.1.0->ipywidgets) (0.1.6)\n", + "Requirement already satisfied: prompt-toolkit<3.1.0,>=3.0.41 in /Users/engs2510/.pyenv/versions/3.11.7/envs/pybop/lib/python3.11/site-packages (from ipython>=6.1.0->ipywidgets) (3.0.43)\n", + "Requirement already satisfied: pygments>=2.4.0 in /Users/engs2510/.pyenv/versions/3.11.7/envs/pybop/lib/python3.11/site-packages (from ipython>=6.1.0->ipywidgets) (2.17.2)\n", + "Requirement already satisfied: stack-data in /Users/engs2510/.pyenv/versions/3.11.7/envs/pybop/lib/python3.11/site-packages (from ipython>=6.1.0->ipywidgets) (0.6.3)\n", + "Requirement already satisfied: pexpect>4.3 in /Users/engs2510/.pyenv/versions/3.11.7/envs/pybop/lib/python3.11/site-packages (from ipython>=6.1.0->ipywidgets) (4.9.0)\n", + "Requirement already satisfied: parso<0.9.0,>=0.8.3 in /Users/engs2510/.pyenv/versions/3.11.7/envs/pybop/lib/python3.11/site-packages (from jedi>=0.16->ipython>=6.1.0->ipywidgets) (0.8.3)\n", + "Requirement already satisfied: ptyprocess>=0.5 in /Users/engs2510/.pyenv/versions/3.11.7/envs/pybop/lib/python3.11/site-packages (from pexpect>4.3->ipython>=6.1.0->ipywidgets) (0.7.0)\n", + "Requirement already satisfied: wcwidth in /Users/engs2510/.pyenv/versions/3.11.7/envs/pybop/lib/python3.11/site-packages (from prompt-toolkit<3.1.0,>=3.0.41->ipython>=6.1.0->ipywidgets) (0.2.13)\n", + "Requirement already satisfied: executing>=1.2.0 in /Users/engs2510/.pyenv/versions/3.11.7/envs/pybop/lib/python3.11/site-packages (from stack-data->ipython>=6.1.0->ipywidgets) (2.0.1)\n", + "Requirement already satisfied: asttokens>=2.1.0 in /Users/engs2510/.pyenv/versions/3.11.7/envs/pybop/lib/python3.11/site-packages (from stack-data->ipython>=6.1.0->ipywidgets) (2.4.1)\n", + "Requirement already satisfied: pure-eval in /Users/engs2510/.pyenv/versions/3.11.7/envs/pybop/lib/python3.11/site-packages (from stack-data->ipython>=6.1.0->ipywidgets) (0.2.2)\n", + "Requirement already satisfied: six>=1.12.0 in /Users/engs2510/.pyenv/versions/3.11.7/envs/pybop/lib/python3.11/site-packages (from asttokens>=2.1.0->stack-data->ipython>=6.1.0->ipywidgets) (1.16.0)\n", + "Note: you may need to restart the kernel to use updated packages.\n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "%pip install --upgrade pip ipywidgets\n", + "%pip install pybop -q" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "jAvD5fk104p0" + }, + "source": [ + "### Importing Libraries\n", + "\n", + "With the environment set up, we can now import PyBOP alongside other libraries we will need:" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "execution": { + "iopub.execute_input": "2024-04-04T13:51:41.936561Z", + "iopub.status.busy": "2024-04-04T13:51:41.936439Z", + "iopub.status.idle": "2024-04-04T13:51:42.508083Z", + "shell.execute_reply": "2024-04-04T13:51:42.507654Z" + }, + "id": "SQdt4brD04p1" + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "\n", + "import pybop" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5XU-dMtU04p2" + }, + "source": [ + "### Generate Synthetic Data\n", + "\n", + "To demonstrate parameter estimation, we first need some data. We will generate synthetic data using the PyBOP forward model, which requires defining a parameter set and the model itself.\n", + "\n", + "#### Defining Parameters and Model\n", + "\n", + "We start by creating an example parameter set and then instantiate the single-particle model (SPM):" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "execution": { + "iopub.execute_input": "2024-04-04T13:51:42.509591Z", + "iopub.status.busy": "2024-04-04T13:51:42.509437Z", + "iopub.status.idle": "2024-04-04T13:51:42.534794Z", + "shell.execute_reply": "2024-04-04T13:51:42.534452Z" + } + }, + "outputs": [], + "source": [ + "parameter_set = pybop.ParameterSet.pybamm(\"Chen2020\")\n", + "model = pybop.lithium_ion.SPM(parameter_set=parameter_set)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Simulating Forward Model\n", + "\n", + "We can then simulate the model using the `predict` method, with a default constant current to generate voltage data." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "execution": { + "iopub.execute_input": "2024-04-04T13:51:42.536154Z", + "iopub.status.busy": "2024-04-04T13:51:42.536069Z", + "iopub.status.idle": "2024-04-04T13:51:42.610305Z", + "shell.execute_reply": "2024-04-04T13:51:42.609892Z" + }, + "id": "sBasxv8U04p3" + }, + "outputs": [], + "source": [ + "t_eval = np.arange(0, 900, 2)\n", + "values = model.predict(t_eval=t_eval)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Adding Noise to Voltage Data\n", + "\n", + "To make the parameter estimation more realistic, we add Gaussian noise to the data." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "execution": { + "iopub.execute_input": "2024-04-04T13:51:42.611946Z", + "iopub.status.busy": "2024-04-04T13:51:42.611728Z", + "iopub.status.idle": "2024-04-04T13:51:42.621525Z", + "shell.execute_reply": "2024-04-04T13:51:42.621156Z" + } + }, + "outputs": [], + "source": [ + "sigma = 0.001\n", + "corrupt_values = values[\"Voltage [V]\"].data + np.random.normal(0, sigma, len(t_eval))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "X8-tubYY04p_" + }, + "source": [ + "## Identify the Parameters" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "PQqhvSZN04p_" + }, + "source": [ + "We will now set up the parameter estimation process by defining the datasets for optimisation and selecting the model parameters we wish to estimate." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Creating Optimisation Dataset\n", + "\n", + "The dataset for optimisation is composed of time, current, and the noisy voltage data:" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "execution": { + "iopub.execute_input": "2024-04-04T13:51:42.622671Z", + "iopub.status.busy": "2024-04-04T13:51:42.622478Z", + "iopub.status.idle": "2024-04-04T13:51:42.628864Z", + "shell.execute_reply": "2024-04-04T13:51:42.628519Z" + }, + "id": "zuvGHWID04p_" + }, + "outputs": [], + "source": [ + "dataset = pybop.Dataset(\n", + " {\n", + " \"Time [s]\": t_eval,\n", + " \"Current function [A]\": values[\"Current [A]\"].data,\n", + " \"Voltage [V]\": corrupt_values,\n", + " }\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ffS3CF_704qA" + }, + "source": [ + "### Defining Parameters to Estimate\n", + "\n", + "We select the parameters for estimation and set up their prior distributions and bounds:" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "execution": { + "iopub.execute_input": "2024-04-04T13:51:42.629987Z", + "iopub.status.busy": "2024-04-04T13:51:42.629809Z", + "iopub.status.idle": "2024-04-04T13:51:42.631895Z", + "shell.execute_reply": "2024-04-04T13:51:42.631621Z" + }, + "id": "WPCybXIJ04qA" + }, + "outputs": [], + "source": [ + "parameters = [\n", + " pybop.Parameter(\n", + " \"Negative electrode active material volume fraction\",\n", + " prior=pybop.Gaussian(0.6, 0.02),\n", + " bounds=[0.5, 0.8],\n", + " ),\n", + " pybop.Parameter(\n", + " \"Positive electrode active material volume fraction\",\n", + " prior=pybop.Gaussian(0.48, 0.02),\n", + " bounds=[0.4, 0.7],\n", + " ),\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "n4OHa-aF04qA" + }, + "source": [ + "### Setting up the Optimisation Problem\n", + "\n", + "With the datasets and parameters defined, we can set up the optimisation problem, its cost function, and the optimiser." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "execution": { + "iopub.execute_input": "2024-04-04T13:51:42.632931Z", + "iopub.status.busy": "2024-04-04T13:51:42.632782Z", + "iopub.status.idle": "2024-04-04T13:51:42.705454Z", + "shell.execute_reply": "2024-04-04T13:51:42.705066Z" + }, + "id": "etMzRtx404qA" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NOTE: Boundaries ignored by AdamW\n" + ] + } + ], + "source": [ + "problem = pybop.FittingProblem(model, parameters, dataset)\n", + "cost = pybop.SumSquaredError(problem)\n", + "optim = pybop.Optimisation(cost, optimiser=pybop.AdamW)\n", + "optim.set_max_unchanged_iterations(40)\n", + "optim.set_max_iterations(150)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "caprp-bV04qB" + }, + "source": [ + "### Running the Optimisation\n", + "\n", + "We proceed to run the AdamW optimisation algorithm to estimate the parameters:" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "execution": { + "iopub.execute_input": "2024-04-04T13:51:42.706564Z", + "iopub.status.busy": "2024-04-04T13:51:42.706469Z", + "iopub.status.idle": "2024-04-04T13:51:50.537424Z", + "shell.execute_reply": "2024-04-04T13:51:50.537032Z" + }, + "id": "-9OVt0EQ04qB" + }, + "outputs": [], + "source": [ + "x, final_cost = optim.run()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-4pZsDmS04qC" + }, + "source": [ + "### Viewing the Estimated Parameters\n", + "\n", + "After the optimisation, we can examine the estimated parameter values:" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "execution": { + "iopub.execute_input": "2024-04-04T13:51:50.538815Z", + "iopub.status.busy": "2024-04-04T13:51:50.538619Z", + "iopub.status.idle": "2024-04-04T13:51:50.541683Z", + "shell.execute_reply": "2024-04-04T13:51:50.541465Z" + }, + "id": "Hgz8SV4i04qC", + "outputId": "e1e42ae7-5075-4c47-dd68-1b22ecc170f6" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "array([0.76334915, 0.66225839])" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "x # This will output the estimated parameters" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "KxKURtH704qC" + }, + "source": [ + "## Plotting and Visualisation\n", + "\n", + "PyBOP provides various plotting utilities to visualise the results of the optimisation." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-cWCOiqR04qC" + }, + "source": [ + "### Comparing System Response\n", + "\n", + "We can quickly plot the system's response using the estimated parameters compared to the target:" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 467 + }, + "execution": { + "iopub.execute_input": "2024-04-04T13:51:50.542618Z", + "iopub.status.busy": "2024-04-04T13:51:50.542472Z", + "iopub.status.idle": "2024-04-04T13:51:50.986055Z", + "shell.execute_reply": "2024-04-04T13:51:50.985844Z" + }, + "id": "tJUJ80Ve04qD", + "outputId": "855fbaa2-1e09-4935-eb1a-8caf7f99eb75" + }, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "02004006008003.83.853.93.9544.05ReferenceModelOptimised ComparisonTime / sVoltage / V" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "pybop.quick_plot(problem, parameter_values=x, title=\"Optimised Comparison\");" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Convergence and Parameter Trajectories\n", + "\n", + "To assess the optimisation process, we can plot the convergence of the cost function and the trajectories of the parameters:" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "execution": { + "iopub.execute_input": "2024-04-04T13:51:50.987237Z", + "iopub.status.busy": "2024-04-04T13:51:50.986963Z", + "iopub.status.idle": "2024-04-04T13:51:52.766386Z", + "shell.execute_reply": "2024-04-04T13:51:52.766178Z" + }, + "id": "N5XYkevi04qD" + }, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "2040608010012014000.511.522.533.54ConvergenceIterationCost" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "image/svg+xml": [ + "0501000.60.650.70.750.80.850501000.50.550.60.650.7Negative electrode active material volume fractionPositive electrode active material volume fractionParameter ConvergenceFunction CallFunction CallNegative electrode active material volume fractionPositive electrode active material volume fraction" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "pybop.plot_convergence(optim)\n", + "pybop.plot_parameters(optim);" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Cost Landscape\n", + "\n", + "Finally, we can visualise the cost landscape and the path taken by the optimiser:" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "execution": { + "iopub.execute_input": "2024-04-04T13:51:52.767346Z", + "iopub.status.busy": "2024-04-04T13:51:52.767261Z", + "iopub.status.idle": "2024-04-04T13:51:57.666000Z", + "shell.execute_reply": "2024-04-04T13:51:57.665745Z" + } + }, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "0.50.550.60.650.70.750.80.40.450.50.550.60.650.70246810Cost LandscapeNegative electrode active material volume fractionPositive electrode active material volume fraction" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "image/svg+xml": [ + "0.60.650.70.750.80.850.90.50.550.60.650.70.750.80.40.81.21.622.4Cost LandscapeNegative electrode active material volume fractionPositive electrode active material volume fraction" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# Plot the cost landscape\n", + "pybop.plot2d(cost, steps=15)\n", + "# Plot the cost landscape with optimisation path and updated bounds\n", + "bounds = np.array([[0.6, 0.9], [0.5, 0.8]])\n", + "pybop.plot2d(optim, bounds=bounds, steps=15);" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Conclusion\n", + "\n", + "This notebook illustrates how to perform parameter estimation using AdamW in PyBOP, providing insights into the optimisation process through various visualisations." + ] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.7" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "06f2374f91c8455bb63252092512f2ed": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "423bffea3a1c42b49a9ad71218e5811b": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "56ff19291e464d63b23e63b8e2ac9ea3": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "SliderStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "SliderStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "description_width": "", + "handle_color": null + } + }, + "646a8670cb204a31bb56bc2380898093": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "7d46516469314b88be3500e2afcafcf6": { + "model_module": "@jupyter-widgets/output", + "model_module_version": "1.0.0", + "model_name": "OutputModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/output", + "_model_module_version": "1.0.0", + "_model_name": "OutputModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/output", + "_view_module_version": "1.0.0", + "_view_name": "OutputView", + "layout": "IPY_MODEL_646a8670cb204a31bb56bc2380898093", + "msg_id": "", + "outputs": [], + "tabbable": null, + "tooltip": null + } + }, + "8d003c14da5f4fa68284b28c15cee6e6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "VBoxModel", + "state": { + "_dom_classes": [ + "widget-interact" + ], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "VBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "VBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_aef2fa7adcc14ad0854b73d5910ae3b4", + "IPY_MODEL_7d46516469314b88be3500e2afcafcf6" + ], + "layout": "IPY_MODEL_423bffea3a1c42b49a9ad71218e5811b", + "tabbable": null, + "tooltip": null + } + }, + "aef2fa7adcc14ad0854b73d5910ae3b4": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "FloatSliderModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "FloatSliderModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "FloatSliderView", + "behavior": "drag-tap", + "continuous_update": true, + "description": "t", + "description_allow_html": false, + "disabled": false, + "layout": "IPY_MODEL_06f2374f91c8455bb63252092512f2ed", + "max": 1.1333333333333333, + "min": 0, + "orientation": "horizontal", + "readout": true, + "readout_format": ".2f", + "step": 0.011333333333333332, + "style": "IPY_MODEL_56ff19291e464d63b23e63b8e2ac9ea3", + "tabbable": null, + "tooltip": null, + "value": 0 + } + } + } + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/pybop/_optimisation.py b/pybop/_optimisation.py index 68835158..bc68f504 100644 --- a/pybop/_optimisation.py +++ b/pybop/_optimisation.py @@ -102,7 +102,7 @@ def __init__( if self.pints: self.optimiser = self.optimiser( - self.x0, sigma0=self.sigma0, boundaries=self.bounds + self.x0, sigma0=self.sigma0, bounds=self.bounds ) # Check if sensitivities are required diff --git a/pybop/optimisers/AdamW.py b/pybop/optimisers/AdamW.py index 54d8aa5f..46799dcc 100644 --- a/pybop/optimisers/AdamW.py +++ b/pybop/optimisers/AdamW.py @@ -53,8 +53,12 @@ class AdamW(PintsOptimiser): https://doi.org/10.48550/arXiv.1711.05101 """ - def __init__(self, x0, sigma0=0.015, boundaries=None): - super().__init__(x0, sigma0, boundaries) + def __init__(self, x0, sigma0=0.015, bounds=None): + if bounds is not None: + print("NOTE: Boundaries ignored by AdamW") + + self.boundaries = None # Bounds ignored in pints.Adam + super().__init__(x0, sigma0, self.boundaries) # Set optimiser state self._running = False diff --git a/tests/integration/test_parameterisations.py b/tests/integration/test_parameterisations.py index 4a922e9b..76b1defd 100644 --- a/tests/integration/test_parameterisations.py +++ b/tests/integration/test_parameterisations.py @@ -83,6 +83,7 @@ def spm_costs(self, model, parameters, cost_class, init_soc): pybop.SciPyMinimize, pybop.SciPyDifferentialEvolution, pybop.Adam, + pybop.AdamW, pybop.CMAES, pybop.GradientDescent, pybop.IRPropMin, diff --git a/tests/unit/test_optimisation.py b/tests/unit/test_optimisation.py index 54674c95..e8898ff6 100644 --- a/tests/unit/test_optimisation.py +++ b/tests/unit/test_optimisation.py @@ -75,6 +75,7 @@ def two_param_cost(self, model, two_parameters, dataset): (pybop.SciPyDifferentialEvolution, "SciPyDifferentialEvolution"), (pybop.GradientDescent, "Gradient descent"), (pybop.Adam, "Adam"), + (pybop.AdamW, "AdamW"), (pybop.CMAES, "Covariance Matrix Adaptation Evolution Strategy (CMA-ES)"), (pybop.SNES, "Seperable Natural Evolution Strategy (SNES)"), (pybop.XNES, "Exponential Natural Evolution Strategy (xNES)"), From 8f6a38419932b866571d6cfaafc398ce39cf41ee Mon Sep 17 00:00:00 2001 From: Brady Planden Date: Fri, 10 May 2024 09:27:36 +0100 Subject: [PATCH 03/11] updt filename, increase coverage --- pybop/__init__.py | 2 +- pybop/optimisers/{AdamW.py => _adamw.py} | 6 +++--- tests/unit/test_optimisation.py | 18 ++++++++++++++++++ 3 files changed, 22 insertions(+), 4 deletions(-) rename pybop/optimisers/{AdamW.py => _adamw.py} (97%) diff --git a/pybop/__init__.py b/pybop/__init__.py index bc6fa54a..bb635bc4 100644 --- a/pybop/__init__.py +++ b/pybop/__init__.py @@ -113,7 +113,7 @@ SNES, XNES, ) -from .optimisers.AdamW import AdamW +from .optimisers._adamw import AdamW # # Parameter classes diff --git a/pybop/optimisers/AdamW.py b/pybop/optimisers/_adamw.py similarity index 97% rename from pybop/optimisers/AdamW.py rename to pybop/optimisers/_adamw.py index 46799dcc..0e1a1b4b 100644 --- a/pybop/optimisers/AdamW.py +++ b/pybop/optimisers/_adamw.py @@ -226,7 +226,7 @@ def set_lambda(self, lambda_=0.01): try: lambda_ = float(lambda_) # Ensure b1 is a floatable except Exception: - raise ("lambda_ must be numeric, floatable value.") + raise TypeError("lambda_ must be numeric, floatable value.") if not 0 < lambda_ <= 1: print("lambda_ must a positive value between 0 and 1") @@ -241,7 +241,7 @@ def set_b1(self, b1): try: b1 = float(b1) # Ensure b1 is a floatable except Exception: - raise ("b1 must be numeric, floatable value.") + raise TypeError("b1 must be numeric, floatable value.") if not 0 < b1 <= 1: print("b1 must a positive value between 0 and 1") @@ -256,7 +256,7 @@ def set_b2(self, b2): try: b2 = float(b2) # Ensure b2 is a floatable except Exception: - raise ("b2 must be numeric, floatable value.") + raise TypeError("b2 must be numeric, floatable value.") if not 0 < b2 <= 1: print("b2 must a positive value between 0 and 1") diff --git a/tests/unit/test_optimisation.py b/tests/unit/test_optimisation.py index e8898ff6..07b4c56e 100644 --- a/tests/unit/test_optimisation.py +++ b/tests/unit/test_optimisation.py @@ -113,6 +113,24 @@ def test_optimiser_classes(self, two_param_cost, optimiser_class, expected_name) # Correct value opt.optimiser.set_population_size(5) + # Test AdamW hyperparameters + if optimiser_class in [pybop.AdamW]: + opt.optimiser.set_b1(0.9) + opt.optimiser.set_b2(0.9) + opt.optimiser.set_lambda(0.1) + + assert opt.optimiser._b1 == 0.9 + assert opt.optimiser._b2 == 0.9 + assert opt.optimiser._lambda == 0.1 + + # Incorrect values + with pytest.raises(TypeError): + opt.optimiser.set_b1("Value") + with pytest.raises(TypeError): + opt.optimiser.set_b2("Value") + with pytest.raises(TypeError): + opt.optimiser.set_lambda("Value") + @pytest.mark.unit def test_single_parameter(self, cost): # Test catch for optimisers that can only run with multiple parameters From c8f8bb6e09d97c09e83811f68efbf283d0e092aa Mon Sep 17 00:00:00 2001 From: Brady Planden Date: Sat, 11 May 2024 15:15:16 +0100 Subject: [PATCH 04/11] refactor: replace Adam optimiser with AdamW, add changelog entry --- CHANGELOG.md | 1 + benchmarks/benchmark_parameterisation.py | 2 +- .../benchmark_track_parameterisation.py | 2 +- .../multi_optimiser_identification.ipynb | 144 +++++++++--------- .../scripts/{spm_adam.py => spm_AdamW.py} | 2 +- pybop/__init__.py | 1 - pybop/optimisers/pints_optimisers.py | 30 ---- tests/integration/test_parameterisations.py | 1 - tests/unit/test_optimisation.py | 1 - 9 files changed, 76 insertions(+), 108 deletions(-) rename examples/scripts/{spm_adam.py => spm_AdamW.py} (98%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21a72551..05635001 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Features +- [#316](https://github.com/pybop-team/PyBOP/pull/316) - Adds Adam with weight decay (AdamW) optimiser, replaces pints.Adam implementation. - [#301](https://github.com/pybop-team/PyBOP/pull/301) - Updates default echem solver to "fast with events" mode. - [#251](https://github.com/pybop-team/PyBOP/pull/251) - Increment PyBaMM > v23.5, remove redundant tests within integration tests, increment citation version, fix examples with incorrect model definitions. - [#285](https://github.com/pybop-team/PyBOP/pull/285) - Drop support for Python 3.8. diff --git a/benchmarks/benchmark_parameterisation.py b/benchmarks/benchmark_parameterisation.py index 76976887..e62150e5 100644 --- a/benchmarks/benchmark_parameterisation.py +++ b/benchmarks/benchmark_parameterisation.py @@ -13,7 +13,7 @@ class BenchmarkParameterisation: [ pybop.SciPyMinimize, pybop.SciPyDifferentialEvolution, - pybop.Adam, + pybop.AdamW, pybop.CMAES, pybop.GradientDescent, pybop.IRPropMin, diff --git a/benchmarks/benchmark_track_parameterisation.py b/benchmarks/benchmark_track_parameterisation.py index 793ab462..0c15c651 100644 --- a/benchmarks/benchmark_track_parameterisation.py +++ b/benchmarks/benchmark_track_parameterisation.py @@ -13,7 +13,7 @@ class BenchmarkTrackParameterisation: [ pybop.SciPyMinimize, pybop.SciPyDifferentialEvolution, - pybop.Adam, + pybop.AdamW, pybop.CMAES, pybop.GradientDescent, pybop.IRPropMin, diff --git a/examples/notebooks/multi_optimiser_identification.ipynb b/examples/notebooks/multi_optimiser_identification.ipynb index 44e75747..52968d5a 100644 --- a/examples/notebooks/multi_optimiser_identification.ipynb +++ b/examples/notebooks/multi_optimiser_identification.ipynb @@ -36,27 +36,27 @@ "name": "stdout", "output_type": "stream", "text": [ - "Requirement already satisfied: pip in /Users/engs2510/.pyenv/versions/3.11.7/envs/pybop/lib/python3.11/site-packages (24.0)\n", - "Requirement already satisfied: ipywidgets in /Users/engs2510/.pyenv/versions/3.11.7/envs/pybop/lib/python3.11/site-packages (8.1.2)\n", - "Requirement already satisfied: comm>=0.1.3 in /Users/engs2510/.pyenv/versions/3.11.7/envs/pybop/lib/python3.11/site-packages (from ipywidgets) (0.2.1)\n", - "Requirement already satisfied: ipython>=6.1.0 in /Users/engs2510/.pyenv/versions/3.11.7/envs/pybop/lib/python3.11/site-packages (from ipywidgets) (8.22.1)\n", - "Requirement already satisfied: traitlets>=4.3.1 in /Users/engs2510/.pyenv/versions/3.11.7/envs/pybop/lib/python3.11/site-packages (from ipywidgets) (5.14.1)\n", - "Requirement already satisfied: widgetsnbextension~=4.0.10 in /Users/engs2510/.pyenv/versions/3.11.7/envs/pybop/lib/python3.11/site-packages (from ipywidgets) (4.0.10)\n", - "Requirement already satisfied: jupyterlab-widgets~=3.0.10 in /Users/engs2510/.pyenv/versions/3.11.7/envs/pybop/lib/python3.11/site-packages (from ipywidgets) (3.0.10)\n", - "Requirement already satisfied: decorator in /Users/engs2510/.pyenv/versions/3.11.7/envs/pybop/lib/python3.11/site-packages (from ipython>=6.1.0->ipywidgets) (5.1.1)\n", - "Requirement already satisfied: jedi>=0.16 in /Users/engs2510/.pyenv/versions/3.11.7/envs/pybop/lib/python3.11/site-packages (from ipython>=6.1.0->ipywidgets) (0.19.1)\n", - "Requirement already satisfied: matplotlib-inline in /Users/engs2510/.pyenv/versions/3.11.7/envs/pybop/lib/python3.11/site-packages (from ipython>=6.1.0->ipywidgets) (0.1.6)\n", - "Requirement already satisfied: prompt-toolkit<3.1.0,>=3.0.41 in /Users/engs2510/.pyenv/versions/3.11.7/envs/pybop/lib/python3.11/site-packages (from ipython>=6.1.0->ipywidgets) (3.0.43)\n", - "Requirement already satisfied: pygments>=2.4.0 in /Users/engs2510/.pyenv/versions/3.11.7/envs/pybop/lib/python3.11/site-packages (from ipython>=6.1.0->ipywidgets) (2.17.2)\n", - "Requirement already satisfied: stack-data in /Users/engs2510/.pyenv/versions/3.11.7/envs/pybop/lib/python3.11/site-packages (from ipython>=6.1.0->ipywidgets) (0.6.3)\n", - "Requirement already satisfied: pexpect>4.3 in /Users/engs2510/.pyenv/versions/3.11.7/envs/pybop/lib/python3.11/site-packages (from ipython>=6.1.0->ipywidgets) (4.9.0)\n", - "Requirement already satisfied: parso<0.9.0,>=0.8.3 in /Users/engs2510/.pyenv/versions/3.11.7/envs/pybop/lib/python3.11/site-packages (from jedi>=0.16->ipython>=6.1.0->ipywidgets) (0.8.3)\n", - "Requirement already satisfied: ptyprocess>=0.5 in /Users/engs2510/.pyenv/versions/3.11.7/envs/pybop/lib/python3.11/site-packages (from pexpect>4.3->ipython>=6.1.0->ipywidgets) (0.7.0)\n", - "Requirement already satisfied: wcwidth in /Users/engs2510/.pyenv/versions/3.11.7/envs/pybop/lib/python3.11/site-packages (from prompt-toolkit<3.1.0,>=3.0.41->ipython>=6.1.0->ipywidgets) (0.2.13)\n", - "Requirement already satisfied: executing>=1.2.0 in /Users/engs2510/.pyenv/versions/3.11.7/envs/pybop/lib/python3.11/site-packages (from stack-data->ipython>=6.1.0->ipywidgets) (2.0.1)\n", - "Requirement already satisfied: asttokens>=2.1.0 in /Users/engs2510/.pyenv/versions/3.11.7/envs/pybop/lib/python3.11/site-packages (from stack-data->ipython>=6.1.0->ipywidgets) (2.4.1)\n", - "Requirement already satisfied: pure-eval in /Users/engs2510/.pyenv/versions/3.11.7/envs/pybop/lib/python3.11/site-packages (from stack-data->ipython>=6.1.0->ipywidgets) (0.2.2)\n", - "Requirement already satisfied: six>=1.12.0 in /Users/engs2510/.pyenv/versions/3.11.7/envs/pybop/lib/python3.11/site-packages (from asttokens>=2.1.0->stack-data->ipython>=6.1.0->ipywidgets) (1.16.0)\n", + "Requirement already satisfied: pip in /Users/engs2510/.pyenv/versions/3.12.2/envs/pybop-3.12/lib/python3.12/site-packages (24.0)\n", + "Requirement already satisfied: ipywidgets in /Users/engs2510/.pyenv/versions/3.12.2/envs/pybop-3.12/lib/python3.12/site-packages (8.1.2)\n", + "Requirement already satisfied: comm>=0.1.3 in /Users/engs2510/.pyenv/versions/3.12.2/envs/pybop-3.12/lib/python3.12/site-packages (from ipywidgets) (0.2.2)\n", + "Requirement already satisfied: ipython>=6.1.0 in /Users/engs2510/.pyenv/versions/3.12.2/envs/pybop-3.12/lib/python3.12/site-packages (from ipywidgets) (8.23.0)\n", + "Requirement already satisfied: traitlets>=4.3.1 in /Users/engs2510/.pyenv/versions/3.12.2/envs/pybop-3.12/lib/python3.12/site-packages (from ipywidgets) (5.14.2)\n", + "Requirement already satisfied: widgetsnbextension~=4.0.10 in /Users/engs2510/.pyenv/versions/3.12.2/envs/pybop-3.12/lib/python3.12/site-packages (from ipywidgets) (4.0.10)\n", + "Requirement already satisfied: jupyterlab-widgets~=3.0.10 in /Users/engs2510/.pyenv/versions/3.12.2/envs/pybop-3.12/lib/python3.12/site-packages (from ipywidgets) (3.0.10)\n", + "Requirement already satisfied: decorator in /Users/engs2510/.pyenv/versions/3.12.2/envs/pybop-3.12/lib/python3.12/site-packages (from ipython>=6.1.0->ipywidgets) (5.1.1)\n", + "Requirement already satisfied: jedi>=0.16 in /Users/engs2510/.pyenv/versions/3.12.2/envs/pybop-3.12/lib/python3.12/site-packages (from ipython>=6.1.0->ipywidgets) (0.19.1)\n", + "Requirement already satisfied: matplotlib-inline in /Users/engs2510/.pyenv/versions/3.12.2/envs/pybop-3.12/lib/python3.12/site-packages (from ipython>=6.1.0->ipywidgets) (0.1.6)\n", + "Requirement already satisfied: prompt-toolkit<3.1.0,>=3.0.41 in /Users/engs2510/.pyenv/versions/3.12.2/envs/pybop-3.12/lib/python3.12/site-packages (from ipython>=6.1.0->ipywidgets) (3.0.43)\n", + "Requirement already satisfied: pygments>=2.4.0 in /Users/engs2510/.pyenv/versions/3.12.2/envs/pybop-3.12/lib/python3.12/site-packages (from ipython>=6.1.0->ipywidgets) (2.17.2)\n", + "Requirement already satisfied: stack-data in /Users/engs2510/.pyenv/versions/3.12.2/envs/pybop-3.12/lib/python3.12/site-packages (from ipython>=6.1.0->ipywidgets) (0.6.3)\n", + "Requirement already satisfied: pexpect>4.3 in /Users/engs2510/.pyenv/versions/3.12.2/envs/pybop-3.12/lib/python3.12/site-packages (from ipython>=6.1.0->ipywidgets) (4.9.0)\n", + "Requirement already satisfied: parso<0.9.0,>=0.8.3 in /Users/engs2510/.pyenv/versions/3.12.2/envs/pybop-3.12/lib/python3.12/site-packages (from jedi>=0.16->ipython>=6.1.0->ipywidgets) (0.8.4)\n", + "Requirement already satisfied: ptyprocess>=0.5 in /Users/engs2510/.pyenv/versions/3.12.2/envs/pybop-3.12/lib/python3.12/site-packages (from pexpect>4.3->ipython>=6.1.0->ipywidgets) (0.7.0)\n", + "Requirement already satisfied: wcwidth in /Users/engs2510/.pyenv/versions/3.12.2/envs/pybop-3.12/lib/python3.12/site-packages (from prompt-toolkit<3.1.0,>=3.0.41->ipython>=6.1.0->ipywidgets) (0.2.13)\n", + "Requirement already satisfied: executing>=1.2.0 in /Users/engs2510/.pyenv/versions/3.12.2/envs/pybop-3.12/lib/python3.12/site-packages (from stack-data->ipython>=6.1.0->ipywidgets) (2.0.1)\n", + "Requirement already satisfied: asttokens>=2.1.0 in /Users/engs2510/.pyenv/versions/3.12.2/envs/pybop-3.12/lib/python3.12/site-packages (from stack-data->ipython>=6.1.0->ipywidgets) (2.4.1)\n", + "Requirement already satisfied: pure-eval in /Users/engs2510/.pyenv/versions/3.12.2/envs/pybop-3.12/lib/python3.12/site-packages (from stack-data->ipython>=6.1.0->ipywidgets) (0.2.2)\n", + "Requirement already satisfied: six>=1.12.0 in /Users/engs2510/.pyenv/versions/3.12.2/envs/pybop-3.12/lib/python3.12/site-packages (from asttokens>=2.1.0->stack-data->ipython>=6.1.0->ipywidgets) (1.16.0)\n", "Note: you may need to restart the kernel to use updated packages.\n", "Note: you may need to restart the kernel to use updated packages.\n" ] @@ -278,7 +278,7 @@ "source": [ "### Selecting the Optimisers\n", "\n", - "Now, we can select the optimisers to investigate. The first object is a list of non-gradient-based PINTS's optimisers. The next object comprises the gradient-based PINTS's optimisers (Adam, GradientDescent, IRPropMin). The final object forms the SciPy optimisers which can have gradient and non-gradient-based algorithms." + "Now, we can select the optimisers to investigate. The first object is a list of non-gradient-based PINTS's optimisers. The next object comprises the gradient-based PINTS's optimisers (AdamW, GradientDescent, IRPropMin). The final object forms the SciPy optimisers which can have gradient and non-gradient-based algorithms." ] }, { @@ -295,7 +295,7 @@ "outputs": [], "source": [ "gradient_optimisers = [\n", - " pybop.Adam,\n", + " pybop.AdamW,\n", " pybop.GradientDescent,\n", " pybop.IRPropMin,\n", "]\n", @@ -342,8 +342,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "Running Adam\n", - "NOTE: Boundaries ignored by Adam\n", + "Running AdamW\n", + "NOTE: Boundaries ignored by AdamW\n", "Running GradientDescent\n", "NOTE: Boundaries ignored by Gradient Descent\n", "Running IRPropMin\n" @@ -446,16 +446,16 @@ "name": "stdout", "output_type": "stream", "text": [ - "| Optimiser: Adam | Results: [0.79454453 0.66136682] |\n", - "| Optimiser: Gradient descent | Results: [0.44491726 1.59641046] |\n", - "| Optimiser: iRprop- | Results: [0.8 0.66516786] |\n", - "| Optimiser: Covariance Matrix Adaptation Evolution Strategy (CMA-ES) | Results: [0.79997886 0.6651564 ] |\n", + "| Optimiser: AdamW | Results: [0.80186169 0.66943058] |\n", + "| Optimiser: Gradient descent | Results: [0.44491146 1.59642543] |\n", + "| Optimiser: iRprop- | Results: [0.8 0.66516386] |\n", + "| Optimiser: Covariance Matrix Adaptation Evolution Strategy (CMA-ES) | Results: [0.7999994 0.66516056] |\n", "| Optimiser: Seperable Natural Evolution Strategy (SNES) | Results: [0.79672265 0.66566242] |\n", "| Optimiser: Particle Swarm Optimisation (PSO) | Results: [0.79978922 0.66557426] |\n", "| Optimiser: Exponential Natural Evolution Strategy (xNES) | Results: [0.79992605 0.66513294] |\n", "| Optimiser: Nelder-Mead | Results: [0.81389091 0.66318217] |\n", - "| Optimiser: SciPyMinimize | Results: [0.63597003 0.7 ] |\n", - "| Optimiser: SciPyDifferentialEvolution | Results: [0.79999988 0.66516959] |\n" + "| Optimiser: SciPyMinimize | Results: [0.63594266 0.7 ] |\n", + "| Optimiser: SciPyDifferentialEvolution | Results: [0.79999973 0.6651644 ] |\n" ] } ], @@ -521,7 +521,7 @@ { "data": { "image/svg+xml": [ - "02004006008003.753.83.853.93.9544.05ReferenceModelAdamTime / sVoltage / V" + "02004006008003.753.83.853.93.9544.05ReferenceModelAdamWTime / sVoltage / V" ] }, "metadata": {}, @@ -530,7 +530,7 @@ { "data": { "image/svg+xml": [ - "02004006008003.753.83.853.93.9544.05ReferenceModelGradient descentTime / sVoltage / V" + "02004006008003.753.83.853.93.9544.05ReferenceModelGradient descentTime / sVoltage / V" ] }, "metadata": {}, @@ -539,7 +539,7 @@ { "data": { "image/svg+xml": [ - "02004006008003.753.83.853.93.9544.05ReferenceModeliRprop-Time / sVoltage / V" + "02004006008003.753.83.853.93.9544.05ReferenceModeliRprop-Time / sVoltage / V" ] }, "metadata": {}, @@ -548,7 +548,7 @@ { "data": { "image/svg+xml": [ - "02004006008003.753.83.853.93.9544.05ReferenceModelCovariance Matrix Adaptation Evolution Strategy (CMA-ES)Time / sVoltage / V" + "02004006008003.753.83.853.93.9544.05ReferenceModelCovariance Matrix Adaptation Evolution Strategy (CMA-ES)Time / sVoltage / V" ] }, "metadata": {}, @@ -557,7 +557,7 @@ { "data": { "image/svg+xml": [ - "02004006008003.753.83.853.93.9544.05ReferenceModelSeperable Natural Evolution Strategy (SNES)Time / sVoltage / V" + "02004006008003.753.83.853.93.9544.05ReferenceModelSeperable Natural Evolution Strategy (SNES)Time / sVoltage / V" ] }, "metadata": {}, @@ -566,7 +566,7 @@ { "data": { "image/svg+xml": [ - "02004006008003.753.83.853.93.9544.05ReferenceModelParticle Swarm Optimisation (PSO)Time / sVoltage / V" + "02004006008003.753.83.853.93.9544.05ReferenceModelParticle Swarm Optimisation (PSO)Time / sVoltage / V" ] }, "metadata": {}, @@ -575,7 +575,7 @@ { "data": { "image/svg+xml": [ - "02004006008003.753.83.853.93.9544.05ReferenceModelExponential Natural Evolution Strategy (xNES)Time / sVoltage / V" + "02004006008003.753.83.853.93.9544.05ReferenceModelExponential Natural Evolution Strategy (xNES)Time / sVoltage / V" ] }, "metadata": {}, @@ -584,7 +584,7 @@ { "data": { "image/svg+xml": [ - "02004006008003.753.83.853.93.9544.05ReferenceModelNelder-MeadTime / sVoltage / V" + "02004006008003.753.83.853.93.9544.05ReferenceModelNelder-MeadTime / sVoltage / V" ] }, "metadata": {}, @@ -593,7 +593,7 @@ { "data": { "image/svg+xml": [ - "02004006008003.753.83.853.93.9544.05ReferenceModelSciPyMinimizeTime / sVoltage / V" + "02004006008003.753.83.853.93.9544.05ReferenceModelSciPyMinimizeTime / sVoltage / V" ] }, "metadata": {}, @@ -602,7 +602,7 @@ { "data": { "image/svg+xml": [ - "02004006008003.753.83.853.93.9544.05ReferenceModelSciPyDifferentialEvolutionTime / sVoltage / V" + "02004006008003.753.83.853.93.9544.05ReferenceModelSciPyDifferentialEvolutionTime / sVoltage / V" ] }, "metadata": {}, @@ -641,7 +641,7 @@ { "data": { "image/svg+xml": [ - "510152025012345AdamIterationCost" + "51015202530012345AdamWIterationCost" ] }, "metadata": {}, @@ -650,7 +650,7 @@ { "data": { "image/svg+xml": [ - "05101520250.60.650.70.750.80.8505101520250.450.50.550.60.650.7Negative electrode active material volume fractionPositive electrode active material volume fractionParameter ConvergenceFunction CallFunction CallNegative electrode active material volume fractionPositive electrode active material volume fraction" + "05101520250.60.650.70.750.80.8505101520250.450.50.550.60.650.7Negative electrode active material volume fractionPositive electrode active material volume fractionParameter ConvergenceFunction CallFunction CallNegative electrode active material volume fractionPositive electrode active material volume fraction" ] }, "metadata": {}, @@ -659,7 +659,7 @@ { "data": { "image/svg+xml": [ - "5101520246810121416Gradient descentIterationCost" + "510152024681012141618Gradient descentIterationCost" ] }, "metadata": {}, @@ -668,7 +668,7 @@ { "data": { "image/svg+xml": [ - "0510150510152025300510150.511.522.5Negative electrode active material volume fractionPositive electrode active material volume fractionParameter ConvergenceFunction CallFunction CallNegative electrode active material volume fractionPositive electrode active material volume fraction" + "0510150510152025300510150.511.522.5Negative electrode active material volume fractionPositive electrode active material volume fractionParameter ConvergenceFunction CallFunction CallNegative electrode active material volume fractionPositive electrode active material volume fraction" ] }, "metadata": {}, @@ -677,7 +677,7 @@ { "data": { "image/svg+xml": [ - "10203040012345iRprop-IterationCost" + "10203040012345iRprop-IterationCost" ] }, "metadata": {}, @@ -686,7 +686,7 @@ { "data": { "image/svg+xml": [ - "0102030400.60.650.70.750.80102030400.450.50.550.60.650.7Negative electrode active material volume fractionPositive electrode active material volume fractionParameter ConvergenceFunction CallFunction CallNegative electrode active material volume fractionPositive electrode active material volume fraction" + "0102030400.60.650.70.750.80102030400.450.50.550.60.650.7Negative electrode active material volume fractionPositive electrode active material volume fractionParameter ConvergenceFunction CallFunction CallNegative electrode active material volume fractionPositive electrode active material volume fraction" ] }, "metadata": {}, @@ -695,7 +695,7 @@ { "data": { "image/svg+xml": [ - "1020304000.511.522.53Covariance Matrix Adaptation Evolution Strategy (CMA-ES)IterationCost" + "1020304000.511.522.53Covariance Matrix Adaptation Evolution Strategy (CMA-ES)IterationCost" ] }, "metadata": {}, @@ -704,7 +704,7 @@ { "data": { "image/svg+xml": [ - "0501001502002500.550.60.650.70.750.80501001502002500.450.50.550.60.650.7Negative electrode active material volume fractionPositive electrode active material volume fractionParameter ConvergenceFunction CallFunction CallNegative electrode active material volume fractionPositive electrode active material volume fraction" + "0501001502002500.550.60.650.70.750.80501001502002500.450.50.550.60.650.7Negative electrode active material volume fractionPositive electrode active material volume fractionParameter ConvergenceFunction CallFunction CallNegative electrode active material volume fractionPositive electrode active material volume fraction" ] }, "metadata": {}, @@ -713,7 +713,7 @@ { "data": { "image/svg+xml": [ - "10203040506000.511.522.533.5Seperable Natural Evolution Strategy (SNES)IterationCost" + "10203040506000.511.522.533.5Seperable Natural Evolution Strategy (SNES)IterationCost" ] }, "metadata": {}, @@ -722,7 +722,7 @@ { "data": { "image/svg+xml": [ - "01002003000.550.60.650.70.750.801002003000.450.50.550.60.650.7Negative electrode active material volume fractionPositive electrode active material volume fractionParameter ConvergenceFunction CallFunction CallNegative electrode active material volume fractionPositive electrode active material volume fraction" + "01002003000.550.60.650.70.750.801002003000.450.50.550.60.650.7Negative electrode active material volume fractionPositive electrode active material volume fractionParameter ConvergenceFunction CallFunction CallNegative electrode active material volume fractionPositive electrode active material volume fraction" ] }, "metadata": {}, @@ -731,7 +731,7 @@ { "data": { "image/svg+xml": [ - "1020304000.511.522.5Particle Swarm Optimisation (PSO)IterationCost" + "1020304000.511.522.5Particle Swarm Optimisation (PSO)IterationCost" ] }, "metadata": {}, @@ -740,7 +740,7 @@ { "data": { "image/svg+xml": [ - "0501001502000.50.550.60.650.70.750.80501001502000.40.450.50.550.60.650.7Negative electrode active material volume fractionPositive electrode active material volume fractionParameter ConvergenceFunction CallFunction CallNegative electrode active material volume fractionPositive electrode active material volume fraction" + "0501001502000.50.550.60.650.70.750.80501001502000.40.450.50.550.60.650.7Negative electrode active material volume fractionPositive electrode active material volume fractionParameter ConvergenceFunction CallFunction CallNegative electrode active material volume fractionPositive electrode active material volume fraction" ] }, "metadata": {}, @@ -749,7 +749,7 @@ { "data": { "image/svg+xml": [ - "10203040506000.511.522.5Exponential Natural Evolution Strategy (xNES)IterationCost" + "10203040506000.511.522.5Exponential Natural Evolution Strategy (xNES)IterationCost" ] }, "metadata": {}, @@ -758,7 +758,7 @@ { "data": { "image/svg+xml": [ - "01002003000.60.650.70.750.801002003000.450.50.550.60.650.7Negative electrode active material volume fractionPositive electrode active material volume fractionParameter ConvergenceFunction CallFunction CallNegative electrode active material volume fractionPositive electrode active material volume fraction" + "01002003000.60.650.70.750.801002003000.450.50.550.60.650.7Negative electrode active material volume fractionPositive electrode active material volume fractionParameter ConvergenceFunction CallFunction CallNegative electrode active material volume fractionPositive electrode active material volume fraction" ] }, "metadata": {}, @@ -767,7 +767,7 @@ { "data": { "image/svg+xml": [ - "10203040506000.511.522.533.5Nelder-MeadIterationCost" + "10203040506000.511.522.533.5Nelder-MeadIterationCost" ] }, "metadata": {}, @@ -776,7 +776,7 @@ { "data": { "image/svg+xml": [ - "02040600.60.650.70.750.80.8502040600.450.50.550.60.650.70.750.80.850.9Negative electrode active material volume fractionPositive electrode active material volume fractionParameter ConvergenceFunction CallFunction CallNegative electrode active material volume fractionPositive electrode active material volume fraction" + "02040600.60.650.70.750.80.8502040600.450.50.550.60.650.70.750.80.850.9Negative electrode active material volume fractionPositive electrode active material volume fractionParameter ConvergenceFunction CallFunction CallNegative electrode active material volume fractionPositive electrode active material volume fraction" ] }, "metadata": {}, @@ -785,7 +785,7 @@ { "data": { "image/svg+xml": [ - "510152025012345SciPyMinimizeIterationCost" + "510152025012345SciPyMinimizeIterationCost" ] }, "metadata": {}, @@ -794,7 +794,7 @@ { "data": { "image/svg+xml": [ - "05101520250.60.610.620.630.640.650.660.6705101520250.450.50.550.60.650.7Negative electrode active material volume fractionPositive electrode active material volume fractionParameter ConvergenceFunction CallFunction CallNegative electrode active material volume fractionPositive electrode active material volume fraction" + "05101520250.60.610.620.630.640.650.660.6705101520250.450.50.550.60.650.7Negative electrode active material volume fractionPositive electrode active material volume fractionParameter ConvergenceFunction CallFunction CallNegative electrode active material volume fractionPositive electrode active material volume fraction" ] }, "metadata": {}, @@ -803,7 +803,7 @@ { "data": { "image/svg+xml": [ - "510152025300.00360.00380.0040.00420.0044SciPyDifferentialEvolutionIterationCost" + "510152025300.00360.00380.0040.00420.0044SciPyDifferentialEvolutionIterationCost" ] }, "metadata": {}, @@ -812,7 +812,7 @@ { "data": { "image/svg+xml": [ - "01020300.7550.760.7650.770.7750.780.7850.790.7950.801020300.6650.6660.6670.6680.6690.67Negative electrode active material volume fractionPositive electrode active material volume fractionParameter ConvergenceFunction CallFunction CallNegative electrode active material volume fractionPositive electrode active material volume fraction" + "05101520250.7550.760.7650.770.7750.780.7850.790.7950.805101520250.6650.6660.6670.6680.6690.67Negative electrode active material volume fractionPositive electrode active material volume fractionParameter ConvergenceFunction CallFunction CallNegative electrode active material volume fractionPositive electrode active material volume fraction" ] }, "metadata": {}, @@ -849,7 +849,7 @@ { "data": { "image/svg+xml": [ - "0.50.550.60.650.70.750.80.50.550.60.650.70.750.80.511.522.533.5AdamNegative electrode active material volume fractionPositive electrode active material volume fraction" + "0.50.550.60.650.70.750.80.50.550.60.650.70.750.80.511.522.533.5AdamWNegative electrode active material volume fractionPositive electrode active material volume fraction" ] }, "metadata": {}, @@ -858,7 +858,7 @@ { "data": { "image/svg+xml": [ - "0.50.550.60.650.70.750.80.50.550.60.650.70.750.80.511.522.533.5Gradient descentNegative electrode active material volume fractionPositive electrode active material volume fraction" + "0.50.550.60.650.70.750.80.50.550.60.650.70.750.80.511.522.533.5Gradient descentNegative electrode active material volume fractionPositive electrode active material volume fraction" ] }, "metadata": {}, @@ -867,7 +867,7 @@ { "data": { "image/svg+xml": [ - "0.50.550.60.650.70.750.80.50.550.60.650.70.750.80.511.522.533.5iRprop-Negative electrode active material volume fractionPositive electrode active material volume fraction" + "0.50.550.60.650.70.750.80.50.550.60.650.70.750.80.511.522.533.5iRprop-Negative electrode active material volume fractionPositive electrode active material volume fraction" ] }, "metadata": {}, @@ -876,7 +876,7 @@ { "data": { "image/svg+xml": [ - "0.50.550.60.650.70.750.80.50.550.60.650.70.750.80.511.522.533.5Covariance Matrix Adaptation Evolution Strategy (CMA-ES)Negative electrode active material volume fractionPositive electrode active material volume fraction" + "0.50.550.60.650.70.750.80.50.550.60.650.70.750.80.511.522.533.5Covariance Matrix Adaptation Evolution Strategy (CMA-ES)Negative electrode active material volume fractionPositive electrode active material volume fraction" ] }, "metadata": {}, @@ -885,7 +885,7 @@ { "data": { "image/svg+xml": [ - "0.50.550.60.650.70.750.80.50.550.60.650.70.750.80.511.522.533.5Seperable Natural Evolution Strategy (SNES)Negative electrode active material volume fractionPositive electrode active material volume fraction" + "0.50.550.60.650.70.750.80.50.550.60.650.70.750.80.511.522.533.5Seperable Natural Evolution Strategy (SNES)Negative electrode active material volume fractionPositive electrode active material volume fraction" ] }, "metadata": {}, @@ -894,7 +894,7 @@ { "data": { "image/svg+xml": [ - "0.50.550.60.650.70.750.80.50.550.60.650.70.750.80.511.522.533.5Particle Swarm Optimisation (PSO)Negative electrode active material volume fractionPositive electrode active material volume fraction" + "0.50.550.60.650.70.750.80.50.550.60.650.70.750.80.511.522.533.5Particle Swarm Optimisation (PSO)Negative electrode active material volume fractionPositive electrode active material volume fraction" ] }, "metadata": {}, @@ -903,7 +903,7 @@ { "data": { "image/svg+xml": [ - "0.50.550.60.650.70.750.80.50.550.60.650.70.750.80.511.522.533.5Exponential Natural Evolution Strategy (xNES)Negative electrode active material volume fractionPositive electrode active material volume fraction" + "0.50.550.60.650.70.750.80.50.550.60.650.70.750.80.511.522.533.5Exponential Natural Evolution Strategy (xNES)Negative electrode active material volume fractionPositive electrode active material volume fraction" ] }, "metadata": {}, @@ -912,7 +912,7 @@ { "data": { "image/svg+xml": [ - "0.50.550.60.650.70.750.80.50.550.60.650.70.750.80.511.522.533.5Nelder-MeadNegative electrode active material volume fractionPositive electrode active material volume fraction" + "0.50.550.60.650.70.750.80.50.550.60.650.70.750.80.511.522.533.5Nelder-MeadNegative electrode active material volume fractionPositive electrode active material volume fraction" ] }, "metadata": {}, @@ -921,7 +921,7 @@ { "data": { "image/svg+xml": [ - "0.50.550.60.650.70.750.80.50.550.60.650.70.750.80.511.522.533.5SciPyMinimizeNegative electrode active material volume fractionPositive electrode active material volume fraction" + "0.50.550.60.650.70.750.80.50.550.60.650.70.750.80.511.522.533.5SciPyMinimizeNegative electrode active material volume fractionPositive electrode active material volume fraction" ] }, "metadata": {}, @@ -930,7 +930,7 @@ { "data": { "image/svg+xml": [ - "0.50.550.60.650.70.750.80.50.550.60.650.70.750.80.511.522.533.5SciPyDifferentialEvolutionNegative electrode active material volume fractionPositive electrode active material volume fraction" + "0.50.550.60.650.70.750.80.50.550.60.650.70.750.80.511.522.533.5SciPyDifferentialEvolutionNegative electrode active material volume fractionPositive electrode active material volume fraction" ] }, "metadata": {}, @@ -980,7 +980,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.7" + "version": "3.12.2" }, "widgets": { "application/vnd.jupyter.widget-state+json": { diff --git a/examples/scripts/spm_adam.py b/examples/scripts/spm_AdamW.py similarity index 98% rename from examples/scripts/spm_adam.py rename to examples/scripts/spm_AdamW.py index f6a75354..f478aa6b 100644 --- a/examples/scripts/spm_adam.py +++ b/examples/scripts/spm_AdamW.py @@ -56,7 +56,7 @@ def noise(sigma): cost = pybop.RootMeanSquaredError(problem) optim = pybop.Optimisation( cost, - optimiser=pybop.Adam, + optimiser=pybop.AdamW, verbose=True, allow_infeasible_solutions=True, sigma0=0.05, diff --git a/pybop/__init__.py b/pybop/__init__.py index bb635bc4..9c161f6f 100644 --- a/pybop/__init__.py +++ b/pybop/__init__.py @@ -105,7 +105,6 @@ from .optimisers.scipy_optimisers import SciPyMinimize, SciPyDifferentialEvolution from .optimisers.pints_optimisers import ( GradientDescent, - Adam, CMAES, IRPropMin, NelderMead, diff --git a/pybop/optimisers/pints_optimisers.py b/pybop/optimisers/pints_optimisers.py index 1ee289f1..24383667 100644 --- a/pybop/optimisers/pints_optimisers.py +++ b/pybop/optimisers/pints_optimisers.py @@ -32,36 +32,6 @@ def __init__(self, x0, sigma0=0.1, bounds=None): super().__init__(x0, sigma0, self.boundaries) -class Adam(pints.Adam): - """ - Implements the Adam optimization algorithm. - - This class extends the Adam optimiser from the PINTS library, which combines - ideas from RMSProp and Stochastic Gradient Descent with momentum. Note that - this optimiser does not support boundary constraints. - - Parameters - ---------- - x0 : array_like - Initial position from which optimization will start. - sigma0 : float, optional - Initial step size (default is 0.1). - bounds : dict, optional - Ignored by this optimiser, provided for API consistency. - - See Also - -------- - pints.Adam : The PINTS implementation this class is based on. - """ - - def __init__(self, x0, sigma0=0.1, bounds=None): - if bounds is not None: - print("NOTE: Boundaries ignored by Adam") - - self.boundaries = None # Bounds ignored in pints.Adam - super().__init__(x0, sigma0, self.boundaries) - - class IRPropMin(pints.IRPropMin): """ Implements the iRpropMin optimization algorithm. diff --git a/tests/integration/test_parameterisations.py b/tests/integration/test_parameterisations.py index 76b1defd..e59deb3a 100644 --- a/tests/integration/test_parameterisations.py +++ b/tests/integration/test_parameterisations.py @@ -82,7 +82,6 @@ def spm_costs(self, model, parameters, cost_class, init_soc): [ pybop.SciPyMinimize, pybop.SciPyDifferentialEvolution, - pybop.Adam, pybop.AdamW, pybop.CMAES, pybop.GradientDescent, diff --git a/tests/unit/test_optimisation.py b/tests/unit/test_optimisation.py index 07b4c56e..cd8af1d9 100644 --- a/tests/unit/test_optimisation.py +++ b/tests/unit/test_optimisation.py @@ -74,7 +74,6 @@ def two_param_cost(self, model, two_parameters, dataset): (pybop.SciPyMinimize, "SciPyMinimize"), (pybop.SciPyDifferentialEvolution, "SciPyDifferentialEvolution"), (pybop.GradientDescent, "Gradient descent"), - (pybop.Adam, "Adam"), (pybop.AdamW, "AdamW"), (pybop.CMAES, "Covariance Matrix Adaptation Evolution Strategy (CMA-ES)"), (pybop.SNES, "Seperable Natural Evolution Strategy (SNES)"), From ccc4ddbd97fa2e72153bc2c0b2e7e08b4fac200a Mon Sep 17 00:00:00 2001 From: Brady Planden Date: Mon, 20 May 2024 09:04:27 +0100 Subject: [PATCH 05/11] fix: revert removal of Adam, adds depreciation warning instead --- examples/scripts/spm_AdamW.py | 2 +- pybop/__init__.py | 1 + pybop/optimisers/pints_optimisers.py | 37 ++++++++++++++++++++++++++++ tests/unit/test_optimisation.py | 1 + 4 files changed, 40 insertions(+), 1 deletion(-) diff --git a/examples/scripts/spm_AdamW.py b/examples/scripts/spm_AdamW.py index ce96e1f7..e5df4d0c 100644 --- a/examples/scripts/spm_AdamW.py +++ b/examples/scripts/spm_AdamW.py @@ -56,7 +56,7 @@ def noise(sigma): cost = pybop.RootMeanSquaredError(problem) optim = pybop.Optimisation( cost, - optimiser=pybop.AdamW, + optimiser=pybop.Adam, verbose=True, allow_infeasible_solutions=True, sigma0=0.05, diff --git a/pybop/__init__.py b/pybop/__init__.py index 9c161f6f..bb635bc4 100644 --- a/pybop/__init__.py +++ b/pybop/__init__.py @@ -105,6 +105,7 @@ from .optimisers.scipy_optimisers import SciPyMinimize, SciPyDifferentialEvolution from .optimisers.pints_optimisers import ( GradientDescent, + Adam, CMAES, IRPropMin, NelderMead, diff --git a/pybop/optimisers/pints_optimisers.py b/pybop/optimisers/pints_optimisers.py index 24383667..28598ac8 100644 --- a/pybop/optimisers/pints_optimisers.py +++ b/pybop/optimisers/pints_optimisers.py @@ -1,3 +1,5 @@ +from warnings import warn + import numpy as np import pints @@ -65,6 +67,41 @@ def __init__(self, x0, sigma0=0.1, bounds=None): super().__init__(x0, sigma0, self.boundaries) +class Adam(pints.Adam): + """ + Implements the Adam optimization algorithm. + + This class extends the Adam optimiser from the PINTS library, which combines + ideas from RMSProp and Stochastic Gradient Descent with momentum. Note that + this optimiser does not support boundary constraints. + + Parameters + ---------- + x0 : array_like + Initial position from which optimization will start. + sigma0 : float, optional + Initial step size (default is 0.1). + bounds : dict, optional + Ignored by this optimiser, provided for API consistency. + + See Also + -------- + pints.Adam : The PINTS implementation this class is based on. + """ + + def __init__(self, x0, sigma0=0.1, bounds=None): + warn( + "Adam is deprecated and will be removed in a future release. Please use AdamW instead.", + DeprecationWarning, + stacklevel=2, + ) + if bounds is not None: + print("NOTE: Boundaries ignored by AdamW") + + self.boundaries = None # Bounds ignored in pints.Adam + super().__init__(x0, sigma0, self.boundaries) + + class PSO(pints.PSO): """ Implements a particle swarm optimization (PSO) algorithm. diff --git a/tests/unit/test_optimisation.py b/tests/unit/test_optimisation.py index cd8af1d9..07b4c56e 100644 --- a/tests/unit/test_optimisation.py +++ b/tests/unit/test_optimisation.py @@ -74,6 +74,7 @@ def two_param_cost(self, model, two_parameters, dataset): (pybop.SciPyMinimize, "SciPyMinimize"), (pybop.SciPyDifferentialEvolution, "SciPyDifferentialEvolution"), (pybop.GradientDescent, "Gradient descent"), + (pybop.Adam, "Adam"), (pybop.AdamW, "AdamW"), (pybop.CMAES, "Covariance Matrix Adaptation Evolution Strategy (CMA-ES)"), (pybop.SNES, "Seperable Natural Evolution Strategy (SNES)"), From faee7eb66e05498795f6c407eee927b57fba6a8e Mon Sep 17 00:00:00 2001 From: Brady Planden Date: Fri, 7 Jun 2024 11:20:34 +0100 Subject: [PATCH 06/11] tests: adds coverage, updates _AdamW for improved error catching --- pybop/optimisers/_adamw.py | 20 ++++------------- tests/unit/test_optimisation.py | 40 +++++++++++++++++++++++---------- 2 files changed, 32 insertions(+), 28 deletions(-) diff --git a/pybop/optimisers/_adamw.py b/pybop/optimisers/_adamw.py index 779da837..8e1b123e 100644 --- a/pybop/optimisers/_adamw.py +++ b/pybop/optimisers/_adamw.py @@ -123,18 +123,6 @@ def f_guessed(self): """ return self._current_f - def _log_init(self, logger): - """See :meth:`Loggable._log_init()`.""" - logger.add_float("b1") - logger.add_float("b2") - logger.add_float("lambda") - - def _log_write(self, logger): - """See :meth:`Loggable._log_write()`.""" - logger.log(self._b1t) - logger.log(self._b2t) - logger.log(self._lambda) - def name(self): """ Returns the name of the optimiser. @@ -152,7 +140,7 @@ def n_hyper_parameters(self): """ The number of hyper-parameters used by this optimiser. """ - return 1 + return 5 def running(self): """ @@ -229,7 +217,7 @@ def set_lambda(self, lambda_=0.01): raise TypeError("lambda_ must be numeric, floatable value.") if not 0 < lambda_ <= 1: - print("lambda_ must a positive value between 0 and 1") + raise ValueError("lambda_ must a positive value between 0 and 1") self._lambda = lambda_ return @@ -244,7 +232,7 @@ def set_b1(self, b1): raise TypeError("b1 must be numeric, floatable value.") if not 0 < b1 <= 1: - print("b1 must a positive value between 0 and 1") + raise ValueError("b1 must a positive value between 0 and 1") self._b1 = b1 return @@ -259,7 +247,7 @@ def set_b2(self, b2): raise TypeError("b2 must be numeric, floatable value.") if not 0 < b2 <= 1: - print("b2 must a positive value between 0 and 1") + raise ValueError("b2 must a positive value between 0 and 1") self._b2 = b2 return diff --git a/tests/unit/test_optimisation.py b/tests/unit/test_optimisation.py index 5e7b7c41..9ca8a49e 100644 --- a/tests/unit/test_optimisation.py +++ b/tests/unit/test_optimisation.py @@ -111,6 +111,7 @@ def test_optimiser_classes(self, two_param_cost, optimiser, expected_name): pybop.SciPyDifferentialEvolution, pybop.GradientDescent, pybop.Adam, + pybop.AdamW, pybop.SNES, pybop.XNES, pybop.PSO, @@ -232,21 +233,36 @@ def test_optimiser_kwargs(self, cost, optimiser): # Test AdamW hyperparameters if optimiser in [pybop.AdamW]: - optimiser.set_b1(0.9) - optimiser.set_b2(0.9) - optimiser.set_lambda(0.1) + optim = optimiser(cost=cost, b1=0.9, b2=0.999, lambda_=0.1) + optim.pints_optimiser.set_b1(0.9) + optim.pints_optimiser.set_b2(0.9) + optim.pints_optimiser.set_lambda(0.1) - assert optimiser._b1 == 0.9 - assert optimiser._b2 == 0.9 - assert optimiser._lambda == 0.1 + assert optim.pints_optimiser._b1 == 0.9 + assert optim.pints_optimiser._b2 == 0.9 + assert optim.pints_optimiser._lambda == 0.1 # Incorrect values - with pytest.raises(TypeError): - optimiser.set_b1("Value") - with pytest.raises(TypeError): - optimiser.set_b2("Value") - with pytest.raises(TypeError): - optimiser.set_lambda("Value") + for i, match in zip( + ("Value", -1), + ( + "must be numeric, floatable value.", + "must a positive value between 0 and 1", + ), + ): + with pytest.raises(Exception, match=match): + optim.pints_optimiser.set_b1(i) + with pytest.raises(Exception, match=match): + optim.pints_optimiser.set_b2(i) + with pytest.raises(Exception, match=match): + optim.pints_optimiser.set_lambda(i) + + # Check defaults + assert optim.pints_optimiser.n_hyper_parameters() == 5 + assert not optim.pints_optimiser.running() + assert optim.pints_optimiser.x_guessed() == optim.pints_optimiser._x0 + with pytest.raises(Exception): + optim.pints_optimiser.tell([0.1]) @pytest.mark.unit def test_single_parameter(self, cost): From 66ad9d133278dd2d4ff78b8421df4013d6ffbea3 Mon Sep 17 00:00:00 2001 From: Brady Planden Date: Fri, 7 Jun 2024 13:11:14 +0100 Subject: [PATCH 07/11] refactor: Updt _AdamW to AdamImpl, updt set methods, tests --- examples/scripts/spm_AdamW.py | 2 +- pybop/__init__.py | 2 +- pybop/optimisers/_adamw.py | 44 ++++++++-------------------- pybop/optimisers/pints_optimisers.py | 22 +++++++------- tests/unit/test_optimisation.py | 20 ++++++------- 5 files changed, 37 insertions(+), 53 deletions(-) diff --git a/examples/scripts/spm_AdamW.py b/examples/scripts/spm_AdamW.py index 82f884e2..10351512 100644 --- a/examples/scripts/spm_AdamW.py +++ b/examples/scripts/spm_AdamW.py @@ -54,7 +54,7 @@ def noise(sigma): model, parameters, dataset, signal=signal, init_soc=init_soc ) cost = pybop.RootMeanSquaredError(problem) -optim = pybop.Adam( +optim = pybop.AdamW( cost, verbose=True, allow_infeasible_solutions=True, diff --git a/pybop/__init__.py b/pybop/__init__.py index 84c34656..7ba35aeb 100644 --- a/pybop/__init__.py +++ b/pybop/__init__.py @@ -102,7 +102,7 @@ # # Optimiser class # -from .optimisers._adamw import _AdamW +from .optimisers._adamw import AdamWImpl from .optimisers.base_optimiser import BaseOptimiser from .optimisers.base_pints_optimiser import BasePintsOptimiser from .optimisers.scipy_optimisers import ( diff --git a/pybop/optimisers/_adamw.py b/pybop/optimisers/_adamw.py index 8e1b123e..49c758d9 100644 --- a/pybop/optimisers/_adamw.py +++ b/pybop/optimisers/_adamw.py @@ -6,7 +6,7 @@ from pints import Optimiser as PintsOptimiser -class _AdamW(PintsOptimiser): +class AdamWImpl(PintsOptimiser): """ AdamW optimiser (adaptive moment estimation with weight decay), as described in [1]_. @@ -206,48 +206,30 @@ def x_guessed(self): """ return self._current - def set_lambda(self, lambda_=0.01): + def set_lambda(self, lambda_: float = 0.01) -> None: """ Sets the lambda_ decay constant. This is the weight decay rate that helps in finding the optimal solution. """ - try: - lambda_ = float(lambda_) # Ensure b1 is a floatable - except Exception: - raise TypeError("lambda_ must be numeric, floatable value.") + if not isinstance(lambda_, (int, float)) or not 0 < lambda_ <= 1: + raise ValueError("lambda_ must be a numeric value between 0 and 1.") - if not 0 < lambda_ <= 1: - raise ValueError("lambda_ must a positive value between 0 and 1") + self._lambda = float(lambda_) - self._lambda = lambda_ - return - - def set_b1(self, b1): + def set_b1(self, b1: float) -> None: """ Sets the b1 momentum decay constant. """ - try: - b1 = float(b1) # Ensure b1 is a floatable - except Exception: - raise TypeError("b1 must be numeric, floatable value.") - - if not 0 < b1 <= 1: - raise ValueError("b1 must a positive value between 0 and 1") + if not isinstance(b1, (int, float)) or not 0 < b1 <= 1: + raise ValueError("b1 must be a numeric value between 0 and 1.") - self._b1 = b1 - return + self._b1 = float(b1) - def set_b2(self, b2): + def set_b2(self, b2: float) -> None: """ Sets the b2 momentum decay constant. """ - try: - b2 = float(b2) # Ensure b2 is a floatable - except Exception: - raise TypeError("b2 must be numeric, floatable value.") - - if not 0 < b2 <= 1: - raise ValueError("b2 must a positive value between 0 and 1") + if not isinstance(b2, (int, float)) or not 0 < b2 <= 1: + raise ValueError("b2 must be a numeric value between 0 and 1.") - self._b2 = b2 - return + self._b2 = float(b2) diff --git a/pybop/optimisers/pints_optimisers.py b/pybop/optimisers/pints_optimisers.py index 9278a80b..c0bb77b2 100644 --- a/pybop/optimisers/pints_optimisers.py +++ b/pybop/optimisers/pints_optimisers.py @@ -9,7 +9,7 @@ from pints import IRPropMin as PintsIRPropMin from pints import NelderMead as PintsNelderMead -from pybop import BasePintsOptimiser, _AdamW +from pybop import AdamWImpl, BasePintsOptimiser class GradientDescent(BasePintsOptimiser): @@ -74,27 +74,29 @@ def __init__(self, cost, **optimiser_kwargs): class AdamW(BasePintsOptimiser): """ - Adapter for the Cuckoo Search optimiser in PyBOP. - Cuckoo Search is a population-based optimisation algorithm inspired by the brood parasitism of some cuckoo species. - It is designed to be simple, efficient, and robust, and is suitable for global optimisation problems. + Implements the AdamW optimization algorithm in PyBOP. + + This class extends the AdamW optimiser, which is a variant of the Adam + optimiser that incorporates weight decay. AdamW is designed to be more + robust and stable for training deep neural networks, particularly when + using larger learning rates. + Parameters ---------- **optimiser_kwargs : optional Valid PyBOP option keys and their values, for example: x0 : array_like - Initial parameter values. + Initial position from which optimization will start. sigma0 : float Initial step size. - bounds : dict - A dictionary with 'lower' and 'upper' keys containing arrays for lower and - upper bounds on the parameters. + See Also -------- - pybop.CuckooSearch : PyBOP implementation of Cuckoo Search algorithm. + pybop.AdamWImpl : The PyBOP implementation this class is based on. """ def __init__(self, cost, **optimiser_kwargs): - super().__init__(cost, _AdamW, **optimiser_kwargs) + super().__init__(cost, AdamWImpl, **optimiser_kwargs) class IRPropMin(BasePintsOptimiser): diff --git a/tests/unit/test_optimisation.py b/tests/unit/test_optimisation.py index 9ca8a49e..b0e8d3f0 100644 --- a/tests/unit/test_optimisation.py +++ b/tests/unit/test_optimisation.py @@ -243,18 +243,18 @@ def test_optimiser_kwargs(self, cost, optimiser): assert optim.pints_optimiser._lambda == 0.1 # Incorrect values - for i, match in zip( - ("Value", -1), - ( - "must be numeric, floatable value.", - "must a positive value between 0 and 1", - ), - ): - with pytest.raises(Exception, match=match): + for i, match in (("Value", -1),): + with pytest.raises( + Exception, match="must be a numeric value between 0 and 1." + ): optim.pints_optimiser.set_b1(i) - with pytest.raises(Exception, match=match): + with pytest.raises( + Exception, match="must be a numeric value between 0 and 1." + ): optim.pints_optimiser.set_b2(i) - with pytest.raises(Exception, match=match): + with pytest.raises( + Exception, match="must be a numeric value between 0 and 1." + ): optim.pints_optimiser.set_lambda(i) # Check defaults From f3c1454ef4addb8cf484fc574e6552f89350fde0 Mon Sep 17 00:00:00 2001 From: Brady Planden <55357039+BradyPlanden@users.noreply.github.com> Date: Fri, 7 Jun 2024 13:15:11 +0100 Subject: [PATCH 08/11] Apply suggestions from code review --- CHANGELOG.md | 2 +- pybop/optimisers/_adamw.py | 8 ++++---- pybop/optimisers/pints_optimisers.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d508530b..3393f0c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Features -- [#316](https://github.com/pybop-team/PyBOP/pull/316) - Adds Adam with weight decay (AdamW) optimiser, replaces pints.Adam implementation. +- [#316](https://github.com/pybop-team/PyBOP/pull/316) - Adds Adam with weight decay (AdamW) optimiser, adds depreciation warning for pints.Adam implementation. - [#315](https://github.com/pybop-team/PyBOP/pull/315) - Updates __init__ structure to remove circular import issues and minimises dependancy imports across codebase for faster PyBOP module import. Adds type-hints to BaseModel and refactors rebuild parameter variables. - [#236](https://github.com/pybop-team/PyBOP/issues/236) - Restructures the optimiser classes, adds a new optimisation API through direct construction and keyword arguments, and fixes the setting of `max_iterations`, and `_minimising`. Introduces `pybop.BaseOptimiser`, `pybop.BasePintsOptimiser`, and `pybop.BaseSciPyOptimiser` classes. - [#321](https://github.com/pybop-team/PyBOP/pull/321) - Updates Prior classes with BaseClass, adds a `problem.sample_initial_conditions` method to improve stability of SciPy.Minimize optimiser. diff --git a/pybop/optimisers/_adamw.py b/pybop/optimisers/_adamw.py index 49c758d9..24e5ec98 100644 --- a/pybop/optimisers/_adamw.py +++ b/pybop/optimisers/_adamw.py @@ -1,5 +1,5 @@ # -# Reimplements the Pints' Adam Class with a Weight Decay addition +# Extends the Pints' Adam Class with a Weight Decay addition # import numpy as np @@ -10,8 +10,8 @@ class AdamWImpl(PintsOptimiser): """ AdamW optimiser (adaptive moment estimation with weight decay), as described in [1]_. - This method is an extension of the Adam optimizer that introduces weight decay, - which helps to regularize the weights and prevent overfitting. + This method is an extension of the Adam optimiser that introduces weight decay, + which helps to regularise the weights and prevent overfitting. This class reimplements the Pints' Adam Optimiser, but with the weight decay functionality mentioned above. Original creation and credit is attributed to Pints. @@ -33,7 +33,7 @@ class AdamWImpl(PintsOptimiser): ``beta1 = 0.9`` and ``beta2 = 0.999``. The terms ``m_j'`` and ``v_j'`` are "initialisation bias corrected" - versions of ``m_j`` and ``v_j`` (see section 3 of the paper). + versions of ``m_j`` and ``v_j`` (see section 2 of the paper). The parameter ``alpha`` is a step size, which is set as ``min(sigma0)`` in this implementation. diff --git a/pybop/optimisers/pints_optimisers.py b/pybop/optimisers/pints_optimisers.py index c0bb77b2..4872973a 100644 --- a/pybop/optimisers/pints_optimisers.py +++ b/pybop/optimisers/pints_optimisers.py @@ -74,7 +74,7 @@ def __init__(self, cost, **optimiser_kwargs): class AdamW(BasePintsOptimiser): """ - Implements the AdamW optimization algorithm in PyBOP. + Implements the AdamW optimisation algorithm in PyBOP. This class extends the AdamW optimiser, which is a variant of the Adam optimiser that incorporates weight decay. AdamW is designed to be more @@ -86,7 +86,7 @@ class AdamW(BasePintsOptimiser): **optimiser_kwargs : optional Valid PyBOP option keys and their values, for example: x0 : array_like - Initial position from which optimization will start. + Initial position from which optimisation will start. sigma0 : float Initial step size. From 9d57d2cf2792587d4ad14029237d4c0af182cb4b Mon Sep 17 00:00:00 2001 From: Brady Planden Date: Fri, 7 Jun 2024 13:15:54 +0100 Subject: [PATCH 09/11] remove old file artifact from merge develop --- pybop/_optimisation.py | 532 ----------------------------------------- 1 file changed, 532 deletions(-) delete mode 100644 pybop/_optimisation.py diff --git a/pybop/_optimisation.py b/pybop/_optimisation.py deleted file mode 100644 index cd7f811d..00000000 --- a/pybop/_optimisation.py +++ /dev/null @@ -1,532 +0,0 @@ -import warnings - -import numpy as np -import pints - -import pybop - - -class Optimisation: - """ - A class for conducting optimization using PyBOP or PINTS optimisers. - - Parameters - ---------- - cost : pybop.BaseCost or pints.ErrorMeasure - An objective function to be optimized, which can be either a pybop.Cost or PINTS error measure - optimiser : pybop.Optimiser or subclass of pybop.BaseOptimiser, optional - An optimiser from either the PINTS or PyBOP framework to perform the optimization (default: None). - sigma0 : float or sequence, optional - Initial step size or standard deviation for the optimiser (default: None). - verbose : bool, optional - If True, the optimization progress is printed (default: False). - physical_viability : bool, optional - If True, the feasibility of the optimised parameters is checked (default: True). - allow_infeasible_solutions : bool, optional - If True, infeasible parameter values will be allowed in the optimisation (default: True). - - Attributes - ---------- - x0 : numpy.ndarray - Initial parameter values for the optimization. - bounds : dict - Dictionary containing the parameter bounds with keys 'lower' and 'upper'. - _n_parameters : int - Number of parameters in the optimization problem. - sigma0 : float or sequence - Initial step size or standard deviation for the optimiser. - log : list - Log of the optimization process. - """ - - def __init__( - self, - cost, - x0=None, - optimiser=None, - sigma0=None, - verbose=False, - physical_viability=True, - allow_infeasible_solutions=True, - ): - self.cost = cost - self.x0 = x0 or cost.x0 - self.optimiser = optimiser - self.verbose = verbose - self.bounds = cost.bounds - self.sigma0 = sigma0 or cost.sigma0 - self._n_parameters = cost._n_parameters - self.physical_viability = physical_viability - self.allow_infeasible_solutions = allow_infeasible_solutions - self.log = [] - - # Convert x0 to pints vector - self._x0 = pints.vector(self.x0) - - # Set whether to allow infeasible locations - if self.cost.problem is not None and hasattr(self.cost.problem, "_model"): - self.cost.problem._model.allow_infeasible_solutions = ( - self.allow_infeasible_solutions - ) - else: - # Turn off this feature as there is no model - self.physical_viability = False - self.allow_infeasible_solutions = False - - # PyBOP doesn't currently support the pints transformation class - self._transformation = None - - # Check if minimising or maximising - if isinstance(cost, pybop.BaseLikelihood): - self.cost._minimising = False - self._minimising = self.cost._minimising - self._function = self.cost - - # Construct Optimiser - self.pints = True - - if self.optimiser is None: - self.optimiser = pybop.XNES - elif issubclass(self.optimiser, pints.Optimiser): - pass - else: - self.pints = False - - if issubclass( - self.optimiser, (pybop.SciPyMinimize, pybop.SciPyDifferentialEvolution) - ): - self.optimiser = self.optimiser(bounds=self.bounds) - - else: - raise ValueError("Unknown optimiser type") - - if self.pints: - self.optimiser = self.optimiser( - self.x0, sigma0=self.sigma0, bounds=self.bounds - ) - - # Check if sensitivities are required - self._needs_sensitivities = self.optimiser.needs_sensitivities() - - # Track optimiser's f_best or f_guessed - self._use_f_guessed = None - self.set_f_guessed_tracking() - - # Parallelisation - self._parallel = False - self._n_workers = 1 - self.set_parallel() - - # User callback - self._callback = None - - # Define stopping criteria - # Maximum iterations - self._max_iterations = None - self.set_max_iterations() - - # Minimum iterations - self._min_iterations = None - self.set_min_iterations() - - # Maximum unchanged iterations - self._unchanged_threshold = 1 # smallest significant f change - self._unchanged_max_iterations = None - self.set_max_unchanged_iterations() - - # Maximum evaluations - self._max_evaluations = None - - # Threshold value - self._threshold = None - - # Post-run statistics - self._evaluations = None - self._iterations = None - - def run(self): - """ - Run the optimization and return the optimized parameters and final cost. - - Returns - ------- - x : numpy.ndarray - The best parameter set found by the optimization. - final_cost : float - The final cost associated with the best parameters. - """ - - if self.pints: - x, final_cost = self._run_pints() - elif not self.pints: - x, final_cost = self._run_pybop() - - # Store the optimised parameters - if self.cost.problem is not None: - self.store_optimised_parameters(x) - - # Check if parameters are viable - if self.physical_viability: - self.check_optimal_parameters(x) - - return x, final_cost - - def _run_pybop(self): - """ - Internal method to run the optimization using a PyBOP optimiser. - - Returns - ------- - x : numpy.ndarray - The best parameter set found by the optimization. - final_cost : float - The final cost associated with the best parameters. - """ - self.result = self.optimiser.optimise( - cost_function=self.cost, - x0=self.x0, - maxiter=self._max_iterations, - ) - self.log = self.optimiser.log - self._iterations = self.result.nit - - return self.result.x, self.cost(self.result.x) - - def _run_pints(self): - """ - Internal method to run the optimization using a PINTS optimiser. - - Returns - ------- - x : numpy.ndarray - The best parameter set found by the optimization. - final_cost : float - The final cost associated with the best parameters. - - See Also - -------- - This method is heavily based on the run method in the PINTS.OptimisationController class. - """ - - # Check stopping criteria - has_stopping_criterion = False - has_stopping_criterion |= self._max_iterations is not None - has_stopping_criterion |= self._unchanged_max_iterations is not None - has_stopping_criterion |= self._max_evaluations is not None - has_stopping_criterion |= self._threshold is not None - if not has_stopping_criterion: - raise ValueError("At least one stopping criterion must be set.") - - # Iterations and function evaluations - iteration = 0 - evaluations = 0 - - # Unchanged iterations counter - unchanged_iterations = 0 - - # Choose method to evaluate - f = self._function - if self._needs_sensitivities: - f = f.evaluateS1 - - # Create evaluator object - if self._parallel: - # Get number of workers - n_workers = self._n_workers - - # For population based optimisers, don't use more workers than - # particles! - if isinstance(self.optimiser, pints.PopulationBasedOptimiser): - n_workers = min(n_workers, self.optimiser.population_size()) - evaluator = pints.ParallelEvaluator(f, n_workers=n_workers) - else: - evaluator = pints.SequentialEvaluator(f) - - # Keep track of current best and best-guess scores. - fb = fg = np.inf - - # Internally we always minimise! Keep a 2nd value to show the user. - fg_user = (fb, fg) if self._minimising else (-fb, -fg) - - # Keep track of the last significant change - f_sig = np.inf - - # Run the ask-and-tell loop - running = True - try: - while running: - # Ask optimiser for new points - xs = self.optimiser.ask() - - # Evaluate points - fs = evaluator.evaluate(xs) - - # Tell optimiser about function values - self.optimiser.tell(fs) - - # Update the scores - fb = self.optimiser.f_best() - fg = self.optimiser.f_guessed() - fg_user = (fb, fg) if self._minimising else (-fb, -fg) - - # Check for significant changes - f_new = fg if self._use_f_guessed else fb - if np.abs(f_new - f_sig) >= self._unchanged_threshold: - unchanged_iterations = 0 - f_sig = f_new - else: - unchanged_iterations += 1 - - # Update counts - evaluations += len(fs) - iteration += 1 - self.log.append(xs) - - # Check stopping criteria: - # Maximum number of iterations - if ( - self._max_iterations is not None - and iteration >= self._max_iterations - ): - running = False - halt_message = ( - "Maximum number of iterations (" + str(iteration) + ") reached." - ) - - # Maximum number of iterations without significant change - halt = ( - self._unchanged_max_iterations is not None - and unchanged_iterations >= self._unchanged_max_iterations - and iteration >= self._min_iterations - ) - if running and halt: - running = False - halt_message = ( - "No significant change for " - + str(unchanged_iterations) - + " iterations." - ) - - # Maximum number of evaluations - if ( - self._max_evaluations is not None - and evaluations >= self._max_evaluations - ): - running = False - halt_message = ( - "Maximum number of evaluations (" - + str(self._max_evaluations) - + ") reached." - ) - - # Threshold value - halt = self._threshold is not None and f_new < self._threshold - if running and halt: - running = False - halt_message = ( - "Objective function crossed threshold: " - + str(self._threshold) - + "." - ) - - # Error in optimiser - error = self.optimiser.stop() - if error: - running = False - halt_message = str(error) - - elif self._callback is not None: - self._callback(iteration - 1, self.optimiser) - - except (Exception, SystemExit, KeyboardInterrupt): - # Show last result and exit - print("\n" + "-" * 40) - print("Unexpected termination.") - print("Current score: " + str(fg_user)) - print("Current position:") - - # Show current parameters - x_user = self.optimiser.x_guessed() - if self._transformation is not None: - x_user = self._transformation.to_model(x_user) - for p in x_user: - print(pints.strfloat(p)) - print("-" * 40) - raise - - if self.verbose: - print("Halt: " + halt_message) - - # Save post-run statistics - self._evaluations = evaluations - self._iterations = iteration - - # Get best parameters - if self._use_f_guessed: - x = self.optimiser.x_guessed() - f = self.optimiser.f_guessed() - else: - x = self.optimiser.x_best() - f = self.optimiser.f_best() - - # Inverse transform search parameters - if self._transformation is not None: - x = self._transformation.to_model(x) - - # Store the optimised parameters - self.store_optimised_parameters(x) - - # Return best position and the score used internally, - # i.e the negative log-likelihood in the case of - # self._minimising = False - return x, f - - def f_guessed_tracking(self): - """ - Check if f_guessed instead of f_best is being tracked. - Credit: PINTS - - Returns - ------- - bool - True if f_guessed is being tracked, False otherwise. - """ - return self._use_f_guessed - - def set_f_guessed_tracking(self, use_f_guessed=False): - """ - Set the method used to track the optimiser progress. - Credit: PINTS - - Parameters - ---------- - use_f_guessed : bool, optional - If True, track f_guessed; otherwise, track f_best (default: False). - """ - self._use_f_guessed = bool(use_f_guessed) - - def set_max_evaluations(self, evaluations=None): - """ - Set a maximum number of evaluations stopping criterion. - Credit: PINTS - - Parameters - ---------- - evaluations : int, optional - The maximum number of evaluations after which to stop the optimization (default: None). - """ - if evaluations is not None: - evaluations = int(evaluations) - if evaluations < 0: - raise ValueError("Maximum number of evaluations cannot be negative.") - self._max_evaluations = evaluations - - def set_parallel(self, parallel=False): - """ - Enable or disable parallel evaluation. - Credit: PINTS - - Parameters - ---------- - parallel : bool or int, optional - If True, use as many worker processes as there are CPU cores. If an integer, use that many workers. - If False or 0, disable parallelism (default: False). - """ - if parallel is True: - self._parallel = True - self._n_workers = pints.ParallelEvaluator.cpu_count() - elif parallel >= 1: - self._parallel = True - self._n_workers = int(parallel) - else: - self._parallel = False - self._n_workers = 1 - - def set_max_iterations(self, iterations=1000): - """ - Set the maximum number of iterations as a stopping criterion. - Credit: PINTS - - Parameters - ---------- - iterations : int, optional - The maximum number of iterations to run (default is 1000). - Set to `None` to remove this stopping criterion. - """ - if iterations is not None: - iterations = int(iterations) - if iterations < 0: - raise ValueError("Maximum number of iterations cannot be negative.") - self._max_iterations = iterations - - def set_min_iterations(self, iterations=2): - """ - Set the minimum number of iterations as a stopping criterion. - - Parameters - ---------- - iterations : int, optional - The minimum number of iterations to run (default is 100). - Set to `None` to remove this stopping criterion. - """ - if iterations is not None: - iterations = int(iterations) - if iterations < 0: - raise ValueError("Minimum number of iterations cannot be negative.") - self._min_iterations = iterations - - def set_max_unchanged_iterations(self, iterations=15, threshold=1e-5): - """ - Set the maximum number of iterations without significant change as a stopping criterion. - Credit: PINTS - - Parameters - ---------- - iterations : int, optional - The maximum number of unchanged iterations to run (default is 15). - Set to `None` to remove this stopping criterion. - threshold : float, optional - The minimum significant change in the objective function value that resets the unchanged iteration counter (default is 1e-5). - """ - if iterations is not None: - iterations = int(iterations) - if iterations < 0: - raise ValueError("Maximum number of iterations cannot be negative.") - - threshold = float(threshold) - if threshold < 0: - raise ValueError("Minimum significant change cannot be negative.") - - self._unchanged_max_iterations = iterations - self._unchanged_threshold = threshold - - def store_optimised_parameters(self, x): - """ - Update the problem parameters with optimized values. - - The optimized parameter values are stored within the associated PyBOP parameter class. - - Parameters - ---------- - x : array-like - Optimized parameter values. - """ - for i, param in enumerate(self.cost.parameters): - param.update(value=x[i]) - - def check_optimal_parameters(self, x): - """ - Check if the optimised parameters are physically viable. - """ - - if self.cost.problem._model.check_params( - inputs=x, allow_infeasible_solutions=False - ): - return - else: - warnings.warn( - "Optimised parameters are not physically viable! \nConsider retrying the optimisation" - + " with a non-gradient-based optimiser and the option allow_infeasible_solutions=False", - UserWarning, - stacklevel=2, - ) From baf61c5c18185aa3a4514545cbbff54c6d3fa1bc Mon Sep 17 00:00:00 2001 From: Brady Planden Date: Sat, 8 Jun 2024 12:42:26 +0100 Subject: [PATCH 10/11] refactor: updt. readme methods table --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 6ff0af9f..8fd09c0a 100644 --- a/README.md +++ b/README.md @@ -85,13 +85,13 @@ The table below lists the currently supported [models](https://github.com/pybop- |-----------------------------------------------|-------------------------------------------------------------|------------------------------------------| | Single Particle Model (SPM) | Covariance Matrix Adaptation Evolution Strategy (CMA-ES) | Sum of Squared Errors (SSE) | | Single Particle Model with Electrolyte (SPMe) | Particle Swarm Optimization (PSO) | Root Mean Squared Error (RMSE) | -| Doyle-Fuller-Newman (DFN) | Adaptive Moment Estimation (Adam) | Maximum Likelihood Estimation (MLE) | -| Many Particle Model (MPM) | Improved Resilient Backpropagation (iRProp-) | Maximum a Posteriori (MAP) | -| Multi-Species Multi-Reactants (MSMR) | Exponential Natural Evolution Strategy (xNES) | Unscented Kalman Filter (UKF) | -| Equivalent Circuit Models (ECM) | Separable Natural Evolution Strategy (sNES) | Gravimetric Energy Density | -| | Gradient Descent | Volumetric Energy Density | +| Doyle-Fuller-Newman (DFN) | Exponential Natural Evolution Strategy (xNES) | Gaussian Log Likelihood | +| Many Particle Model (MPM) | Separable Natural Evolution Strategy (sNES) | Gaussian Log Likelihood w/ known variance | +| Multi-Species Multi-Reactants (MSMR) | Adaptive Moment Estimation with Weight Decay (AdamW) | Maximum a Posteriori (MAP) | +| Equivalent Circuit Models (ECM) | Improved Resilient Backpropagation (iRProp-) | Unscented Kalman Filter (UKF) | +| | SciPy Minimize & Differential Evolution | Gravimetric Energy Density | +| | Gradient Descent| Volumetric Energy Density | | | Nelder-Mead | | -| | SciPy Minimize & Differential Evolution | |

From fc6f1c9b7cf813fa8ea8b46e188c4ba0361b179a Mon Sep 17 00:00:00 2001 From: Brady Planden Date: Mon, 10 Jun 2024 09:02:34 +0100 Subject: [PATCH 11/11] fix: AdaMW unit test location --- tests/unit/test_optimisation.py | 42 ++++++++++++++++----------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/tests/unit/test_optimisation.py b/tests/unit/test_optimisation.py index b1430534..97fe12fc 100644 --- a/tests/unit/test_optimisation.py +++ b/tests/unit/test_optimisation.py @@ -212,27 +212,6 @@ def test_optimiser_kwargs(self, cost, optimiser): ): optimiser(cost=cost, bounds={"upper": [np.inf], "lower": [0.57]}) - else: - # Check and update initial values - assert optim.x0 == cost.x0 - x0_new = np.array([0.6]) - optim = optimiser(cost=cost, x0=x0_new) - assert optim.x0 == x0_new - assert optim.x0 != cost.x0 - - @pytest.mark.unit - def test_scipy_minimize_with_jac(self, cost): - # Check a method that uses gradient information - optim = pybop.SciPyMinimize(cost=cost, method="L-BFGS-B", jac=True, maxiter=10) - optim.run() - assert optim.result.scipy_result.success is True - - with pytest.raises( - ValueError, - match="Expected the jac option to be either True, False or None.", - ): - optim = pybop.SciPyMinimize(cost=cost, jac="Invalid string") - # Test AdamW hyperparameters if optimiser in [pybop.AdamW]: optim = optimiser(cost=cost, b1=0.9, b2=0.999, lambda_=0.1) @@ -266,6 +245,27 @@ def test_scipy_minimize_with_jac(self, cost): with pytest.raises(Exception): optim.pints_optimiser.tell([0.1]) + else: + # Check and update initial values + assert optim.x0 == cost.x0 + x0_new = np.array([0.6]) + optim = optimiser(cost=cost, x0=x0_new) + assert optim.x0 == x0_new + assert optim.x0 != cost.x0 + + @pytest.mark.unit + def test_scipy_minimize_with_jac(self, cost): + # Check a method that uses gradient information + optim = pybop.SciPyMinimize(cost=cost, method="L-BFGS-B", jac=True, maxiter=10) + optim.run() + assert optim.result.scipy_result.success is True + + with pytest.raises( + ValueError, + match="Expected the jac option to be either True, False or None.", + ): + optim = pybop.SciPyMinimize(cost=cost, jac="Invalid string") + @pytest.mark.unit def test_single_parameter(self, cost): # Test catch for optimisers that can only run with multiple parameters