From b112c0c90720f018ec786d7a70af41990be79a57 Mon Sep 17 00:00:00 2001 From: c-bata Date: Tue, 9 Mar 2021 10:44:03 +0900 Subject: [PATCH 1/5] Add hypothesis testing --- .gitignore | 1 + tests/test_fuzzing.py | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 tests/test_fuzzing.py diff --git a/.gitignore b/.gitignore index 73fb1b2..c9d38a9 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ __pycache__/ .eggs/ *.egg-info/ .tox/ +.hypothesis tmp/ benchmark/*.json diff --git a/tests/test_fuzzing.py b/tests/test_fuzzing.py new file mode 100644 index 0000000..16c8779 --- /dev/null +++ b/tests/test_fuzzing.py @@ -0,0 +1,26 @@ +import hypothesis.extra.numpy as npst +import unittest +from hypothesis import given, strategies as st + +from cmaes import CMA + + +class TestFuzzing(unittest.TestCase): + @given( + data=st.data(), + ) + def test_cma_tell(self, data): + dim = data.draw(st.integers(min_value=2, max_value=100)) + mean = data.draw(npst.arrays(dtype=float, shape=dim)) + sigma = data.draw(st.floats(min_value=1e-16)) + n_iterations = data.draw(st.integers(min_value=1)) + optimizer = CMA(mean, sigma) + popsize = optimizer.population_size + for _ in range(n_iterations): + tell_solutions = data.draw( + st.lists(st.tuples(npst.arrays(dtype=float, shape=dim), st.floats()), + min_size=popsize, max_size=popsize) + ) + optimizer.ask() + optimizer.tell(tell_solutions) + optimizer.ask() From 3c1dd0d0190745b400e35c6430e97f0c814e7974 Mon Sep 17 00:00:00 2001 From: c-bata Date: Sun, 11 Apr 2021 01:07:05 +0900 Subject: [PATCH 2/5] Add assertion for too large elements of mean vector --- cmaes/_cma.py | 5 +++++ tests/test_fuzzing.py | 12 +++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/cmaes/_cma.py b/cmaes/_cma.py index a8afb3c..35d051c 100644 --- a/cmaes/_cma.py +++ b/cmaes/_cma.py @@ -10,6 +10,7 @@ _EPS = 1e-8 +_MEAN_MAX = 1e32 _SIGMA_MAX = 1e32 @@ -78,6 +79,10 @@ def __init__( ): assert sigma > 0, "sigma must be non-zero positive value" + assert np.all( + mean < _MEAN_MAX + ), f"All elements of mean vector must be less than {_MEAN_MAX} to avoid overflow errors" + n_dim = len(mean) assert n_dim > 1, "The dimension of mean must be larger than 1" diff --git a/tests/test_fuzzing.py b/tests/test_fuzzing.py index 16c8779..3129e71 100644 --- a/tests/test_fuzzing.py +++ b/tests/test_fuzzing.py @@ -14,12 +14,18 @@ def test_cma_tell(self, data): mean = data.draw(npst.arrays(dtype=float, shape=dim)) sigma = data.draw(st.floats(min_value=1e-16)) n_iterations = data.draw(st.integers(min_value=1)) - optimizer = CMA(mean, sigma) + try: + optimizer = CMA(mean, sigma) + except AssertionError: + return popsize = optimizer.population_size for _ in range(n_iterations): tell_solutions = data.draw( - st.lists(st.tuples(npst.arrays(dtype=float, shape=dim), st.floats()), - min_size=popsize, max_size=popsize) + st.lists( + st.tuples(npst.arrays(dtype=float, shape=dim), st.floats()), + min_size=popsize, + max_size=popsize, + ) ) optimizer.ask() optimizer.tell(tell_solutions) From 4c7b63271e0c82d738511b4d5e059106d8161272 Mon Sep 17 00:00:00 2001 From: c-bata Date: Sun, 11 Apr 2021 01:22:14 +0900 Subject: [PATCH 3/5] Add assertions to avoid overflow errors --- cmaes/_cma.py | 12 ++++++++---- tests/test_fuzzing.py | 5 ++++- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/cmaes/_cma.py b/cmaes/_cma.py index 35d051c..79c8b5d 100644 --- a/cmaes/_cma.py +++ b/cmaes/_cma.py @@ -80,8 +80,8 @@ def __init__( assert sigma > 0, "sigma must be non-zero positive value" assert np.all( - mean < _MEAN_MAX - ), f"All elements of mean vector must be less than {_MEAN_MAX} to avoid overflow errors" + np.abs(mean) < _MEAN_MAX + ), f"Abs of all elements of mean vector must be less than {_MEAN_MAX} to avoid overflow errors" n_dim = len(mean) assert n_dim > 1, "The dimension of mean must be larger than 1" @@ -285,8 +285,12 @@ def _repair_infeasible_params(self, param: np.ndarray) -> np.ndarray: def tell(self, solutions: List[Tuple[np.ndarray, float]]) -> None: """Tell evaluation values""" - if len(solutions) != self._popsize: - raise ValueError("Must tell popsize-length solutions.") + + assert len(solutions) == self._popsize, "Must tell popsize-length solutions." + for s in solutions: + assert np.all( + np.abs(s[0]) < _MEAN_MAX + ), f"Abs of all param values must be less than {_MEAN_MAX} to avoid overflow errors" self._g += 1 solutions.sort(key=lambda s: s[1]) diff --git a/tests/test_fuzzing.py b/tests/test_fuzzing.py index 3129e71..df11c91 100644 --- a/tests/test_fuzzing.py +++ b/tests/test_fuzzing.py @@ -28,5 +28,8 @@ def test_cma_tell(self, data): ) ) optimizer.ask() - optimizer.tell(tell_solutions) + try: + optimizer.tell(tell_solutions) + except AssertionError: + return optimizer.ask() From f0a7b18e7068c8f5785134a5946543de3d43cad2 Mon Sep 17 00:00:00 2001 From: c-bata Date: Sun, 11 Apr 2021 01:27:35 +0900 Subject: [PATCH 4/5] Update requirements --- .github/workflows/tests.yml | 2 +- requirements-dev.txt | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2907a9f..2cc1a8b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -37,6 +37,6 @@ jobs: architecture: x64 - name: Install dependencies run: | - python -m pip install --upgrade pip setuptools numpy + python -m pip install --upgrade pip setuptools numpy hypothesis pip install --progress-bar off . - run: python -m unittest diff --git a/requirements-dev.txt b/requirements-dev.txt index 327c36f..ef02e88 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,6 +5,9 @@ numpy>=1.20.0 matplotlib scipy +# tests +hypothesis + # lint mypy flake8 From bab8df2f178822e0f2218aac9cd7edf2e52459ed Mon Sep 17 00:00:00 2001 From: c-bata Date: Sun, 11 Apr 2021 01:30:54 +0900 Subject: [PATCH 5/5] Fix a flake8 error --- cmaes/_cma.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmaes/_cma.py b/cmaes/_cma.py index 79c8b5d..ec0fb2b 100644 --- a/cmaes/_cma.py +++ b/cmaes/_cma.py @@ -81,7 +81,7 @@ def __init__( assert np.all( np.abs(mean) < _MEAN_MAX - ), f"Abs of all elements of mean vector must be less than {_MEAN_MAX} to avoid overflow errors" + ), f"Abs of all elements of mean vector must be less than {_MEAN_MAX}" n_dim = len(mean) assert n_dim > 1, "The dimension of mean must be larger than 1"