From 670ebb2e1265d64981711feae327d4dadee9c8b4 Mon Sep 17 00:00:00 2001 From: Suvarsha Chennareddy Date: Mon, 2 Jan 2023 11:20:32 +0530 Subject: [PATCH 01/34] attempt to fix current cmaes inconsistencies --- include/ensmallen_bits/cmaes/cmaes.hpp | 57 +++--- include/ensmallen_bits/cmaes/cmaes_impl.hpp | 59 ++++--- .../boundary_box_constraint.hpp | 166 ++++++++++++++++++ .../empty_transformation.hpp | 45 +++++ 4 files changed, 276 insertions(+), 51 deletions(-) create mode 100644 include/ensmallen_bits/cmaes/transformation_policies/boundary_box_constraint.hpp create mode 100644 include/ensmallen_bits/cmaes/transformation_policies/empty_transformation.hpp diff --git a/include/ensmallen_bits/cmaes/cmaes.hpp b/include/ensmallen_bits/cmaes/cmaes.hpp index 98500a44e..c8db0e8d7 100644 --- a/include/ensmallen_bits/cmaes/cmaes.hpp +++ b/include/ensmallen_bits/cmaes/cmaes.hpp @@ -17,6 +17,8 @@ #include "full_selection.hpp" #include "random_selection.hpp" +#include "transformation_policies/empty_transformation.hpp" +#include "transformation_policies/boundary_box_constraint.hpp" namespace ens { @@ -46,8 +48,11 @@ namespace ens { * ensmallen website. * * @tparam SelectionPolicy The selection strategy used for the evaluation step. + * @tparam transformationPolicy The transformation strategy used to + * map cooridnates to the desired domain. */ -template +template> class CMAES { public: @@ -60,8 +65,8 @@ class CMAES * equal one pass over the dataset). * * @param lambda The population size (0 use the default size). - * @param lowerBound Lower bound of decision variables. - * @param upperBound Upper bound of decision variables. + * @param transformationPolicy Instantiated transformation policy used to + * map the cooridnates to the desired domain. * @param batchSize Batch size to use for the objective calculation. * @param maxIterations Maximum number of iterations allowed (0 means no * limit). @@ -70,8 +75,8 @@ class CMAES * objective. */ CMAES(const size_t lambda = 0, - const double lowerBound = -10, - const double upperBound = 10, + const TransformationPolicyType& + transformationPolicy = TransformationPolicyType(), const size_t batchSize = 32, const size_t maxIterations = 1000, const double tolerance = 1e-5, @@ -87,31 +92,23 @@ class CMAES * @tparam CallbackTypes Types of callback functions. * @param function Function to optimize. * @param iterate Starting point (will be modified). + * @param stepSize Starting sigma/step size (will be modified). * @param callbacks Callback functions. * @return Objective value of the final point. */ template - typename MatType::elem_type Optimize(SeparableFunctionType& function, - MatType& iterate, - CallbackTypes&&... callbacks); + typename MatType, + typename... CallbackTypes> + typename MatType::elem_type Optimize(SeparableFunctionType& function, + MatType& iterate, + double stepSize = 0.6, + CallbackTypes&&... callbacks); //! Get the population size. size_t PopulationSize() const { return lambda; } //! Modify the population size. size_t& PopulationSize() { return lambda; } - //! Get the lower bound of decision variables. - double LowerBound() const { return lowerBound; } - //! Modify the lower bound of decision variables. - double& LowerBound() { return lowerBound; } - - //! Get the upper bound of decision variables - double UpperBound() const { return upperBound; } - //! Modify the upper bound of decision variables - double& UpperBound() { return upperBound; } - //! Get the batch size. size_t BatchSize() const { return batchSize; } //! Modify the batch size. @@ -132,16 +129,17 @@ class CMAES //! Modify the selection policy. SelectionPolicyType& SelectionPolicy() { return selectionPolicy; } + //! Get the transformation policy. + const TransformationPolicyType& TransformationPolicy() const + { return transformationPolicy; } + //! Modify the transformation policy. + TransformationPolicyType& TransformationPolicy() + { return transformationPolicy; } + private: //! Population size. size_t lambda; - //! Lower bound of decision variables. - double lowerBound; - - //! Upper bound of decision variables - double upperBound; - //! The batch size for processing. size_t batchSize; @@ -153,13 +151,16 @@ class CMAES //! The selection policy used to calculate the objective. SelectionPolicyType selectionPolicy; + + TransformationPolicyType transformationPolicy; }; /** * Convenient typedef for CMAES approximation. */ -template -using ApproxCMAES = CMAES; +template, + typename SelectionPolicyType = RandomSelection> +using ApproxCMAES = CMAES; } // namespace ens diff --git a/include/ensmallen_bits/cmaes/cmaes_impl.hpp b/include/ensmallen_bits/cmaes/cmaes_impl.hpp index 90e047ae4..3b152b22c 100644 --- a/include/ensmallen_bits/cmaes/cmaes_impl.hpp +++ b/include/ensmallen_bits/cmaes/cmaes_impl.hpp @@ -22,31 +22,32 @@ namespace ens { -template -CMAES::CMAES(const size_t lambda, - const double lowerBound, - const double upperBound, +template +CMAES::CMAES(const size_t lambda, + const TransformationPolicyType& + transformationPolicy, const size_t batchSize, const size_t maxIterations, const double tolerance, const SelectionPolicyType& selectionPolicy) : lambda(lambda), - lowerBound(lowerBound), - upperBound(upperBound), batchSize(batchSize), maxIterations(maxIterations), tolerance(tolerance), - selectionPolicy(selectionPolicy) + selectionPolicy(selectionPolicy), + transformationPolicy(transformationPolicy) { /* Nothing to do. */ } //! Optimize the function (minimize). -template +template template -typename MatType::elem_type CMAES::Optimize( +typename MatType::elem_type CMAES::Optimize( SeparableFunctionType& function, MatType& iterateIn, + double stepSizeIn, CallbackTypes&&... callbacks) { // Convenience typedefs. @@ -78,7 +79,7 @@ typename MatType::elem_type CMAES::Optimize( // Step size control parameters. BaseMatType sigma(2, 1); // sigma is vector-shaped. - sigma(0) = 0.3 * (upperBound - lowerBound); + sigma(0) = stepSizeIn; //0.3 * (upperBound - lowerBound); const double cs = (muEffective + 2) / (iterate.n_elem + muEffective + 5); const double ds = 1 + cs + 2 * std::max(std::sqrt((muEffective - 1) / (iterate.n_elem + 1)) - 1, 0.0); @@ -99,8 +100,7 @@ typename MatType::elem_type CMAES::Optimize( std::vector mPosition(2, BaseMatType(iterate.n_rows, iterate.n_cols)); - mPosition[0] = lowerBound + arma::randu( - iterate.n_rows, iterate.n_cols) * (upperBound - lowerBound); + mPosition[0] = iterate; BaseMatType step(iterate.n_rows, iterate.n_cols); step.zeros(); @@ -110,11 +110,13 @@ typename MatType::elem_type CMAES::Optimize( for (size_t f = 0; f < numFunctions; f += batchSize) { const size_t effectiveBatchSize = std::min(batchSize, numFunctions - f); - const ElemType objective = function.Evaluate(mPosition[0], f, + const ElemType objective = + function.Evaluate(transformationPolicy.Transform(mPosition[0]), f, effectiveBatchSize); currentObjective += objective; - Callback::Evaluate(*this, function, mPosition[0], objective, + Callback::Evaluate(*this, function, + transformationPolicy.Transform(mPosition[0]), objective, callbacks...); } @@ -146,10 +148,12 @@ typename MatType::elem_type CMAES::Optimize( // Controls early termination of the optimization process. bool terminate = false; + BaseMatType transformedIterate = transformationPolicy.Transform(iterate); + // Now iterate! - terminate |= Callback::BeginOptimization(*this, function, iterate, - callbacks...); - for (size_t i = 1; i < maxIterations && !terminate; ++i) + terminate |= Callback::BeginOptimization(*this, function, + transformedIterate, callbacks...); + for (size_t i = 1; (i != maxIterations) && !terminate; ++i) { // To keep track of where we are. const size_t idx0 = (i - 1) % 2; @@ -161,6 +165,8 @@ typename MatType::elem_type CMAES::Optimize( while (!arma::chol(covLower, C[idx0], "lower")) C[idx0].diag() += std::numeric_limits::epsilon(); + arma::eig_sym(eigval, eigvec, C[idx0]); + for (size_t j = 0; j < lambda; ++j) { if (iterate.n_rows > iterate.n_cols) @@ -171,14 +177,14 @@ typename MatType::elem_type CMAES::Optimize( else { pStep[idx(j)] = arma::randn(iterate.n_rows, iterate.n_cols) - * covLower; + * covLower.t(); } pPosition[idx(j)] = mPosition[idx0] + sigma(idx0) * pStep[idx(j)]; // Calculate the objective function. pObjective(idx(j)) = selectionPolicy.Select(function, batchSize, - pPosition[idx(j)], callbacks...); + transformationPolicy.Transform(pPosition[idx(j)]), callbacks...); } // Sort population. @@ -192,7 +198,7 @@ typename MatType::elem_type CMAES::Optimize( // Calculate the objective function. currentObjective = selectionPolicy.Select(function, batchSize, - mPosition[idx1], callbacks...); + transformationPolicy.Transform(mPosition[idx1]), callbacks...); // Update best parameters. if (currentObjective < overallObjective) @@ -200,19 +206,23 @@ typename MatType::elem_type CMAES::Optimize( overallObjective = currentObjective; iterate = mPosition[idx1]; - terminate |= Callback::StepTaken(*this, function, iterate, callbacks...); + transformedIterate = transformationPolicy.Transform(iterate); + terminate |= Callback::StepTaken(*this, function, + transformedIterate, callbacks...); } // Update Step Size. if (iterate.n_rows > iterate.n_cols) { ps[idx1] = (1 - cs) * ps[idx0] + std::sqrt( - cs * (2 - cs) * muEffective) * covLower.t() * step; + cs * (2 - cs) * muEffective) * + eigvec * diagmat(1 / eigval) * eigvec.t() * step; } else { ps[idx1] = (1 - cs) * ps[idx0] + std::sqrt( - cs * (2 - cs) * muEffective) * step * covLower.t(); + cs * (2 - cs) * muEffective) * step * + eigvec * diagmat(1 / eigval) * eigvec.t(); } const ElemType psNorm = arma::norm(ps[idx1]); @@ -293,6 +303,7 @@ typename MatType::elem_type CMAES::Optimize( Warn << "CMA-ES: converged to " << overallObjective << "; " << "terminating with failure. Try a smaller step size?" << std::endl; + iterate = transformationPolicy.Transform(iterate); Callback::EndOptimization(*this, function, iterate, callbacks...); return overallObjective; } @@ -302,6 +313,7 @@ typename MatType::elem_type CMAES::Optimize( Info << "CMA-ES: minimized within tolerance " << tolerance << "; " << "terminating optimization." << std::endl; + iterate = transformationPolicy.Transform(iterate); Callback::EndOptimization(*this, function, iterate, callbacks...); return overallObjective; } @@ -309,6 +321,7 @@ typename MatType::elem_type CMAES::Optimize( lastObjective = overallObjective; } + iterate = transformationPolicy.Transform(iterate); Callback::EndOptimization(*this, function, iterate, callbacks...); return overallObjective; } diff --git a/include/ensmallen_bits/cmaes/transformation_policies/boundary_box_constraint.hpp b/include/ensmallen_bits/cmaes/transformation_policies/boundary_box_constraint.hpp new file mode 100644 index 000000000..299240575 --- /dev/null +++ b/include/ensmallen_bits/cmaes/transformation_policies/boundary_box_constraint.hpp @@ -0,0 +1,166 @@ +/** + * @file boundary_box_constraint.hpp + * @author Suvarsha Chennareddy + * + * Boundary Box Transformation. + * + * + * ensmallen is free software; you may redistribute it and/or modify it under + * the terms of the 3-clause BSD license. You should have received a copy of + * the 3-clause BSD license along with ensmallen. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ + +namespace ens { + /** + * More often than not, coordinates must be bounded by some constraints. + * In a particular case, the domain of a specific function is restricted + * by boundaries. + * The implemented transformation transforms given coordinates into a region + * bounded by the given lower and upper bounds (a box). First, the + * coordinates are shifted into a feasible preimage bounded by lower_bound - al + * and upper_bound + au where al and au and calculated internally. + * These shifted coordinates are then transformed into coordinates bounded by + * lower_bound and upper_bound. It is an identity transformation in between + * the lower and upper bounds. + * The inverse transformation transforms the coordinates back to the preimage. + * + * For more information, check the original implementation in C by N. Hansen: + * https://github.com/CMA-ES/c-cmaes/blob/master/src/boundary_transformation.c + * + * @tparam MatType The matrix type of the coordinates and bounds. + */ +template +class BoundaryBoxConstraint +{ +public: + /** + * Construct the boundary box constraint poilcy. + * + * @param lowerBound The lower bound of the coordinates. + * @param upperBound The upper bound of the coordinates. + */ + BoundaryBoxConstraint(const MatType& lowerBound, + const MatType& upperBound): + lowerBound(lowerBound), + upperBound(upperBound) + {} + + /** + * Map the given coordinates to the range + * [lowerBound, upperBound] + * + * @param x Given coordinates. + * @return Transformed coordinates. + */ + MatType Transform(const MatType& x) + { + typedef typename MatType::elem_type ElemType; + double diff, al, au, xlow, xup, r; + size_t Bi, Bj; + MatType y = x; + for (size_t i = 0; i < x.n_rows; i++) + { + Bi = (i < lowerBound.n_rows) ? i : (lowerBound.n_rows - 1); + for (size_t j = 0; j < x.n_cols; j++) + { + Bj = (j < lowerBound.n_cols) ? j : (lowerBound.n_cols - 1); + + diff = (upperBound(Bi, Bj) - lowerBound(Bi, Bj)) / 2.0; + al = std::min(diff, (1 + std::abs(lowerBound(Bi, Bj))) / 20.0); + au = std::min(diff, (1 + std::abs(upperBound(Bi, Bj))) / 20.0); + xlow = lowerBound(Bi, Bj) - 2 * al - diff; + xup = upperBound(Bi, Bj) + 2 * au + diff; + r = 2 * (2 * diff + al + au); + + // Shift y into feasible pre-image. + if (y(i, j) < xlow) + { + y(i,j) += (ElemType)(r * (1 + (int)(xlow - y(i, j)) / r)); + } + if (y(i, j) > xup) + { + y(i, j) -= (ElemType)(r * (1 + (int)(y(i, j) - xup) / r)); + } + if (y(i, j) < lowerBound(Bi, Bj) - al) + { + y(i, j) += (ElemType)(2 * (lowerBound(Bi, Bj) - al - y(i, j))); + } + if (y(i, j) > upperBound(Bi, Bj) + au) + { + y(i, j) -= (ElemType)(2 * (y(i, j) - upperBound(Bi, Bj) - au)); + } + + // Boundary transformation. + if (y(i, j) < lowerBound(Bi, Bj) + al) + { + y(i, j) = (ElemType)(lowerBound(Bi, Bj) + + (y(i, j) - (lowerBound(Bi, Bj) - al)) * + (y(i, j) - (lowerBound(Bi, Bj) - al)) / 4.0 / al); + } + else if (y(i,j) > upperBound(Bi,Bj) - au) + { + y(i, j) = (ElemType)(upperBound(Bi, Bj) - + (y(i, j) - (upperBound(Bi, Bj) + au)) * + (y(i, j) - (upperBound(Bi, Bj) + au)) / 4.0 / au); + } + } + } + + return y; + } + + /** + * Map the given coordinates back to the preimage, + * which is bounded by [lowerBound - al, upperBound + au]. + * + * @param y Given coordinates. + * @return Transformed coordinates. + */ + MatType Inverse(const MatType& y) + { + typedef typename MatType::elem_type ElemType; + double diff, al, au; + size_t Bi, Bj; + MatType x = y; + for (size_t i = 0; i < y.n_rows; i++) + { + Bi = (i < lowerBound.n_rows) ? i : (lowerBound.n_rows - 1); + for (size_t j = 0; j < y.n_cols; j++) + { + Bj = (j < lowerBound.n_cols) ? j : (lowerBound.n_cols - 1); + + diff = (upperBound(Bi, Bj) - lowerBound(Bi, Bj)) / 2.0; + al = std::min(diff, (1 + std::abs(lowerBound(Bi, Bj))) / 20.0); + au = std::min(diff, (1 + std::abs(upperBound(Bi, Bj))) / 20.0); + if (x(i, j) < lowerBound(Bi, Bj) + al) + { + x(i, j) = (ElemType)(lowerBound(Bi, Bj) - al) + 2 * + std::sqrt(std::abs(al * (x(i, j) - lowerBound(Bi, Bj)))); + } + else if (x(i, j) > upperBound(Bi, Bj) - au) + { + x(i, j) = (ElemType)(upperBound(Bi, Bj) + au) - 2 * + std::sqrt(std::abs(au * (upperBound(Bi, Bj) - x(i, j)))); + } + } + } + + return x; + } + //! Get the lower bound of decision variables. + MatType LowerBound() const { return lowerBound; } + //! Modify the lower bound of decision variables. + MatType& LowerBound() { return lowerBound; } + + //! Get the upper bound of decision variables. + MatType UpperBound() const { return upperBound; } + //! Modify the upper bound of decision variables. + MatType& UpperBound() { return upperBound; } + +private: + MatType upperBound; + MatType lowerBound; +}; + +} // namespace ens diff --git a/include/ensmallen_bits/cmaes/transformation_policies/empty_transformation.hpp b/include/ensmallen_bits/cmaes/transformation_policies/empty_transformation.hpp new file mode 100644 index 000000000..b2ca32c5b --- /dev/null +++ b/include/ensmallen_bits/cmaes/transformation_policies/empty_transformation.hpp @@ -0,0 +1,45 @@ +/** + * @file empty_transformation.hpp + * @author Suvarsha Chennareddy + * + * Empty Transformation, can also be called an Indentiy Transformation. + * + * + * ensmallen is free software; you may redistribute it and/or modify it under + * the terms of the 3-clause BSD license. You should have received a copy of + * the 3-clause BSD license along with ensmallen. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ + +namespace ens { +/** + * This is an empty transformation. As the name indicates, it does + * not do anything. It is essentially an identity + * transformation and is meant to be used when there are no + * sorts of constraints on the coordinates. + * + * @tparam MatType The matrix type of the coordinates. + */ +template +class EmptyTransformation +{ +public: + +/** +* Transforms coordinates to themselves (effectively no transformation). +* +* @param x Input coordinates. +* @return Transformed coordinates (the coordinates themselves). +*/ +MatType Transform(const MatType& x) { return x; } + +/** +* Inverse transformtion (effectively no transformation). +* +* @param y Input coordinates. +* @return Transformed coordinates (the coordinates themselves). +*/ +MatType Inverse(const MatType& y) { return y; } +}; + +} // namespace ens \ No newline at end of file From f28fd1800457b5a679098b02d9cbb7f727a8e1da Mon Sep 17 00:00:00 2001 From: Suvarsha Chennareddy Date: Mon, 2 Jan 2023 12:20:56 +0530 Subject: [PATCH 02/34] update tests --- tests/cmaes_test.cpp | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/tests/cmaes_test.cpp b/tests/cmaes_test.cpp index 6dccb4505..c41231d78 100644 --- a/tests/cmaes_test.cpp +++ b/tests/cmaes_test.cpp @@ -23,7 +23,11 @@ using namespace ens::test; */ TEST_CASE("CMAESLogisticRegressionTest", "[CMAESTest]") { - CMAES<> cmaes(0, -1, 1, 32, 200, 1e-3); + arma::mat lb(1, 1, arma::fill::value(-1)); + arma::mat ub(1, 1, arma::fill::value(1)); + BoundaryBoxConstraint b(lb, ub); + + CMAES cmaes(0, b, 32, 200, 1e-3); LogisticRegressionFunctionTest(cmaes, 0.003, 0.006, 5); } @@ -33,7 +37,11 @@ TEST_CASE("CMAESLogisticRegressionTest", "[CMAESTest]") */ TEST_CASE("ApproxCMAESLogisticRegressionTest", "[CMAESTest]") { - ApproxCMAES<> cmaes(0, -1, 1, 32, 200, 1e-3); + arma::mat lb(1, 1, arma::fill::value(-1)); + arma::mat ub(1, 1, arma::fill::value(1)); + BoundaryBoxConstraint b(lb, ub); + + ApproxCMAES> cmaes(0, b, 32, 200, 1e-3); LogisticRegressionFunctionTest(cmaes, 0.003, 0.006, 5); } @@ -43,7 +51,11 @@ TEST_CASE("ApproxCMAESLogisticRegressionTest", "[CMAESTest]") */ TEST_CASE("CMAESLogisticRegressionFMatTest", "[CMAESTest]") { - CMAES<> cmaes(0, -1, 1, 32, 200, 1e-3); + arma::fmat lb(1, 1, arma::fill::value(-1)); + arma::fmat ub(1, 1, arma::fill::value(1)); + BoundaryBoxConstraint b(lb, ub); + + CMAES cmaes(0, b, 32, 200, 1e-3); LogisticRegressionFunctionTest(cmaes, 0.01, 0.02, 5); } @@ -53,6 +65,10 @@ TEST_CASE("CMAESLogisticRegressionFMatTest", "[CMAESTest]") */ TEST_CASE("ApproxCMAESLogisticRegressionFMatTest", "[CMAESTest]") { - ApproxCMAES<> cmaes(0, -1, 1, 32, 200, 1e-3); + arma::fmat lb(1, 1, arma::fill::value(-1)); + arma::fmat ub(1, 1, arma::fill::value(1)); + BoundaryBoxConstraint b(lb, ub); + + ApproxCMAES> cmaes(0, b, 32, 200, 1e-3); LogisticRegressionFunctionTest(cmaes, 0.01, 0.02, 5); } From 39feebba1440d4d875e07625f56d1cde62bc35d8 Mon Sep 17 00:00:00 2001 From: Suvarsha Chennareddy Date: Tue, 17 Jan 2023 00:52:58 +0530 Subject: [PATCH 03/34] fix style, add deprecated constructor, and update tests --- include/ensmallen_bits/cmaes/cmaes.hpp | 53 +++++++- include/ensmallen_bits/cmaes/cmaes_impl.hpp | 69 +++++++++-- .../boundary_box_constraint.hpp | 114 ++++++++++++------ .../empty_transformation.hpp | 56 +++++---- tests/callbacks_test.cpp | 3 +- tests/cmaes_test.cpp | 43 ++++--- 6 files changed, 240 insertions(+), 98 deletions(-) diff --git a/include/ensmallen_bits/cmaes/cmaes.hpp b/include/ensmallen_bits/cmaes/cmaes.hpp index c8db0e8d7..735de3d54 100644 --- a/include/ensmallen_bits/cmaes/cmaes.hpp +++ b/include/ensmallen_bits/cmaes/cmaes.hpp @@ -48,8 +48,9 @@ namespace ens { * ensmallen website. * * @tparam SelectionPolicy The selection strategy used for the evaluation step. - * @tparam transformationPolicy The transformation strategy used to - * map cooridnates to the desired domain. + * @tparam TransformationPolicy The transformation strategy used to + * map decision variables to the desired domain during fitness evaluation + * and termination. Use EmptyTransformation if the domain isn't bounded. */ template> @@ -66,13 +67,14 @@ class CMAES * * @param lambda The population size (0 use the default size). * @param transformationPolicy Instantiated transformation policy used to - * map the cooridnates to the desired domain. + * map the coordinates to the desired domain. * @param batchSize Batch size to use for the objective calculation. * @param maxIterations Maximum number of iterations allowed (0 means no * limit). * @param tolerance Maximum absolute tolerance to terminate algorithm. * @param selectionPolicy Instantiated selection policy used to calculate the * objective. + * @param stepSize Starting sigma/step size (will be modified). */ CMAES(const size_t lambda = 0, const TransformationPolicyType& @@ -80,7 +82,37 @@ class CMAES const size_t batchSize = 32, const size_t maxIterations = 1000, const double tolerance = 1e-5, - const SelectionPolicyType& selectionPolicy = SelectionPolicyType()); + const SelectionPolicyType& selectionPolicy = SelectionPolicyType(), + double stepSize = 0); + + /** + * Construct the CMA-ES optimizer with the given function and parameters + * (including lower and upper bounds). The defaults here are not necessarily + * good for the given problem, so it is suggested that the values used be + * tailored to the task at hand. The maximum number of iterations refers to + * the maximum number of points that are processed (i.e., one iteration + * equals one point; one iteration does not equal one pass over the dataset). + * This constructor is deprecated. + * + * @param lambda The population size(0 use the default size). + * @param lowerBound Lower bound of decision variables. + * @param upperBound Upper bound of decision variables. + * @param batchSize Batch size to use for the objective calculation. + * @param maxIterations Maximum number of iterations allowed(0 means no + limit). + * @param tolerance Maximum absolute tolerance to terminate algorithm. + * @param selectionPolicy Instantiated selection policy used to calculate the + * objective. + * @param stepSize Starting sigma/step size (will be modified). + */ + CMAES(const size_t lambda = 0, + const double lowerBound = -10, + const double upperBound = 10, + const size_t batchSize = 32, + const size_t maxIterations = 1000, + const double tolerance = 1e-5, + const SelectionPolicyType & selectionPolicy = SelectionPolicyType(), + double stepSize = 1); /** * Optimize the given function using CMA-ES. The given starting point will be @@ -92,7 +124,6 @@ class CMAES * @tparam CallbackTypes Types of callback functions. * @param function Function to optimize. * @param iterate Starting point (will be modified). - * @param stepSize Starting sigma/step size (will be modified). * @param callbacks Callback functions. * @return Objective value of the final point. */ @@ -101,7 +132,6 @@ class CMAES typename... CallbackTypes> typename MatType::elem_type Optimize(SeparableFunctionType& function, MatType& iterate, - double stepSize = 0.6, CallbackTypes&&... callbacks); //! Get the population size. @@ -136,6 +166,13 @@ class CMAES TransformationPolicyType& TransformationPolicy() { return transformationPolicy; } + //! Get the step size. + double StepSize() const + { return stepSize; } + //! Modify the step size. + double& StepSize() + { return stepSize; } + private: //! Population size. size_t lambda; @@ -152,7 +189,11 @@ class CMAES //! The selection policy used to calculate the objective. SelectionPolicyType selectionPolicy; + //! The transformationPolicy used to map coordinates to the suitable domain + //! while evaluating fitness. TransformationPolicyType transformationPolicy; + + double stepSize; }; /** diff --git a/include/ensmallen_bits/cmaes/cmaes_impl.hpp b/include/ensmallen_bits/cmaes/cmaes_impl.hpp index 3b152b22c..cf31b7f44 100644 --- a/include/ensmallen_bits/cmaes/cmaes_impl.hpp +++ b/include/ensmallen_bits/cmaes/cmaes_impl.hpp @@ -20,6 +20,25 @@ #include +/* + This partial specialization is used to throw an exception when the + TransformationPolicyType is EmptyTransformation and call a + constructor with paramters 'lowerBound' and 'upperBound' otherwise. +*/ +template +struct notEmptyTransformation : std::true_type { + void assign(T1& obj, double lowerBound, double upperBound) { + obj = T1(lowerBound, upperBound); + } +}; + +template typename T, typename... A, typename... B> +struct notEmptyTransformation, T> : std::false_type { + void assign(T& obj, double lowerBound, double upperBound) { + throw std::logic_error("TransformationPolicyType is EmptyTransformation"); + } +}; + namespace ens { template @@ -29,15 +48,40 @@ CMAES::CMAES(const size_t lambda, const size_t batchSize, const size_t maxIterations, const double tolerance, - const SelectionPolicyType& selectionPolicy) : + const SelectionPolicyType& selectionPolicy, + double stepSizeIn) : lambda(lambda), batchSize(batchSize), maxIterations(maxIterations), tolerance(tolerance), selectionPolicy(selectionPolicy), - transformationPolicy(transformationPolicy) + transformationPolicy(transformationPolicy), + stepSize(stepSizeIn) { /* Nothing to do. */ } +template +CMAES::CMAES(const size_t lambda, + const double lowerBound, + const double upperBound, + const size_t batchSize, + const size_t maxIterations, + const double tolerance, + const SelectionPolicyType& selectionPolicy, + double stepSizeIn) : + lambda(lambda), + batchSize(batchSize), + maxIterations(maxIterations), + tolerance(tolerance), + selectionPolicy(selectionPolicy), + stepSize(stepSizeIn) +{ + Warn << "This is a deprecated constructor and will be removed in a " + "future version of ensmallen" << std::endl; + notEmptyTransformation> d; + d.assign(transformationPolicy, lowerBound, upperBound); +} + + //! Optimize the function (minimize). template template::Optimize( SeparableFunctionType& function, MatType& iterateIn, - double stepSizeIn, CallbackTypes&&... callbacks) { // Convenience typedefs. @@ -79,7 +122,9 @@ typename MatType::elem_type CMAES class BoundaryBoxConstraint { public: + /** - * Construct the boundary box constraint poilcy. - * - * @param lowerBound The lower bound of the coordinates. - * @param upperBound The upper bound of the coordinates. - */ + * Construct the boundary box constraint policy. + */ + BoundaryBoxConstraint() + { /* Nothing to do. */ } + + /** + * Construct the boundary box constraint policy. + * + * @param lowerBound The lower bound of the coordinates. + * @param upperBound The upper bound of the coordinates. + */ BoundaryBoxConstraint(const MatType& lowerBound, - const MatType& upperBound): - lowerBound(lowerBound), - upperBound(upperBound) - {} + const MatType& upperBound) : + lowerBound(lowerBound), + upperBound(upperBound) + { + + } /** - * Map the given coordinates to the range - * [lowerBound, upperBound] - * - * @param x Given coordinates. - * @return Transformed coordinates. - */ + * Construct the boundary box constraint policy. + * + * @param lowerBound The lower bound (for every dimension) of the coordinates. + * @param upperBound The upper bound (for every dimension) of the coordinates. + */ + BoundaryBoxConstraint(const double lowerBound, + const double upperBound) : + lowerBound({ (typename MatType::elem_type) lowerBound }), + upperBound({ (typename MatType::elem_type) upperBound }) + {} + + /** + * Map the given coordinates to the range + * [lowerBound, upperBound] + * + * @param x Given coordinates. + * @return Transformed coordinates. + */ MatType Transform(const MatType& x) { typedef typename MatType::elem_type ElemType; @@ -111,12 +133,21 @@ class BoundaryBoxConstraint } /** - * Map the given coordinates back to the preimage, - * which is bounded by [lowerBound - al, upperBound + au]. - * - * @param y Given coordinates. - * @return Transformed coordinates. - */ + * Return a suitable initial step size. + * + * @return initial step size. + */ + typename MatType::elem_type initialStepSize() { + return 0.3*arma::min(arma::min(upperBound - lowerBound)); + } + + /** + * Map the given coordinates back to the preimage, + * which is bounded by [lowerBound - al, upperBound + au]. + * + * @param y Given coordinates. + * @return Transformed coordinates. + MatType Inverse(const MatType& y) { typedef typename MatType::elem_type ElemType; @@ -148,6 +179,8 @@ class BoundaryBoxConstraint return x; } + */ + //! Get the lower bound of decision variables. MatType LowerBound() const { return lowerBound; } //! Modify the lower bound of decision variables. @@ -159,7 +192,10 @@ class BoundaryBoxConstraint MatType& UpperBound() { return upperBound; } private: + //! Upper bound of decision variables. MatType upperBound; + + //! Lower bound of decision variables. MatType lowerBound; }; diff --git a/include/ensmallen_bits/cmaes/transformation_policies/empty_transformation.hpp b/include/ensmallen_bits/cmaes/transformation_policies/empty_transformation.hpp index b2ca32c5b..9d68eb108 100644 --- a/include/ensmallen_bits/cmaes/transformation_policies/empty_transformation.hpp +++ b/include/ensmallen_bits/cmaes/transformation_policies/empty_transformation.hpp @@ -2,8 +2,7 @@ * @file empty_transformation.hpp * @author Suvarsha Chennareddy * - * Empty Transformation, can also be called an Indentiy Transformation. - * + * Empty Transformation, can also be called an Indentity Transformation. * * ensmallen is free software; you may redistribute it and/or modify it under * the terms of the 3-clause BSD license. You should have received a copy of @@ -12,34 +11,43 @@ */ namespace ens { + /** - * This is an empty transformation. As the name indicates, it does - * not do anything. It is essentially an identity - * transformation and is meant to be used when there are no - * sorts of constraints on the coordinates. - * - * @tparam MatType The matrix type of the coordinates. - */ + * This is an empty transformation. As the name indicates, it does + * not do anything. It is essentially an identity + * transformation and is meant to be used when there are no + * sorts of constraints on the coordinates. + * + * @tparam MatType The matrix type of the coordinates. + */ template class EmptyTransformation { public: -/** -* Transforms coordinates to themselves (effectively no transformation). -* -* @param x Input coordinates. -* @return Transformed coordinates (the coordinates themselves). -*/ -MatType Transform(const MatType& x) { return x; } + /** + * Transforms coordinates to themselves (effectively no transformation). + * + * @param x Input coordinates. + * @return Transformed coordinates (the coordinates themselves). + */ + MatType Transform(const MatType& x) { return x; } -/** -* Inverse transformtion (effectively no transformation). -* -* @param y Input coordinates. -* @return Transformed coordinates (the coordinates themselves). -*/ -MatType Inverse(const MatType& y) { return y; } + /** + * Return a suitable initial step size. + * + * @return initial step size. + */ + typename MatType::elem_type initialStepSize() { return 1; } + + /** + * Inverse transformtion (effectively no transformation). + * + * @param y Input coordinates. + * @return Transformed coordinates (the coordinates themselves). + + MatType Inverse(const MatType& y) { return y; } + */ }; -} // namespace ens \ No newline at end of file +} // namespace ens diff --git a/tests/callbacks_test.cpp b/tests/callbacks_test.cpp index 79a7c4353..a40bbf480 100644 --- a/tests/callbacks_test.cpp +++ b/tests/callbacks_test.cpp @@ -312,7 +312,8 @@ TEST_CASE("BigBatchSGDCallbacksFullFunctionTest", "[CallbacksTest]") */ TEST_CASE("CMAESCallbacksFullFunctionTest", "[CallbacksTest]") { - CMAES<> optimizer(0, -1, 1, 32, 3, 1e-3); + BoundaryBoxConstraint b(-1, 1); + CMAES optimizer(0, b, 32, 3, 1e-3); CallbacksFullFunctionTest(optimizer, true, false, false, false, true, true, false, false, true); } diff --git a/tests/cmaes_test.cpp b/tests/cmaes_test.cpp index c41231d78..94530c1d7 100644 --- a/tests/cmaes_test.cpp +++ b/tests/cmaes_test.cpp @@ -20,13 +20,22 @@ using namespace ens::test; /** * Run CMA-ES with the full selection policy on logistic regression and * make sure the results are acceptable. + * This test uses the deprecated constructor and therefore can be removed + * in a future version of ensmallen. */ TEST_CASE("CMAESLogisticRegressionTest", "[CMAESTest]") { - arma::mat lb(1, 1, arma::fill::value(-1)); - arma::mat ub(1, 1, arma::fill::value(1)); - BoundaryBoxConstraint b(lb, ub); + CMAES> cmaes(0, -1, 1, 32, 200, 1e-3); + LogisticRegressionFunctionTest(cmaes, 0.003, 0.006, 5); +} +/** + * Run CMA-ES with the full selection policy on logistic regression and + * make sure the results are acceptable. + */ +TEST_CASE("CMAESLogisticRegressionTest", "[CMAESTest]") +{ + BoundaryBoxConstraint b(-1, 1); CMAES cmaes(0, b, 32, 200, 1e-3); LogisticRegressionFunctionTest(cmaes, 0.003, 0.006, 5); } @@ -37,10 +46,7 @@ TEST_CASE("CMAESLogisticRegressionTest", "[CMAESTest]") */ TEST_CASE("ApproxCMAESLogisticRegressionTest", "[CMAESTest]") { - arma::mat lb(1, 1, arma::fill::value(-1)); - arma::mat ub(1, 1, arma::fill::value(1)); - BoundaryBoxConstraint b(lb, ub); - + BoundaryBoxConstraint b(-1, 1); ApproxCMAES> cmaes(0, b, 32, 200, 1e-3); LogisticRegressionFunctionTest(cmaes, 0.003, 0.006, 5); } @@ -51,10 +57,7 @@ TEST_CASE("ApproxCMAESLogisticRegressionTest", "[CMAESTest]") */ TEST_CASE("CMAESLogisticRegressionFMatTest", "[CMAESTest]") { - arma::fmat lb(1, 1, arma::fill::value(-1)); - arma::fmat ub(1, 1, arma::fill::value(1)); - BoundaryBoxConstraint b(lb, ub); - + BoundaryBoxConstraint b(-1, 1); CMAES cmaes(0, b, 32, 200, 1e-3); LogisticRegressionFunctionTest(cmaes, 0.01, 0.02, 5); } @@ -65,10 +68,20 @@ TEST_CASE("CMAESLogisticRegressionFMatTest", "[CMAESTest]") */ TEST_CASE("ApproxCMAESLogisticRegressionFMatTest", "[CMAESTest]") { - arma::fmat lb(1, 1, arma::fill::value(-1)); - arma::fmat ub(1, 1, arma::fill::value(1)); - BoundaryBoxConstraint b(lb, ub); - + BoundaryBoxConstraint b(-1, 1); ApproxCMAES> cmaes(0, b, 32, 200, 1e-3); LogisticRegressionFunctionTest(cmaes, 0.01, 0.02, 5); } + +/** + * Run CMA-ES with the random selection and empty transformation policies + * on logistic regression and make sure the results are acceptable. + * Use arma::fmat. + */ +TEST_CASE("ApproxCMAESEmptyTransformationLogisticRegressionFMatTest", + "[CMAESTest]") +{ + ApproxCMAES> + cmaes(5000, EmptyTransformation(), 32, 5000, 1e-3); + LogisticRegressionFunctionTest(cmaes, 0.01, 0.02, 5); +} From 4a4282017658971c7a1a41bec4cc4b74c11fdb45 Mon Sep 17 00:00:00 2001 From: Suvarsha Chennareddy Date: Tue, 17 Jan 2023 01:05:06 +0530 Subject: [PATCH 04/34] change test name --- tests/cmaes_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cmaes_test.cpp b/tests/cmaes_test.cpp index 94530c1d7..4aca40661 100644 --- a/tests/cmaes_test.cpp +++ b/tests/cmaes_test.cpp @@ -23,7 +23,7 @@ using namespace ens::test; * This test uses the deprecated constructor and therefore can be removed * in a future version of ensmallen. */ -TEST_CASE("CMAESLogisticRegressionTest", "[CMAESTest]") +TEST_CASE("CMAESDeprecatedConstructorLogisticRegressionTest", "[CMAESTest]") { CMAES> cmaes(0, -1, 1, 32, 200, 1e-3); LogisticRegressionFunctionTest(cmaes, 0.003, 0.006, 5); From 21e2e89911eb514c482b8b4c2abd451e4ac0afe5 Mon Sep 17 00:00:00 2001 From: Suvarsha Chennareddy Date: Thu, 19 Jan 2023 17:46:01 +0530 Subject: [PATCH 05/34] remove unused code --- include/ensmallen_bits/cmaes/cmaes_impl.hpp | 2 + .../boundary_box_constraint.hpp | 50 ------------------- .../empty_transformation.hpp | 9 ---- 3 files changed, 2 insertions(+), 59 deletions(-) diff --git a/include/ensmallen_bits/cmaes/cmaes_impl.hpp b/include/ensmallen_bits/cmaes/cmaes_impl.hpp index cf31b7f44..67acef983 100644 --- a/include/ensmallen_bits/cmaes/cmaes_impl.hpp +++ b/include/ensmallen_bits/cmaes/cmaes_impl.hpp @@ -24,6 +24,8 @@ This partial specialization is used to throw an exception when the TransformationPolicyType is EmptyTransformation and call a constructor with paramters 'lowerBound' and 'upperBound' otherwise. + This shall be removed when the deprecated constructor is removed in + the next major version of ensmallen. */ template struct notEmptyTransformation : std::true_type { diff --git a/include/ensmallen_bits/cmaes/transformation_policies/boundary_box_constraint.hpp b/include/ensmallen_bits/cmaes/transformation_policies/boundary_box_constraint.hpp index 4afbfb97f..193c4631a 100644 --- a/include/ensmallen_bits/cmaes/transformation_policies/boundary_box_constraint.hpp +++ b/include/ensmallen_bits/cmaes/transformation_policies/boundary_box_constraint.hpp @@ -24,7 +24,6 @@ namespace ens { * These shifted coordinates are then transformed into coordinates bounded by * lower_bound and upper_bound. It is an identity transformation in between * the lower and upper bounds. - * The inverse transformation transforms the coordinates back to the preimage. * * For more information, check the original implementation in C by N. Hansen: * https://github.com/CMA-ES/c-cmaes/blob/master/src/boundary_transformation.c @@ -132,55 +131,6 @@ class BoundaryBoxConstraint return y; } - /** - * Return a suitable initial step size. - * - * @return initial step size. - */ - typename MatType::elem_type initialStepSize() { - return 0.3*arma::min(arma::min(upperBound - lowerBound)); - } - - /** - * Map the given coordinates back to the preimage, - * which is bounded by [lowerBound - al, upperBound + au]. - * - * @param y Given coordinates. - * @return Transformed coordinates. - - MatType Inverse(const MatType& y) - { - typedef typename MatType::elem_type ElemType; - double diff, al, au; - size_t Bi, Bj; - MatType x = y; - for (size_t i = 0; i < y.n_rows; i++) - { - Bi = (i < lowerBound.n_rows) ? i : (lowerBound.n_rows - 1); - for (size_t j = 0; j < y.n_cols; j++) - { - Bj = (j < lowerBound.n_cols) ? j : (lowerBound.n_cols - 1); - - diff = (upperBound(Bi, Bj) - lowerBound(Bi, Bj)) / 2.0; - al = std::min(diff, (1 + std::abs(lowerBound(Bi, Bj))) / 20.0); - au = std::min(diff, (1 + std::abs(upperBound(Bi, Bj))) / 20.0); - if (x(i, j) < lowerBound(Bi, Bj) + al) - { - x(i, j) = (ElemType)(lowerBound(Bi, Bj) - al) + 2 * - std::sqrt(std::abs(al * (x(i, j) - lowerBound(Bi, Bj)))); - } - else if (x(i, j) > upperBound(Bi, Bj) - au) - { - x(i, j) = (ElemType)(upperBound(Bi, Bj) + au) - 2 * - std::sqrt(std::abs(au * (upperBound(Bi, Bj) - x(i, j)))); - } - } - } - - return x; - } - */ - //! Get the lower bound of decision variables. MatType LowerBound() const { return lowerBound; } //! Modify the lower bound of decision variables. diff --git a/include/ensmallen_bits/cmaes/transformation_policies/empty_transformation.hpp b/include/ensmallen_bits/cmaes/transformation_policies/empty_transformation.hpp index 9d68eb108..ec969c46e 100644 --- a/include/ensmallen_bits/cmaes/transformation_policies/empty_transformation.hpp +++ b/include/ensmallen_bits/cmaes/transformation_policies/empty_transformation.hpp @@ -40,14 +40,5 @@ class EmptyTransformation */ typename MatType::elem_type initialStepSize() { return 1; } - /** - * Inverse transformtion (effectively no transformation). - * - * @param y Input coordinates. - * @return Transformed coordinates (the coordinates themselves). - - MatType Inverse(const MatType& y) { return y; } - */ }; - } // namespace ens From 43f5146af51ed28c70e8356cb32ff8f77187f486 Mon Sep 17 00:00:00 2001 From: Suvarsha Chennareddy Date: Fri, 28 Apr 2023 00:48:25 +0530 Subject: [PATCH 06/34] update documentation, add ens_deprecated, and fix errors --- HISTORY.md | 1252 ++++++++++++----- doc/optimizers.md | 38 +- include/ensmallen_bits/cmaes/cmaes.hpp | 19 +- include/ensmallen_bits/cmaes/cmaes_impl.hpp | 24 +- .../boundary_box_constraint.hpp | 12 +- 5 files changed, 968 insertions(+), 377 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 930a0eae3..e48c2c429 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,466 +1,1036 @@ -### ensmallen ?.??.?: "???" +### mlpack ?.?.? ###### ????-??-?? -* Avoid deprecation warnings in Armadillo 11.2+ - ([#347](https://github.com/mlpack/ensmallen/pull/347)). + * Fix mapping of categorical data for Julia bindings (#3305). -### ensmallen 2.19.0: "Eight Ball Deluxe" -###### 2022-04-06 -* Added DemonSGD and DemonAdam optimizers - ([#211](https://github.com/mlpack/ensmallen/pull/211)). +### mlpack 4.0.0 +###### 2022-10-23 + * Bump C++ standard requirement to C++14 (#3233). - * Fix bug with Adam-like optimizers not resetting when `resetPolicy` is `true`. - ([#340](https://github.com/mlpack/ensmallen/pull/340)). + * Fix `Perceptron` to work with cross-validation framework (#3190). - * Add Yogi optimizer - ([#232](https://github.com/mlpack/ensmallen/pull/232)). + * Migrate from boost tests to Catch2 framework (#2523), (#2584). - * Add AdaBelief optimizer - ([#233](https://github.com/mlpack/ensmallen/pull/233)). + * Bump minimum armadillo version from 8.400 to 9.800 (#3043), (#3048). - * Add AdaSqrt optimizer - ([#234](https://github.com/mlpack/ensmallen/pull/234)). - - * Bump check for minimum supported version of Armadillo - ([#342](https://github.com/mlpack/ensmallen/pull/342)). + * Adding a copy constructor in the Convolution layer (#3067). -### ensmallen 2.18.2: "Fairmount Bagel" -###### 2022-02-13 - * Update Catch2 to 2.13.8 - ([#336](https://github.com/mlpack/ensmallen/pull/336)). + * Replace `boost::spirit` parser by a local efficient implementation (#2942). - * Fix epoch timing output - ([#337](https://github.com/mlpack/ensmallen/pull/337)). + * Disable correctly the autodownloader + fix tests stability (#3076). -### ensmallen 2.18.1: "Fairmount Bagel" -###### 2021-11-19 - * Accelerate SGD test time - ([#330](https://github.com/mlpack/ensmallen/pull/300)). + * Replace `boost::any` with `core::v2::any` or `std::any` if available (#3006). - * Fix potential infinite loop in CMAES - ([#331](https://github.com/mlpack/ensmallen/pull/331)). + * Remove old non used Boost headers (#3005). - * Fix SCD partial gradient test - ([#332](https://github.com/mlpack/ensmallen/pull/332)). + * Replace `boost::enable_if` with `std::enable_if` (#2998). -### ensmallen 2.18.0: "Fairmount Bagel" -###### 2021-10-20 - * Add gradient value clipping and gradient norm scaling callback - ([#315](https://github.com/mlpack/ensmallen/pull/315)). + * Replace `boost::is_same` with `std::is_same` (#2993). - * Remove superfluous CMake option to build the tests - ([#313](https://github.com/mlpack/ensmallen/pull/313)). + * Remove invalid option for emsmallen and STB (#2960). - * Bump minimum Armadillo version to 9.800 - ([#318](https://github.com/mlpack/ensmallen/pull/318)). + * Check for armadillo dependencies before downloading armadillo (#2954). - * Update Catch2 to 2.13.7 - ([#322](https://github.com/mlpack/ensmallen/pull/322)). + * Disable the usage of autodownloader by default (#2953). - * Remove redundant template argument for C++20 compatibility - ([#324](https://github.com/mlpack/ensmallen/pull/324)). + * Install dependencies downloaded with the autodownloader (#2952). + + * Download older Boost if the compiler is old (#2940). - * Fix MOEAD test stability - ([#327](https://github.com/mlpack/ensmallen/pull/327)). + * Add support for embedded systems (#2531). -### ensmallen 2.17.0: "Pachis Din Me Pesa Double" -###### 2021-07-06 - * CheckArbitraryFunctionTypeAPI extended for MOO support - ([#283](https://github.com/mlpack/ensmallen/pull/283)). + * Build mlpack executable statically if the library is statically linked (#2931). - * Refactor NSGA2 - ([#263](https://github.com/mlpack/ensmallen/pull/263), - [#304](https://github.com/mlpack/ensmallen/pull/304)). + * Fix cover tree loop bug on embedded arm systems (#2869). - * Add Indicators for Multiobjective optimizers - ([#285](https://github.com/mlpack/ensmallen/pull/285)). + * Fix a LAPACK bug in `FindArmadillo.cmake` (#2929). - * Make Callback flexible for MultiObjective Optimizers - ([#289](https://github.com/mlpack/ensmallen/pull/289)). + * Add an autodownloader to get mlpack dependencies (#2927). - * Add ZDT Test Suite - ([#273](https://github.com/mlpack/ensmallen/pull/273)). + * Remove Coverage files and configurations from CMakeLists (#2866). - * Add MOEA-D/DE Optimizer - ([#269](https://github.com/mlpack/ensmallen/pull/269)). + * Added `Multi Label Soft Margin Loss` loss function for neural networks + (#2345). - * Introduce Policy Methods for MOEA/D-DE - ([#293](https://github.com/mlpack/ensmallen/pull/293)). + * Added Decision Tree Regressor (#2905). It can be used using the class + `mlpack::tree::DecisionTreeRegressor`. It is accessible only though C++. - * Add Das-Dennis weight initialization method - ([#295](https://github.com/mlpack/ensmallen/pull/295)). + * Added dict-style inspection of mlpack models in python bindings (#2868). - * Add Dirichlet Weight Initialization - ([#296](https://github.com/mlpack/ensmallen/pull/296)). + * Added Extra Trees Algorithm (#2883). Currently, it can be used using the + class `mlpack::tree::ExtraTrees`, but only through C++. - * Improved installation and compilation instructions - ([#300](https://github.com/mlpack/ensmallen/pull/300)). + * Add Flatten T Swish activation function (`flatten-t-swish.hpp`) - * Disable building the tests by default for faster installation - ([#303](https://github.com/mlpack/ensmallen/pull/303)). + * Added warm start feature to Random Forest (#2881); this feature is + accessible from mlpack's bindings to different languages. - * Modify matrix initialisation to take into account - default element zeroing in Armadillo 10.5 - ([#305](https://github.com/mlpack/ensmallen/pull/305)). + * Added Pixel Shuffle layer (#2563). -### ensmallen 2.16.2: "Severely Dented Can Of Polyurethane" -###### 2021-03-24 - * Fix CNE test trials - ([#267](https://github.com/mlpack/ensmallen/pull/267)). + * Add "check_input_matrices" option to python bindings that checks + for NaN and inf values in all the input matrices (#2787). - * Update Catch2 to 2.13.4 - ([#268](https://github.com/mlpack/ensmallen/pull/268)). + * Add Adjusted R squared functionality to R2Score::Evaluate (#2624). - * Fix typos in documentation - ([#270](https://github.com/mlpack/ensmallen/pull/270), - [#271](https://github.com/mlpack/ensmallen/pull/271)). + * Disabled all the bindings by default in CMake (#2782). - * Add clarifying comments in problems/ implementations - ([#276](https://github.com/mlpack/ensmallen/pull/276)). + * Added an implementation to Stratify Data (#2671). -### ensmallen 2.16.1: "Severely Dented Can Of Polyurethane" -###### 2021-03-02 - * Fix test compilation issue when `ENS_USE_OPENMP` is set - ([#255](https://github.com/mlpack/ensmallen/pull/255)). + * Add `BUILD_DOCS` CMake option to control whether Doxygen documentation is + built (default ON) (#2730). - * Fix CNE initial population generation to use normal distribution - ([#258](https://github.com/mlpack/ensmallen/pull/258)). + * Add Triplet Margin Loss function (#2762). - * Fix compilation warnings - ([#259](https://github.com/mlpack/ensmallen/pull/259)). + * Add finalizers to Julia binding model types to fix memory handling (#2756). - * Remove `AdamSchafferFunctionN2Test` test from Adam test suite to prevent - spurious issue on some aarch64 ([#265](https://github.com/mlpack/ensmallen/pull/259)). + * HMM: add functions to calculate likelihood for data stream with/without + pre-calculated emission probability (#2142). -### ensmallen 2.16.0: "Severely Dented Can Of Polyurethane" -###### 2021-02-11 - * Expand README with example installation and add simple example program - showing usage of the L-BFGS optimizer - ([#248](https://github.com/mlpack/ensmallen/pull/248)). + * Replace Boost serialization library with Cereal (#2458). - * Refactor tests to increase stability and reduce random errors - ([#249](https://github.com/mlpack/ensmallen/pull/249)). + * Add `PYTHON_INSTALL_PREFIX` CMake option to specify installation root for + Python bindings (#2797). -### ensmallen 2.15.1: "Why Can't I Manage To Grow Any Plants?" -###### 2020-11-05 - * Fix include order to ensure traits is loaded before reports - ([#239](https://github.com/mlpack/ensmallen/pull/239)). + * Removed `boost::visitor` from model classes for `knn`, `kfn`, `cf`, + `range_search`, `krann`, and `kde` bindings (#2803). -### ensmallen 2.15.0: "Why Can't I Manage To Grow Any Plants?" -###### 2020-11-01 - * Make a few tests more robust - ([#228](https://github.com/mlpack/ensmallen/pull/228)). + * Add k-means++ initialization strategy (#2813). - * Add release date to version information. ([#226](https://github.com/mlpack/ensmallen/pull/226)) + * `NegativeLogLikelihood<>` now expects classes in the range `0` to + `numClasses - 1` (#2534). - * Fix typo in release script - ([#236](https://github.com/mlpack/ensmallen/pull/236)). + * Add `Lambda1()`, `Lambda2()`, `UseCholesky()`, and `Tolerance()` members to + `LARS` so parameters for training can be modified (#2861). -### ensmallen 2.14.2: "No Direction Home" -###### 2020-08-31 - * Fix implementation of fonesca fleming problem function f1 and f2 - type usage and negative signs. ([#223](https://github.com/mlpack/ensmallen/pull/223)) + * Remove unused `ElemType` template parameter from `DecisionTree` and + `RandomForest` (#2874). -### ensmallen 2.14.1: "No Direction Home" -###### 2020-08-19 - * Fix release script (remove hardcoded information, trim leading whitespaces - introduced by `wc -l` in MacOS) - ([#216](https://github.com/mlpack/ensmallen/pull/216), - [#220](https://github.com/mlpack/ensmallen/pull/220)). + * Fix Python binding build when the CMake variable `USE_OPENMP` is set to + `OFF` (#2884). - * Adjust tolerance for AugLagrangian convergence based on element type - ([#217](https://github.com/mlpack/ensmallen/pull/217)). + * The `mlpack_test` target is no longer built as part of `make all`. Use + `make mlpack_test` to build the tests. -### ensmallen 2.14.0: "No Direction Home" -###### 2020-08-10 - * Add NSGA2 optimizer for multi-objective functions - ([#149](https://github.com/mlpack/ensmallen/pull/149)). + * Fixes to `HoeffdingTree`: ensure that training still works when empty + constructor is used (#2964). - * Update automatic website update release script - ([#207](https://github.com/mlpack/ensmallen/pull/207)). + * Fix Julia model serialization bug (#2970). - * Clarify and fix documentation for constrained optimizers - ([#201](https://github.com/mlpack/ensmallen/pull/201)). + * Fix `LoadCSV()` to use pre-populated `DatasetInfo` objects (#2980). - * Fix L-BFGS convergence when starting from a minimum - ([#201](https://github.com/mlpack/ensmallen/pull/201)). + * Add `probabilities` option to softmax regression binding, to get class + probabilities for test points (#3001). -* Add optimizer summary report callback - ([#213](https://github.com/mlpack/ensmallen/pull/213)). + * Fix thread safety issues in mlpack bindings to other languages (#2995). -### ensmallen 2.13.0: "Automatically Automated Automation" -###### 2020-07-15 - * Fix CMake package export - ([#198](https://github.com/mlpack/ensmallen/pull/198)). + * Fix double-free of model pointers in R bindings (#3034). - * Allow early stop callback to accept a lambda function - ([#165](https://github.com/mlpack/ensmallen/pull/165)). + * Fix Julia, Python, R, and Go handling of categorical data for + `decision_tree()` and `hoeffding_tree()` (#2971). -### ensmallen 2.12.1: "Stir Crazy" -###### 2020-04-20 - * Fix total number of epochs and time estimation for ProgressBar callback - ([#181](https://github.com/mlpack/ensmallen/pull/181)). + * Depend on `pkgbuild` for R bindings (#3081). + + * Replaced Numpy deprecated code in Python bindings (#3126). - * Handle SpSubview_col and SpSubview_row in Armadillo 9.870 - ([#194](https://github.com/mlpack/ensmallen/pull/194)). +### mlpack 3.4.2 +###### 2020-10-26 + * Added Mean Absolute Percentage Error. - * Minor documentation fixes - ([#197](https://github.com/mlpack/ensmallen/pull/197)). + * Added Softmin activation function as layer in ann/layer. -### ensmallen 2.12.0: "Stir Crazy" -###### 2020-03-28 - * Correction in the formulation of sigma in CMA-ES - ([#183](https://github.com/mlpack/ensmallen/pull/183)). + * Fix spurious ARMA_64BIT_WORD compilation warnings on 32-bit systems (#2665). - * Remove deprecated methods from PrimalDualSolver implementation - ([#185](https://github.com/mlpack/ensmallen/pull/185). +### mlpack 3.4.1 +###### 2020-09-07 + * Fix incorrect parsing of required matrix/model parameters for command-line + bindings (#2600). - * Update logo ([#186](https://github.com/mlpack/ensmallen/pull/186)). + * Add manual type specification support to `data::Load()` and `data::Save()` + (#2084, #2135, #2602). -### ensmallen 2.11.5: "The Poster Session Is Full" -###### 2020-03-11 - * Change "mathematical optimization" term to "numerical optimization" in the - documentation ([#177](https://github.com/mlpack/ensmallen/pull/177)). + * Remove use of internal Armadillo functionality (#2596, #2601, #2602). -### ensmallen 2.11.4: "The Poster Session Is Full" -###### 2020-03-03 - * Require new HISTORY.md entry for each PR. - ([#171](https://github.com/mlpack/ensmallen/pull/171), - [#172](https://github.com/mlpack/ensmallen/pull/172), - [#175](https://github.com/mlpack/ensmallen/pull/175)). +### mlpack 3.4.0 +###### 2020-09-01 + * Issue warnings when metrics produce NaNs in KFoldCV (#2595). - * Update/fix example documentation - ([#174](https://github.com/mlpack/ensmallen/pull/174)). + * Added bindings for _R_ during Google Summer of Code (#2556). -### ensmallen 2.11.3: "The Poster Session Is Full" -###### 2020-02-19 - * Prevent spurious compiler warnings - ([#161](https://github.com/mlpack/ensmallen/pull/161)). + * Added common striptype function for all bindings (#2556). - * Fix minor memory leaks - ([#167](https://github.com/mlpack/ensmallen/pull/167)). + * Refactored common utility function of bindings to bindings/util (#2556). - * Revamp CMake configuration - ([#152](https://github.com/mlpack/ensmallen/pull/152)). + * Renamed InformationGain to HoeffdingInformationGain in + methods/hoeffding_trees/information_gain.hpp (#2556). -### ensmallen 2.11.2: "The Poster Session Is Full" -###### 2020-01-16 - * Allow callback instantiation for SGD based optimizer - ([#138](https://github.com/mlpack/ensmallen/pull/155)). + * Added macro for changing stream of printing and warnings/errors (#2556). - * Minor test stability fixes on i386 - ([#156](https://github.com/mlpack/ensmallen/pull/156)). + * Added Spatial Dropout layer (#2564). - * Fix Lookahead MaxIterations() check. - ([#159](https://github.com/mlpack/ensmallen/pull/159)). + * Force CMake to show error when it didn't find Python/modules (#2568). -### ensmallen 2.11.1: "The Poster Session Is Full" -###### 2019-12-28 - * Fix Lookahead Synchronization period type - ([#153](https://github.com/mlpack/ensmallen/pull/153)). + * Refactor `ProgramInfo()` to separate out all the different + information (#2558). -### ensmallen 2.11.0: "The Poster Session Is Full" -###### 2019-12-24 - * Add Lookahead - ([#138](https://github.com/mlpack/ensmallen/pull/138)). + * Add bindings for one-hot encoding (#2325). - * Add AdaBound and AMSBound - ([#137](https://github.com/mlpack/ensmallen/pull/137)). + * Added Soft Actor-Critic to RL methods (#2487). -### ensmallen 2.10.5: "Fried Chicken" -###### 2019-12-13 - * SGD callback test 32-bit safety (big number) - ([#143](https://github.com/mlpack/ensmallen/pull/143)). + * Added Categorical DQN to q_networks (#2454). - * Use "arbitrary" and "separable" terms in static function type checks - ([#145](https://github.com/mlpack/ensmallen/pull/145)). + * Added N-step DQN to q_networks (#2461). - * Remove 'using namespace std' from `problems/` files - ([#147](https://github.com/mlpack/ensmallen/pull/147)). + * Add Silhoutte Score metric and Pairwise Distances (#2406). -### ensmallen 2.10.4: "Fried Chicken" -###### 2019-11-18 - * Add optional tests building. - ([#141](https://github.com/mlpack/ensmallen/pull/141)). + * Add Go bindings for some missed models (#2460). - * Make code samples collapsible in the documentation. - ([#140](https://github.com/mlpack/ensmallen/pull/140)). + * Replace boost program_options dependency with CLI11 (#2459). -### ensmallen 2.10.3: "Fried Chicken" -###### 2019-09-26 - * Fix ParallelSGD runtime bug. - ([#135](https://github.com/mlpack/ensmallen/pull/135)). + * Additional functionality for the ARFF loader (#2486); use case sensitive + categories (#2516). - * Add additional L-BFGS convergence check - ([#136](https://github.com/mlpack/ensmallen/pull/136)). + * Add `bayesian_linear_regression` binding for the command-line, Python, + Julia, and Go. Also called "Bayesian Ridge", this is equivalent to a + version of linear regression where the regularization parameter is + automatically tuned (#2030). -### ensmallen 2.10.2: "Fried Chicken" -###### 2019-09-11 - * Add release script to rel/ for maintainers - ([#128](https://github.com/mlpack/ensmallen/pull/128)). + * Fix defeatist search for spill tree traversals (#2566, #1269). - * Fix Armadillo version check - ([#133](https://github.com/mlpack/ensmallen/pull/133)). + * Fix incremental training of logistic regression models (#2560). -### ensmallen 2.10.1: "Fried Chicken" -###### 2019-09-10 - * Documentation fix for callbacks - ([#129](https://github.com/mlpack/ensmallen/pull/129). + * Change default configuration of `BUILD_PYTHON_BINDINGS` to `OFF` (#2575). - * Compatibility fixes for ensmallen 1.x - ([#131](https://github.com/mlpack/ensmallen/pull/131)). +### mlpack 3.3.2 +###### 2020-06-18 + * Added Noisy DQN to q_networks (#2446). -### ensmallen 2.10.0: "Fried Chicken" -###### 2019-09-07 - * All `Optimize()` functions now take any matrix type; so, e.g., `arma::fmat` - or `arma::sp_mat` can be used for optimization. See the documentation for - more details ([#113](https://github.com/mlpack/ensmallen/pull/113), - [#119](https://github.com/mlpack/ensmallen/pull/119)). + * Add Go bindings (#1884). - * Introduce callback support. Callbacks can be appended as the last arguments - of an `Optimize()` call, and can perform custom behavior at different points - during the optimization. See the documentation for more details - ([#119](https://github.com/mlpack/ensmallen/pull/119)). + * Added Dueling DQN to q_networks, Noisy linear layer to ann/layer + and Empty loss to ann/loss_functions (#2414). - * Slight speedups for `FrankWolfe` optimizer - ([#127](https://github.com/mlpack/ensmallen/pull/127)). + * Storing and adding accessor method for action in q_learning (#2413). -### ensmallen 1.16.2: "Loud Alarm Clock" -###### 2019-08-12 - * Fix PSO return type bug - ([#126](https://github.com/mlpack/ensmallen/pull/126)). + * Added accessor methods for ANN layers (#2321). -### ensmallen 1.16.1: "Loud Alarm Clock" -###### 2019-08-11 - * Update HISTORY.md to use Markdown links to the PR and add release names. + * Addition of `Elliot` activation function (#2268). - * Fix PSO return type bug - ([#124](https://github.com/mlpack/ensmallen/pull/124)). + * Add adaptive max pooling and adaptive mean pooling layers (#2195). -### ensmallen 1.16.0: "Loud Alarm Clock" -###### 2019-08-09 - * Add option to avoid computing exact objective at the end of the optimization - ([#109](https://github.com/mlpack/ensmallen/pull/109)). + * Add parameter to avoid shuffling of data in preprocess_split (#2293). - * Fix handling of curvature for BigBatchSGD - ([#118](https://github.com/mlpack/ensmallen/pull/118)). + * Add `MatType` parameter to `LSHSearch`, allowing sparse matrices to be used + for search (#2395). - * Reduce runtime of tests - ([#118](https://github.com/mlpack/ensmallen/pull/118)). + * Documentation fixes to resolve Doxygen warnings and issues (#2400). - * Introduce local-best particle swarm optimization, `LBestPSO`, for - unconstrained optimization problems - ([#86](https://github.com/mlpack/ensmallen/pull/86)). + * Add Load and Save of Sparse Matrix (#2344). -### ensmallen 1.15.1: "Wrong Side Of The Road" -###### 2019-05-22 - * Fix `-Wreorder` in `qhadam` warning - ([#115](https://github.com/mlpack/ensmallen/pull/115)). + * Add Intersection over Union (IoU) metric for bounding boxes (#2402). - * Fix `-Wunused-private-field` warning in `spsa` - ([#115](https://github.com/mlpack/ensmallen/pull/115)). + * Add Non Maximal Supression (NMS) metric for bounding boxes (#2410). - * Add more warning output for gcc/clang - ([#116](https://github.com/mlpack/ensmallen/pull/116)). + * Fix `no_intercept` and probability computation for linear SVM bindings + (#2419). -### ensmallen 1.15.0: "Wrong Side Of The Road" -###### 2019-05-14 - * Added QHAdam and QHSGD optimizers - ([#81](https://github.com/mlpack/ensmallen/pull/81)). + * Fix incorrect neighbors for `k > 1` searches in `approx_kfn` binding, for + the `QDAFN` algorithm (#2448). -### ensmallen 1.14.4: "Difficult Crimp" -###### 2019-05-12 - * Fixes for BigBatchSGD ([#91](https://github.com/mlpack/ensmallen/pull/91)). + * Fix serialization of kernels with state for FastMKS (#2452). -### ensmallen 1.14.3: "Difficult Crimp" -###### 2019-05-06 - * Handle `eig_sym()` failures correctly - ([#100](https://github.com/mlpack/ensmallen/pull/100)). + * Add `RBF` layer in ann module to make `RBFN` architecture (#2261). -### ensmallen 1.14.2: "Difficult Crimp" -###### 2019-03-14 - * SPSA test tolerance fix - ([#97](https://github.com/mlpack/ensmallen/pull/97)). +### mlpack 3.3.1 +###### 2020-04-29 + * Minor Julia and Python documentation fixes (#2373). - * Minor documentation fixes (#95, #98). + * Updated terminal state and fixed bugs for Pendulum environment (#2354, + #2369). - * Fix newlines at end of file - ([#92](https://github.com/mlpack/ensmallen/pull/92)). + * Added `EliSH` activation function (#2323). -### ensmallen 1.14.1: "Difficult Crimp" -###### 2019-03-09 - * Fixes for SPSA ([#87](https://github.com/mlpack/ensmallen/pull/87)). + * Add L1 Loss function (#2203). - * Optimized CNE and DE ([#90](https://github.com/mlpack/ensmallen/pull/90)). - Changed initial population generation in CNE to be a normal distribution - about the given starting point, which should accelerate convergence. + * Pass CMAKE_CXX_FLAGS (compilation options) correctly to Python build + (#2367). -### ensmallen 1.14.0: "Difficult Crimp" -###### 2019-02-20 - * Add DE optimizer ([#77](https://github.com/mlpack/ensmallen/pull/77)). + * Expose ensmallen Callbacks for sparseautoencoder (#2198). - * Fix for Cholesky decomposition in CMAES - ([#83](https://github.com/mlpack/ensmallen/pull/83)). + * Bugfix for LARS class causing invalid read (#2374). -### ensmallen 1.13.2: "Coronavirus Invasion" -###### 2019-02-18 - * Minor documentation fixes ([#82](https://github.com/mlpack/ensmallen/pull/82)). + * Add serialization support from Julia; use `mlpack.serialize()` and + `mlpack.deserialize()` to save and load from `IOBuffer`s. -### ensmallen 1.13.1: "Coronavirus Invasion" -###### 2019-01-24 - * Fix -Wreorder warning ([#75](https://github.com/mlpack/ensmallen/pull/75)). +### mlpack 3.3.0 +###### 2020-04-07 + * Added `Normal Distribution` to `ann/dists` (#2382). -### ensmallen 1.13.0: "Coronavirus Invasion" -###### 2019-01-14 - * Enhance options for AugLagrangian optimizer - ([#66](https://github.com/mlpack/ensmallen/pull/66)). + * Templated return type of `Forward function` of loss functions (#2339). - * Add SPSA optimizer ([#69](https://github.com/mlpack/ensmallen/pull/69)). + * Added `R2 Score` regression metric (#2323). -### ensmallen 1.12.2: "New Year's Party" -###### 2019-01-05 - * Fix list of contributors. + * Added `poisson negative log likelihood` loss function (#2196). -### ensmallen 1.12.1: "New Year's Party" -###### 2019-01-03 - * Make sure all files end with newlines. + * Added `huber` loss function (#2199). -### ensmallen 1.12.0: "New Year's Party" -###### 2018-12-30 - * Add link to ensmallen PDF to README.md. + * Added `mean squared logarithmic error` loss function for neural networks + (#2210). - * Minor documentation fixes. Remove too-verbose documentation from source for - each optimizer ([#61](https://github.com/mlpack/ensmallen/pull/61)). + * Added `mean bias loss function` for neural networks (#2210). - * Add FTML optimizer ([#48](https://github.com/mlpack/ensmallen/pull/48)). + * The DecisionStump class has been marked deprecated; use the `DecisionTree` + class with `NoRecursion=true` or use `ID3DecisionStump` instead (#2099). - * Add SWATS optimizer ([#42](https://github.com/mlpack/ensmallen/pull/42)). + * Added `probabilities_file` parameter to get the probabilities matrix of + AdaBoost classifier (#2050). - * Add Padam optimizer ([#46](https://github.com/mlpack/ensmallen/pull/46)). + * Fix STB header search paths (#2104). - * Add Eve optimizer ([#45](https://github.com/mlpack/ensmallen/pull/45)). + * Add `DISABLE_DOWNLOADS` CMake configuration option (#2104). - * Add ResetPolicy() to SGD-like optimizers - ([#60](https://github.com/mlpack/ensmallen/pull/60)). + * Add padding layer in TransposedConvolutionLayer (#2082). -### ensmallen 1.11.1: "Jet Lag" -###### 2018-11-29 - * Minor documentation fixes. + * Fix pkgconfig generation on non-Linux systems (#2101). -### ensmallen 1.11.0: "Jet Lag" -###### 2018-11-28 - * Add WNGrad optimizer. + * Use log-space to represent HMM initial state and transition probabilities + (#2081). - * Fix header name in documentation samples. + * Add functions to access parameters of `Convolution` and `AtrousConvolution` + layers (#1985). -### ensmallen 1.10.1: "Corporate Catabolism" -###### 2018-11-16 - * Fixes for GridSearch optimizer. + * Add Compute Error function in lars regression and changing Train function to + return computed error (#2139). - * Include documentation with release. + * Add Julia bindings (#1949). Build settings can be controlled with the + `BUILD_JULIA_BINDINGS=(ON/OFF)` and `JULIA_EXECUTABLE=/path/to/julia` CMake + parameters. -### ensmallen 1.10.0: "Corporate Catabolism" -###### 2018-10-20 - * Initial release. + * CMake fix for finding STB include directory (#2145). - * Includes the ported optimization framework from mlpack - (http://www.mlpack.org/). + * Add bindings for loading and saving images (#2019); `mlpack_image_converter` + from the command-line, `mlpack.image_converter()` from Python. + + * Add normalization support for CF binding (#2136). + + * Add Mish activation function (#2158). + + * Update `init_rules` in AMF to allow users to merge two initialization + rules (#2151). + + * Add GELU activation function (#2183). + + * Better error handling of eigendecompositions and Cholesky decompositions + (#2088, #1840). + + * Add LiSHT activation function (#2182). + + * Add Valid and Same Padding for Transposed Convolution layer (#2163). + + * Add CELU activation function (#2191) + + * Add Log-Hyperbolic-Cosine Loss function (#2207). + + * Change neural network types to avoid unnecessary use of rvalue references + (#2259). + + * Bump minimum Boost version to 1.58 (#2305). + + * Refactor STB support so `HAS_STB` macro is not needed when compiling against + mlpack (#2312). + + * Add Hard Shrink Activation Function (#2186). + + * Add Soft Shrink Activation Function (#2174). + + * Add Hinge Embedding Loss Function (#2229). + + * Add Cosine Embedding Loss Function (#2209). + + * Add Margin Ranking Loss Function (#2264). + + * Bugfix for incorrect parameter vector sizes in logistic regression and + softmax regression (#2359). + +### mlpack 3.2.2 +###### 2019-11-26 + * Add `valid` and `same` padding option in `Convolution` and `Atrous + Convolution` layer (#1988). + + * Add Model() to the FFN class to access individual layers (#2043). + + * Update documentation for pip and conda installation packages (#2044). + + * Add bindings for linear SVM (#1935); `mlpack_linear_svm` from the + command-line, `linear_svm()` from Python. + + * Add support to return the layer name as `std::string` (#1987). + + * Speed and memory improvements for the Transposed Convolution layer (#1493). + + * Fix Windows Python build configuration (#1885). + + * Validate md5 of STB library after download (#2087). + + * Add `__version__` to `__init__.py` (#2092). + + * Correctly handle RNN sequences that are shorter than the value of rho (#2102). + +### mlpack 3.2.1 +###### 2019-10-01 + * Enforce CMake version check for ensmallen (#2032). + + * Fix CMake check for Armadillo version (#2029). + + * Better handling of when STB is not installed (#2033). + + * Fix Naive Bayes classifier computations in high dimensions (#2022). + +### mlpack 3.2.0 +###### 2019-09-25 + * Fix some potential infinity errors in Naive Bayes Classifier (#2022). + + * Fix occasionally-failing RADICAL test (#1924). + + * Fix gcc 9 OpenMP compilation issue (#1970). + + * Added support for loading and saving of images (#1903). + + * Add Multiple Pole Balancing Environment (#1901, #1951). + + * Added functionality for scaling of data (#1876); see the command-line + binding `mlpack_preprocess_scale` or Python binding `preprocess_scale()`. + + * Add new parameter `maximum_depth` to decision tree and random forest + bindings (#1916). + + * Fix prediction output of softmax regression when test set accuracy is + calculated (#1922). + + * Pendulum environment now checks for termination. All RL environments now + have an option to terminate after a set number of time steps (no limit + by default) (#1941). + + * Add support for probabilistic KDE (kernel density estimation) error bounds + when using the Gaussian kernel (#1934). + + * Fix negative distances for cover tree computation (#1979). + + * Fix cover tree building when all pairwise distances are 0 (#1986). + + * Improve KDE pruning by reclaiming not used error tolerance (#1954, #1984). + + * Optimizations for sparse matrix accesses in z-score normalization for CF + (#1989). + + * Add `kmeans_max_iterations` option to GMM training binding `gmm_train_main`. + + * Bump minimum Armadillo version to 8.400.0 due to ensmallen dependency + requirement (#2015). + +### mlpack 3.1.1 +###### 2019-05-26 + * Fix random forest bug for numerical-only data (#1887). + + * Significant speedups for random forest (#1887). + + * Random forest now has `minimum_gain_split` and `subspace_dim` parameters + (#1887). + + * Decision tree parameter `print_training_error` deprecated in favor of + `print_training_accuracy`. + + * `output` option changed to `predictions` for adaboost and perceptron + binding. Old options are now deprecated and will be preserved until mlpack + 4.0.0 (#1882). + + * Concatenated ReLU layer (#1843). + + * Accelerate NormalizeLabels function using hashing instead of linear search + (see `src/mlpack/core/data/normalize_labels_impl.hpp`) (#1780). + + * Add `ConfusionMatrix()` function for checking performance of classifiers + (#1798). + + * Install ensmallen headers when it is downloaded during build (#1900). + +### mlpack 3.1.0 +###### 2019-04-25 + * Add DiagonalGaussianDistribution and DiagonalGMM classes to speed up the + diagonal covariance computation and deprecate DiagonalConstraint (#1666). + + * Add kernel density estimation (KDE) implementation with bindings to other + languages (#1301). + + * Where relevant, all models with a `Train()` method now return a `double` + value representing the goodness of fit (i.e. final objective value, error, + etc.) (#1678). + + * Add implementation for linear support vector machine (see + `src/mlpack/methods/linear_svm`). + + * Change DBSCAN to use PointSelectionPolicy and add OrderedPointSelection (#1625). + + * Residual block support (#1594). + + * Bidirectional RNN (#1626). + + * Dice loss layer (#1674, #1714) and hard sigmoid layer (#1776). + + * `output` option changed to `predictions` and `output_probabilities` to + `probabilities` for Naive Bayes binding (`mlpack_nbc`/`nbc()`). Old options + are now deprecated and will be preserved until mlpack 4.0.0 (#1616). + + * Add support for Diagonal GMMs to HMM code (#1658, #1666). This can provide + large speedup when a diagonal GMM is acceptable as an emission probability + distribution. + + * Python binding improvements: check parameter type (#1717), avoid copying + Pandas dataframes (#1711), handle Pandas Series objects (#1700). + +### mlpack 3.0.4 +###### 2018-11-13 + * Bump minimum CMake version to 3.3.2. + + * CMake fixes for Ninja generator by Marc Espie. + +### mlpack 3.0.3 +###### 2018-07-27 + * Fix Visual Studio compilation issue (#1443). + + * Allow running local_coordinate_coding binding with no initial_dictionary + parameter when input_model is not specified (#1457). + + * Make use of OpenMP optional via the CMake 'USE_OPENMP' configuration + variable (#1474). + + * Accelerate FNN training by 20-30% by avoiding redundant calculations + (#1467). + + * Fix math::RandomSeed() usage in tests (#1462, #1440). + + * Generate better Python setup.py with documentation (#1460). + +### mlpack 3.0.2 +###### 2018-06-08 + * Documentation generation fixes for Python bindings (#1421). + + * Fix build error for man pages if command-line bindings are not being built + (#1424). + + * Add 'shuffle' parameter and Shuffle() method to KFoldCV (#1412). This will + shuffle the data when the object is constructed, or when Shuffle() is + called. + + * Added neural network layers: AtrousConvolution (#1390), Embedding (#1401), + and LayerNorm (layer normalization) (#1389). + + * Add Pendulum environment for reinforcement learning (#1388) and update + Mountain Car environment (#1394). + +### mlpack 3.0.1 +###### 2018-05-10 + * Fix intermittently failing tests (#1387). + + * Add big-batch SGD (BBSGD) optimizer in + src/mlpack/core/optimizers/bigbatch_sgd/ (#1131). + + * Fix simple compiler warnings (#1380, #1373). + + * Simplify NeighborSearch constructor and Train() overloads (#1378). + + * Add warning for OpenMP setting differences (#1358/#1382). When mlpack is + compiled with OpenMP but another application is not (or vice versa), a + compilation warning will now be issued. + + * Restructured loss functions in src/mlpack/methods/ann/ (#1365). + + * Add environments for reinforcement learning tests (#1368, #1370, #1329). + + * Allow single outputs for multiple timestep inputs for recurrent neural + networks (#1348). + + * Add He and LeCun normal initializations for neural networks (#1342). + Neural networks: add He and LeCun normal initializations (#1342), add FReLU + and SELU activation functions (#1346, #1341), add alpha-dropout (#1349). + +### mlpack 3.0.0 +###### 2018-03-30 + * Speed and memory improvements for DBSCAN. --single_mode can now be used for + situations where previously RAM usage was too high. + + * Bump minimum required version of Armadillo to 6.500.0. + + * Add automatically generated Python bindings. These have the same interface + as the command-line programs. + + * Add deep learning infrastructure in src/mlpack/methods/ann/. + + * Add reinforcement learning infrastructure in + src/mlpack/methods/reinforcement_learning/. + + * Add optimizers: AdaGrad, CMAES, CNE, FrankeWolfe, GradientDescent, + GridSearch, IQN, Katyusha, LineSearch, ParallelSGD, SARAH, SCD, SGDR, + SMORMS3, SPALeRA, SVRG. + + * Add hyperparameter tuning infrastructure and cross-validation infrastructure + in src/mlpack/core/cv/ and src/mlpack/core/hpt/. + + * Fix bug in mean shift. + + * Add random forests (see src/mlpack/methods/random_forest). + + * Numerous other bugfixes and testing improvements. + + * Add randomized Krylov SVD and Block Krylov SVD. + +### mlpack 2.2.5 +###### 2017-08-25 + * Compilation fix for some systems (#1082). + + * Fix PARAM_INT_OUT() (#1100). + +### mlpack 2.2.4 +###### 2017-07-18 + * Speed and memory improvements for DBSCAN. --single_mode can now be used for + situations where previously RAM usage was too high. + + * Fix bug in CF causing incorrect recommendations. + +### mlpack 2.2.3 +###### 2017-05-24 + * Bug fix for --predictions_file in mlpack_decision_tree program. + +### mlpack 2.2.2 +###### 2017-05-04 + * Install backwards-compatibility mlpack_allknn and mlpack_allkfn programs; + note they are deprecated and will be removed in mlpack 3.0.0 (#992). + + * Fix RStarTree bug that surfaced on OS X only (#964). + + * Small fixes for MiniBatchSGD and SGD and tests. + +### mlpack 2.2.1 +###### 2017-04-13 + * Compilation fix for mlpack_nca and mlpack_test on older Armadillo versions + (#984). + +### mlpack 2.2.0 +###### 2017-03-21 + * Bugfix for mlpack_knn program (#816). + + * Add decision tree implementation in methods/decision_tree/. This is very + similar to a C4.5 tree learner. + + * Add DBSCAN implementation in methods/dbscan/. + + * Add support for multidimensional discrete distributions (#810, #830). + + * Better output for Log::Debug/Log::Info/Log::Warn/Log::Fatal for Armadillo + objects (#895, #928). + + * Refactor categorical CSV loading with boost::spirit for faster loading + (#681). + +### mlpack 2.1.1 +###### 2016-12-22 + * HMMs now use random initialization; this should fix some convergence issues + (#828). + + * HMMs now initialize emissions according to the distribution of observations + (#833). + + * Minor fix for formatted output (#814). + + * Fix DecisionStump to properly work with any input type. + +### mlpack 2.1.0 +###### 2016-10-31 + * Fixed CoverTree to properly handle single-point datasets. + + * Fixed a bug in CosineTree (and thus QUIC-SVD) that caused split failures for + some datasets (#717). + + * Added mlpack_preprocess_describe program, which can be used to print + statistics on a given dataset (#742). + + * Fix prioritized recursion for k-furthest-neighbor search (mlpack_kfn and the + KFN class), leading to orders-of-magnitude speedups in some cases. + + * Bump minimum required version of Armadillo to 4.200.0. + + * Added simple Gradient Descent optimizer, found in + src/mlpack/core/optimizers/gradient_descent/ (#792). + + * Added approximate furthest neighbor search algorithms QDAFN and + DrusillaSelect in src/mlpack/methods/approx_kfn/, with command-line program + mlpack_approx_kfn. + +### mlpack 2.0.3 +###### 2016-07-21 + * Added multiprobe LSH (#691). The parameter 'T' to LSHSearch::Search() can + now be used to control the number of extra bins that are probed, as can the + -T (--num_probes) option to mlpack_lsh. + + * Added the Hilbert R tree to src/mlpack/core/tree/rectangle_tree/ (#664). It + can be used as the typedef HilbertRTree, and it is now an option in the + mlpack_knn, mlpack_kfn, mlpack_range_search, and mlpack_krann command-line + programs. + + * Added the mlpack_preprocess_split and mlpack_preprocess_binarize programs, + which can be used for preprocessing code (#650, #666). + + * Added OpenMP support to LSHSearch and mlpack_lsh (#700). + +### mlpack 2.0.2 +###### 2016-06-20 + * Added the function LSHSearch::Projections(), which returns an arma::cube + with each projection table in a slice (#663). Instead of Projection(i), you + should now use Projections().slice(i). + + * A new constructor has been added to LSHSearch that creates objects using + projection tables provided in an arma::cube (#663). + + * Handle zero-variance dimensions in DET (#515). + + * Add MiniBatchSGD optimizer (src/mlpack/core/optimizers/minibatch_sgd/) and + allow its use in mlpack_logistic_regression and mlpack_nca programs. + + * Add better backtrace support from Grzegorz Krajewski for Log::Fatal messages + when compiled with debugging and profiling symbols. This requires libbfd + and libdl to be present during compilation. + + * CosineTree test fix from Mikhail Lozhnikov (#358). + + * Fixed HMM initial state estimation (#600). + + * Changed versioning macros __MLPACK_VERSION_MAJOR, __MLPACK_VERSION_MINOR, + and __MLPACK_VERSION_PATCH to MLPACK_VERSION_MAJOR, MLPACK_VERSION_MINOR, + and MLPACK_VERSION_PATCH. The old names will remain in place until + mlpack 3.0.0. + + * Renamed mlpack_allknn, mlpack_allkfn, and mlpack_allkrann to mlpack_knn, + mlpack_kfn, and mlpack_krann. The mlpack_allknn, mlpack_allkfn, and + mlpack_allkrann programs will remain as copies until mlpack 3.0.0. + + * Add --random_initialization option to mlpack_hmm_train, for use when no + labels are provided. + + * Add --kill_empty_clusters option to mlpack_kmeans and KillEmptyClusters + policy for the KMeans class (#595, #596). + +### mlpack 2.0.1 +###### 2016-02-04 + * Fix CMake to properly detect when MKL is being used with Armadillo. + + * Minor parameter handling fixes to mlpack_logistic_regression (#504, #505). + + * Properly install arma_config.hpp. + + * Memory handling fixes for Hoeffding tree code. + + * Add functions that allow changing training-time parameters to HoeffdingTree + class. + + * Fix infinite loop in sparse coding test. + + * Documentation spelling fixes (#501). + + * Properly handle covariances for Gaussians with large condition number + (#496), preventing GMMs from filling with NaNs during training (and also + HMMs that use GMMs). + + * CMake fixes for finding LAPACK and BLAS as Armadillo dependencies when ATLAS + is used. + + * CMake fix for projects using mlpack's CMake configuration from elsewhere + (#512). + +### mlpack 2.0.0 +###### 2015-12-24 + * Removed overclustering support from k-means because it is not well-tested, + may be buggy, and is (I think) unused. If this was support you were using, + open a bug or get in touch with us; it would not be hard for us to + reimplement it. + + * Refactored KMeans to allow different types of Lloyd iterations. + + * Added implementations of k-means: Elkan's algorithm, Hamerly's algorithm, + Pelleg-Moore's algorithm, and the DTNN (dual-tree nearest neighbor) + algorithm. + + * Significant acceleration of LRSDP via the use of accu(a % b) instead of + trace(a * b). + + * Added MatrixCompletion class (matrix_completion), which performs nuclear + norm minimization to fill unknown values of an input matrix. + + * No more dependence on Boost.Random; now we use C++11 STL random support. + + * Add softmax regression, contributed by Siddharth Agrawal and QiaoAn Chen. + + * Changed NeighborSearch, RangeSearch, FastMKS, LSH, and RASearch API; these + classes now take the query sets in the Search() method, instead of in the + constructor. + + * Use OpenMP, if available. For now OpenMP support is only available in the + DET training code. + + * Add support for predicting new test point values to LARS and the + command-line 'lars' program. + + * Add serialization support for Perceptron and LogisticRegression. + + * Refactor SoftmaxRegression to predict into an arma::Row object, and + add a softmax_regression program. + + * Refactor LSH to allow loading and saving of models. + + * ToString() is removed entirely (#487). + + * Add --input_model_file and --output_model_file options to appropriate + machine learning algorithms. + + * Rename all executables to start with an "mlpack" prefix (#229). + + * Add HoeffdingTree and mlpack_hoeffding_tree, an implementation of the + streaming decision tree methodology from Domingos and Hulten in 2000. + +### mlpack 1.0.12 +###### 2015-01-07 + * Switch to 3-clause BSD license (from LGPL). + +### mlpack 1.0.11 +###### 2014-12-11 + * Proper handling of dimension calculation in PCA. + + * Load parameter vectors properly for LinearRegression models. + + * Linker fixes for AugLagrangian specializations under Visual Studio. + + * Add support for observation weights to LinearRegression. + + * MahalanobisDistance<> now takes the root of the distance by default and + therefore satisfies the triangle inequality (TakeRoot now defaults to true). + + * Better handling of optional Armadillo HDF5 dependency. + + * Fixes for numerous intermittent test failures. + + * math::RandomSeed() now sets the random seed for recent (>=3.930) Armadillo + versions. + + * Handle Newton method convergence better for + SparseCoding::OptimizeDictionary() and make maximum iterations a parameter. + + * Known bug: CosineTree construction may fail in some cases on i386 systems + (#358). + +### mlpack 1.0.10 +###### 2014-08-29 + * Bugfix for NeighborSearch regression which caused very slow allknn/allkfn. + Speeds are now restored to approximately 1.0.8 speeds, with significant + improvement for the cover tree (#347). + + * Detect dependencies correctly when ARMA_USE_WRAPPER is not being defined + (i.e., libarmadillo.so does not exist). + + * Bugfix for compilation under Visual Studio (#348). + +### mlpack 1.0.9 +###### 2014-07-28 + * GMM initialization is now safer and provides a working GMM when constructed + with only the dimensionality and number of Gaussians (#301). + + * Check for division by 0 in Forward-Backward Algorithm in HMMs (#301). + + * Fix MaxVarianceNewCluster (used when re-initializing clusters for k-means) + (#301). + + * Fixed implementation of Viterbi algorithm in HMM::Predict() (#303). + + * Significant speedups for dual-tree algorithms using the cover tree (#235, + #314) including a faster implementation of FastMKS. + + * Fix for LRSDP optimizer so that it compiles and can be used (#312). + + * CF (collaborative filtering) now expects users and items to be zero-indexed, + not one-indexed (#311). + + * CF::GetRecommendations() API change: now requires the number of + recommendations as the first parameter. The number of users in the local + neighborhood should be specified with CF::NumUsersForSimilarity(). + + * Removed incorrect PeriodicHRectBound (#58). + + * Refactor LRSDP into LRSDP class and standalone function to be optimized + (#305). + + * Fix for centering in kernel PCA (#337). + + * Added simulated annealing (SA) optimizer, contributed by Zhihao Lou. + + * HMMs now support initial state probabilities; these can be set in the + constructor, trained, or set manually with HMM::Initial() (#302). + + * Added Nyström method for kernel matrix approximation by Marcus Edel. + + * Kernel PCA now supports using Nyström method for approximation. + + * Ball trees now work with dual-tree algorithms, via the BallBound<> bound + structure (#307); fixed by Yash Vadalia. + + * The NMF class is now AMF<>, and supports far more types of factorizations, + by Sumedh Ghaisas. + + * A QUIC-SVD implementation has returned, written by Siddharth Agrawal and + based on older code from Mudit Gupta. + + * Added perceptron and decision stump by Udit Saxena (these are weak learners + for an eventual AdaBoost class). + + * Sparse autoencoder added by Siddharth Agrawal. + +### mlpack 1.0.8 +###### 2014-01-06 + * Memory leak in NeighborSearch index-mapping code fixed (#298). + + * GMMs can be trained using the existing model as a starting point by + specifying an additional boolean parameter to GMM::Estimate() (#296). + + * Logistic regression implementation added in methods/logistic_regression (see + also #293). + + * L-BFGS optimizer now returns its function via Function(). + + * Version information is now obtainable via mlpack::util::GetVersion() or the + __MLPACK_VERSION_MAJOR, __MLPACK_VERSION_MINOR, and __MLPACK_VERSION_PATCH + macros (#297). + + * Fix typos in allkfn and allkrann output. + +### mlpack 1.0.7 +###### 2013-10-04 + * Cover tree support for range search (range_search), rank-approximate nearest + neighbors (allkrann), minimum spanning tree calculation (emst), and FastMKS + (fastmks). + + * Dual-tree FastMKS implementation added and tested. + + * Added collaborative filtering package (cf) that can provide recommendations + when given users and items. + + * Fix for correctness of Kernel PCA (kernel_pca) (#270). + + * Speedups for PCA and Kernel PCA (#198). + + * Fix for correctness of Neighborhood Components Analysis (NCA) (#279). + + * Minor speedups for dual-tree algorithms. + + * Fix for Naive Bayes Classifier (nbc) (#269). + + * Added a ridge regression option to LinearRegression (linear_regression) + (#286). + + * Gaussian Mixture Models (gmm::GMM<>) now support arbitrary covariance matrix + constraints (#283). + + * MVU (mvu) removed because it is known to not work (#183). + + * Minor updates and fixes for kernels (in mlpack::kernel). + +### mlpack 1.0.6 +###### 2013-06-13 + * Minor bugfix so that FastMKS gets built. + +### mlpack 1.0.5 +###### 2013-05-01 + * Speedups of cover tree traversers (#235). + + * Addition of rank-approximate nearest neighbors (RANN), found in + src/mlpack/methods/rann/. + + * Addition of fast exact max-kernel search (FastMKS), found in + src/mlpack/methods/fastmks/. + + * Fix for EM covariance estimation; this should improve GMM training time. + + * More parameters for GMM estimation. + + * Force GMM and GaussianDistribution covariance matrices to be positive + definite, so that training converges much more often. + + * Add parameter for the tolerance of the Baum-Welch algorithm for HMM + training. + + * Fix for compilation with clang compiler. + + * Fix for k-furthest-neighbor-search. + +### mlpack 1.0.4 +###### 2013-02-08 + * Force minimum Armadillo version to 2.4.2. + + * Better output of class types to streams; a class with a ToString() method + implemented can be sent to a stream with operator<<. + + * Change return type of GMM::Estimate() to double (#257). + + * Style fixes for k-means and RADICAL. + + * Handle size_t support correctly with Armadillo 3.6.2 (#258). + + * Add locality-sensitive hashing (LSH), found in src/mlpack/methods/lsh/. + + * Better tests for SGD (stochastic gradient descent) and NCA (neighborhood + components analysis). + +### mlpack 1.0.3 +###### 2012-09-16 + + * Remove internal sparse matrix support because Armadillo 3.4.0 now includes + it. When using Armadillo versions older than 3.4.0, sparse matrix support + is not available. + + * NCA (neighborhood components analysis) now support an arbitrary optimizer + (#245), including stochastic gradient descent (#249). + +### mlpack 1.0.2 +###### 2012-08-15 + * Added density estimation trees, found in src/mlpack/methods/det/. + + * Added non-negative matrix factorization, found in src/mlpack/methods/nmf/. + + * Added experimental cover tree implementation, found in + src/mlpack/core/tree/cover_tree/ (#157). + + * Better reporting of boost::program_options errors (#225). + + * Fix for timers on Windows (#212, #211). + + * Fix for allknn and allkfn output (#204). + + * Sparse coding dictionary initialization is now a template parameter (#220). + +### mlpack 1.0.1 +###### 2012-03-03 + * Added kernel principal components analysis (kernel PCA), found in + src/mlpack/methods/kernel_pca/ (#74). + + * Fix for Lovasz-Theta AugLagrangian tests (#182). + + * Fixes for allknn output (#185, #186). + + * Added range search executable (#192). + + * Adapted citations in documentation to BibTeX; no citations in -h output + (#195). + + * Stop use of 'const char*' and prefer 'std::string' (#176). + + * Support seeds for random numbers (#177). + +### mlpack 1.0.0 +###### 2011-12-17 + * Initial release. See any resolved tickets numbered less than #196 or + execute this query: + http://www.mlpack.org/trac/query?status=closed&milestone=mlpack+1.0.0 diff --git a/doc/optimizers.md b/doc/optimizers.md index a34195145..2558a5bdb 100644 --- a/doc/optimizers.md +++ b/doc/optimizers.md @@ -696,35 +696,40 @@ matrix within an iterative procedure using the covariance matrix. #### Constructors - * `CMAES<`_`SelectionPolicyType`_`>()` - * `CMAES<`_`SelectionPolicyType`_`>(`_`lambda, lowerBound, upperBound`_`)` - * `CMAES<`_`SelectionPolicyType`_`>(`_`lambda, lowerBound, upperBound, batchSize`_`)` - * `CMAES<`_`SelectionPolicyType`_`>(`_`lambda, lowerBound, upperBound, batchSize, maxIterations, tolerance, selectionPolicy`_`)` + * `CMAES<`_`SelectionPolicyType, TransformationPolicyType`_`>()` + * `CMAES<`_`SelectionPolicyType, TransformationPolicyType`_`>(`_`lambda, transformationPolicy`_`)` + * `CMAES<`_`SelectionPolicyType, TransformationPolicyType`_`>(`_`lambda, transformationPolicy, batchSize`_`)` + * `CMAES<`_`SelectionPolicyType, TransformationPolicyType`_`>(`_`lambda, transformationPolicy, batchSize, maxIterations, tolerance, selectionPolicy`_`)` + * `CMAES<`_`SelectionPolicyType, TransformationPolicyType`_`>(`_`lambda, transformationPolicy, batchSize, maxIterations, tolerance, selectionPolicy, stepSize`_`)` The _`SelectionPolicyType`_ template parameter refers to the strategy used to compute the (approximate) objective function. The `FullSelection` and `RandomSelection` classes are available for use; custom behavior can be achieved by implementing a class with the same method signatures. +The _`TransformationPolicyType`_ template parameter refers to transformation +strategy used to map decision variables to the desired domain during fitness +evaluation and optimization termination. The `EmptyTransformation` and +`BoundaryBoxConstraint` classes are available for use; custom behavior can be +achieved by implementing a class with the same method signatures. For convenience the following types can be used: - * **`CMAES<>`** (equivalent to `CMAES`): uses all separable functions to compute objective - * **`ApproxCMAES`** (equivalent to `CMAES`): uses a small amount of separable functions to compute approximate objective + * **`CMAES<>`** (equivalent to `CMAES>`): uses all separable functions to compute objective + * **`ApproxCMAES<>`** (equivalent to `CMAES>`): uses a small amount of separable functions to compute approximate objective #### Attributes | **type** | **name** | **description** | **default** | |----------|----------|-----------------|-------------| | `size_t` | **`lambda`** | The population size (0 uses a default size). | `0` | -| `double` | **`lowerBound`** | Lower bound of decision variables. | `-10.0` | -| `double` | **`upperBound`** | Upper bound of decision variables. | `10.0` | +| `TransformationPolicyType` | **`transformationPolicy`** | Instantiated transformation policy used to map the coordinates to the desired domain. | `TransformationPolicyType()` | | `size_t` | **`batchSize`** | Batch size to use for the objective calculation. | `32` | | `size_t` | **`maxIterations`** | Maximum number of iterations. | `1000` | | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` | | `SelectionPolicyType` | **`selectionPolicy`** | Instantiated selection policy used to calculate the objective. | `SelectionPolicyType()` | Attributes of the optimizer may also be changed via the member methods -`Lambda()`, `LowerBound()`, `UpperBound()`, `BatchSize()`, `MaxIterations()`, +`Lambda()`, `TransformationPolicy()`, `BatchSize()`, `MaxIterations()`, `Tolerance()`, and `SelectionPolicy()`. The `selectionPolicy` attribute allows an instantiated `SelectionPolicyType` to @@ -733,6 +738,12 @@ the option is not relevant when the `CMAES<>` optimizer type is being used; the `RandomSelection` policy has the constructor `RandomSelection(`_`fraction`_`)` where _`fraction`_ specifies the percentage of separable functions to use to estimate the objective function. +The `transformationPolicy` attribute allows an instantiated +`TransformationPolicyType` to be given. The `EmptyTransformation<`_`MatType`_`>` +has no need to be instantiated. `BoundaryBoxConstraint<`_`MatType`_`>` policy has +the constructor `BoundaryBoxConstraint(`_`lowerBound, upperBound`_`)` +where _`lowerBound`_ and _`lowerBound`_ are the lower bound and upper bound of +the coordinates respectively. #### Examples: @@ -744,12 +755,13 @@ estimate the objective function. RosenbrockFunction f; arma::mat coordinates = f.GetInitialPoint(); -// CMAES with the FullSelection policy. -CMAES<> optimizer(0, -1, 1, 32, 200, 1e-4); +// CMAES with the FullSelection and BoundaryBoxConstraint policies. +BoundaryBoxConstraint b(-1, 1); +CMAES optimizer(0, b, 32, 200, 1e-4); optimizer.Optimize(f, coordinates); -// CMAES with the RandomSelection policy. -ApproxCMAES<> approxOptimizer(0, -1, 1. 32, 200, 1e-4); +// CMAES with the RandomSelection and BoundaryBoxConstraint policies. +ApproxCMAES> cmaes(0, b, 32, 200, 1e-4); approxOptimizer.Optimize(f, coordinates); ``` diff --git a/include/ensmallen_bits/cmaes/cmaes.hpp b/include/ensmallen_bits/cmaes/cmaes.hpp index 735de3d54..934235c3b 100644 --- a/include/ensmallen_bits/cmaes/cmaes.hpp +++ b/include/ensmallen_bits/cmaes/cmaes.hpp @@ -105,14 +105,14 @@ class CMAES * objective. * @param stepSize Starting sigma/step size (will be modified). */ - CMAES(const size_t lambda = 0, - const double lowerBound = -10, - const double upperBound = 10, - const size_t batchSize = 32, - const size_t maxIterations = 1000, - const double tolerance = 1e-5, - const SelectionPolicyType & selectionPolicy = SelectionPolicyType(), - double stepSize = 1); + ens_deprecated CMAES(const size_t lambda = 0, + const double lowerBound = -10, + const double upperBound = 10, + const size_t batchSize = 32, + const size_t maxIterations = 1000, + const double tolerance = 1e-5, + const SelectionPolicyType& selectionPolicy = SelectionPolicyType(), + double stepSize = 1); /** * Optimize the given function using CMA-ES. The given starting point will be @@ -190,7 +190,8 @@ class CMAES SelectionPolicyType selectionPolicy; //! The transformationPolicy used to map coordinates to the suitable domain - //! while evaluating fitness. + //! while evaluating fitness. This mapping is also done after optimization + //! has completed. TransformationPolicyType transformationPolicy; double stepSize; diff --git a/include/ensmallen_bits/cmaes/cmaes_impl.hpp b/include/ensmallen_bits/cmaes/cmaes_impl.hpp index 67acef983..f7dd38bae 100644 --- a/include/ensmallen_bits/cmaes/cmaes_impl.hpp +++ b/include/ensmallen_bits/cmaes/cmaes_impl.hpp @@ -23,20 +23,20 @@ /* This partial specialization is used to throw an exception when the TransformationPolicyType is EmptyTransformation and call a - constructor with paramters 'lowerBound' and 'upperBound' otherwise. + constructor with parameters 'lowerBound' and 'upperBound' otherwise. This shall be removed when the deprecated constructor is removed in the next major version of ensmallen. */ template -struct notEmptyTransformation : std::true_type { - void assign(T1& obj, double lowerBound, double upperBound) { +struct NotEmptyTransformation : std::true_type { + void Assign(T1& obj, double lowerBound, double upperBound) { obj = T1(lowerBound, upperBound); } }; -template typename T, typename... A, typename... B> -struct notEmptyTransformation, T> : std::false_type { - void assign(T& obj, double lowerBound, double upperBound) { +template class T, typename... A, typename... B> +struct NotEmptyTransformation, T> : std::false_type { + void Assign(T& obj, double lowerBound, double upperBound) { throw std::logic_error("TransformationPolicyType is EmptyTransformation"); } }; @@ -79,8 +79,8 @@ CMAES::CMAES(const size_t lambda, { Warn << "This is a deprecated constructor and will be removed in a " "future version of ensmallen" << std::endl; - notEmptyTransformation> d; - d.assign(transformationPolicy, lowerBound, upperBound); + NotEmptyTransformation> d; + d.Assign(transformationPolicy, lowerBound, upperBound); } @@ -124,8 +124,10 @@ typename MatType::elem_type CMAES Date: Fri, 28 Apr 2023 00:55:39 +0530 Subject: [PATCH 07/34] update HISTORY.md --- HISTORY.md | 1257 +++++++++++++++------------------------------------- 1 file changed, 347 insertions(+), 910 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index e48c2c429..989fdbe6c 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,1036 +1,473 @@ -### mlpack ?.?.? +### ensmallen ?.??.?: "???" ###### ????-??-?? - * Fix mapping of categorical data for Julia bindings (#3305). + * Fix the current CMAES inconsistencies + ([#351](https://github.com/mlpack/ensmallen/pull/351)). + * Fix CNE test tolerances + ([#360](https://github.com/mlpack/ensmallen/pull/360)). -### mlpack 4.0.0 -###### 2022-10-23 - * Bump C++ standard requirement to C++14 (#3233). +### ensmallen 2.19.1: "Eight Ball Deluxe" +###### 2023-01-30 + * Avoid deprecation warnings in Armadillo 11.2+ + ([#347](https://github.com/mlpack/ensmallen/pull/347)). - * Fix `Perceptron` to work with cross-validation framework (#3190). +### ensmallen 2.19.0: "Eight Ball Deluxe" +###### 2022-04-06 + * Added DemonSGD and DemonAdam optimizers + ([#211](https://github.com/mlpack/ensmallen/pull/211)). - * Migrate from boost tests to Catch2 framework (#2523), (#2584). + * Fix bug with Adam-like optimizers not resetting when `resetPolicy` is `true`. + ([#340](https://github.com/mlpack/ensmallen/pull/340)). - * Bump minimum armadillo version from 8.400 to 9.800 (#3043), (#3048). + * Add Yogi optimizer + ([#232](https://github.com/mlpack/ensmallen/pull/232)). - * Adding a copy constructor in the Convolution layer (#3067). + * Add AdaBelief optimizer + ([#233](https://github.com/mlpack/ensmallen/pull/233)). - * Replace `boost::spirit` parser by a local efficient implementation (#2942). + * Add AdaSqrt optimizer + ([#234](https://github.com/mlpack/ensmallen/pull/234)). + + * Bump check for minimum supported version of Armadillo + ([#342](https://github.com/mlpack/ensmallen/pull/342)). - * Disable correctly the autodownloader + fix tests stability (#3076). +### ensmallen 2.18.2: "Fairmount Bagel" +###### 2022-02-13 + * Update Catch2 to 2.13.8 + ([#336](https://github.com/mlpack/ensmallen/pull/336)). - * Replace `boost::any` with `core::v2::any` or `std::any` if available (#3006). + * Fix epoch timing output + ([#337](https://github.com/mlpack/ensmallen/pull/337)). - * Remove old non used Boost headers (#3005). +### ensmallen 2.18.1: "Fairmount Bagel" +###### 2021-11-19 + * Accelerate SGD test time + ([#330](https://github.com/mlpack/ensmallen/pull/300)). - * Replace `boost::enable_if` with `std::enable_if` (#2998). + * Fix potential infinite loop in CMAES + ([#331](https://github.com/mlpack/ensmallen/pull/331)). - * Replace `boost::is_same` with `std::is_same` (#2993). + * Fix SCD partial gradient test + ([#332](https://github.com/mlpack/ensmallen/pull/332)). - * Remove invalid option for emsmallen and STB (#2960). +### ensmallen 2.18.0: "Fairmount Bagel" +###### 2021-10-20 + * Add gradient value clipping and gradient norm scaling callback + ([#315](https://github.com/mlpack/ensmallen/pull/315)). - * Check for armadillo dependencies before downloading armadillo (#2954). + * Remove superfluous CMake option to build the tests + ([#313](https://github.com/mlpack/ensmallen/pull/313)). - * Disable the usage of autodownloader by default (#2953). + * Bump minimum Armadillo version to 9.800 + ([#318](https://github.com/mlpack/ensmallen/pull/318)). - * Install dependencies downloaded with the autodownloader (#2952). - - * Download older Boost if the compiler is old (#2940). + * Update Catch2 to 2.13.7 + ([#322](https://github.com/mlpack/ensmallen/pull/322)). - * Add support for embedded systems (#2531). + * Remove redundant template argument for C++20 compatibility + ([#324](https://github.com/mlpack/ensmallen/pull/324)). - * Build mlpack executable statically if the library is statically linked (#2931). + * Fix MOEAD test stability + ([#327](https://github.com/mlpack/ensmallen/pull/327)). - * Fix cover tree loop bug on embedded arm systems (#2869). +### ensmallen 2.17.0: "Pachis Din Me Pesa Double" +###### 2021-07-06 + * CheckArbitraryFunctionTypeAPI extended for MOO support + ([#283](https://github.com/mlpack/ensmallen/pull/283)). - * Fix a LAPACK bug in `FindArmadillo.cmake` (#2929). + * Refactor NSGA2 + ([#263](https://github.com/mlpack/ensmallen/pull/263), + [#304](https://github.com/mlpack/ensmallen/pull/304)). - * Add an autodownloader to get mlpack dependencies (#2927). + * Add Indicators for Multiobjective optimizers + ([#285](https://github.com/mlpack/ensmallen/pull/285)). - * Remove Coverage files and configurations from CMakeLists (#2866). + * Make Callback flexible for MultiObjective Optimizers + ([#289](https://github.com/mlpack/ensmallen/pull/289)). - * Added `Multi Label Soft Margin Loss` loss function for neural networks - (#2345). + * Add ZDT Test Suite + ([#273](https://github.com/mlpack/ensmallen/pull/273)). - * Added Decision Tree Regressor (#2905). It can be used using the class - `mlpack::tree::DecisionTreeRegressor`. It is accessible only though C++. + * Add MOEA-D/DE Optimizer + ([#269](https://github.com/mlpack/ensmallen/pull/269)). - * Added dict-style inspection of mlpack models in python bindings (#2868). + * Introduce Policy Methods for MOEA/D-DE + ([#293](https://github.com/mlpack/ensmallen/pull/293)). - * Added Extra Trees Algorithm (#2883). Currently, it can be used using the - class `mlpack::tree::ExtraTrees`, but only through C++. + * Add Das-Dennis weight initialization method + ([#295](https://github.com/mlpack/ensmallen/pull/295)). - * Add Flatten T Swish activation function (`flatten-t-swish.hpp`) + * Add Dirichlet Weight Initialization + ([#296](https://github.com/mlpack/ensmallen/pull/296)). - * Added warm start feature to Random Forest (#2881); this feature is - accessible from mlpack's bindings to different languages. + * Improved installation and compilation instructions + ([#300](https://github.com/mlpack/ensmallen/pull/300)). - * Added Pixel Shuffle layer (#2563). + * Disable building the tests by default for faster installation + ([#303](https://github.com/mlpack/ensmallen/pull/303)). - * Add "check_input_matrices" option to python bindings that checks - for NaN and inf values in all the input matrices (#2787). + * Modify matrix initialisation to take into account + default element zeroing in Armadillo 10.5 + ([#305](https://github.com/mlpack/ensmallen/pull/305)). - * Add Adjusted R squared functionality to R2Score::Evaluate (#2624). +### ensmallen 2.16.2: "Severely Dented Can Of Polyurethane" +###### 2021-03-24 + * Fix CNE test trials + ([#267](https://github.com/mlpack/ensmallen/pull/267)). - * Disabled all the bindings by default in CMake (#2782). + * Update Catch2 to 2.13.4 + ([#268](https://github.com/mlpack/ensmallen/pull/268)). - * Added an implementation to Stratify Data (#2671). + * Fix typos in documentation + ([#270](https://github.com/mlpack/ensmallen/pull/270), + [#271](https://github.com/mlpack/ensmallen/pull/271)). - * Add `BUILD_DOCS` CMake option to control whether Doxygen documentation is - built (default ON) (#2730). + * Add clarifying comments in problems/ implementations + ([#276](https://github.com/mlpack/ensmallen/pull/276)). - * Add Triplet Margin Loss function (#2762). +### ensmallen 2.16.1: "Severely Dented Can Of Polyurethane" +###### 2021-03-02 + * Fix test compilation issue when `ENS_USE_OPENMP` is set + ([#255](https://github.com/mlpack/ensmallen/pull/255)). - * Add finalizers to Julia binding model types to fix memory handling (#2756). + * Fix CNE initial population generation to use normal distribution + ([#258](https://github.com/mlpack/ensmallen/pull/258)). - * HMM: add functions to calculate likelihood for data stream with/without - pre-calculated emission probability (#2142). + * Fix compilation warnings + ([#259](https://github.com/mlpack/ensmallen/pull/259)). - * Replace Boost serialization library with Cereal (#2458). + * Remove `AdamSchafferFunctionN2Test` test from Adam test suite to prevent + spurious issue on some aarch64 ([#265](https://github.com/mlpack/ensmallen/pull/259)). - * Add `PYTHON_INSTALL_PREFIX` CMake option to specify installation root for - Python bindings (#2797). +### ensmallen 2.16.0: "Severely Dented Can Of Polyurethane" +###### 2021-02-11 + * Expand README with example installation and add simple example program + showing usage of the L-BFGS optimizer + ([#248](https://github.com/mlpack/ensmallen/pull/248)). - * Removed `boost::visitor` from model classes for `knn`, `kfn`, `cf`, - `range_search`, `krann`, and `kde` bindings (#2803). + * Refactor tests to increase stability and reduce random errors + ([#249](https://github.com/mlpack/ensmallen/pull/249)). - * Add k-means++ initialization strategy (#2813). +### ensmallen 2.15.1: "Why Can't I Manage To Grow Any Plants?" +###### 2020-11-05 + * Fix include order to ensure traits is loaded before reports + ([#239](https://github.com/mlpack/ensmallen/pull/239)). - * `NegativeLogLikelihood<>` now expects classes in the range `0` to - `numClasses - 1` (#2534). +### ensmallen 2.15.0: "Why Can't I Manage To Grow Any Plants?" +###### 2020-11-01 + * Make a few tests more robust + ([#228](https://github.com/mlpack/ensmallen/pull/228)). - * Add `Lambda1()`, `Lambda2()`, `UseCholesky()`, and `Tolerance()` members to - `LARS` so parameters for training can be modified (#2861). + * Add release date to version information. ([#226](https://github.com/mlpack/ensmallen/pull/226)) - * Remove unused `ElemType` template parameter from `DecisionTree` and - `RandomForest` (#2874). + * Fix typo in release script + ([#236](https://github.com/mlpack/ensmallen/pull/236)). - * Fix Python binding build when the CMake variable `USE_OPENMP` is set to - `OFF` (#2884). +### ensmallen 2.14.2: "No Direction Home" +###### 2020-08-31 + * Fix implementation of fonesca fleming problem function f1 and f2 + type usage and negative signs. ([#223](https://github.com/mlpack/ensmallen/pull/223)) - * The `mlpack_test` target is no longer built as part of `make all`. Use - `make mlpack_test` to build the tests. +### ensmallen 2.14.1: "No Direction Home" +###### 2020-08-19 + * Fix release script (remove hardcoded information, trim leading whitespaces + introduced by `wc -l` in MacOS) + ([#216](https://github.com/mlpack/ensmallen/pull/216), + [#220](https://github.com/mlpack/ensmallen/pull/220)). - * Fixes to `HoeffdingTree`: ensure that training still works when empty - constructor is used (#2964). + * Adjust tolerance for AugLagrangian convergence based on element type + ([#217](https://github.com/mlpack/ensmallen/pull/217)). - * Fix Julia model serialization bug (#2970). +### ensmallen 2.14.0: "No Direction Home" +###### 2020-08-10 + * Add NSGA2 optimizer for multi-objective functions + ([#149](https://github.com/mlpack/ensmallen/pull/149)). - * Fix `LoadCSV()` to use pre-populated `DatasetInfo` objects (#2980). + * Update automatic website update release script + ([#207](https://github.com/mlpack/ensmallen/pull/207)). - * Add `probabilities` option to softmax regression binding, to get class - probabilities for test points (#3001). + * Clarify and fix documentation for constrained optimizers + ([#201](https://github.com/mlpack/ensmallen/pull/201)). - * Fix thread safety issues in mlpack bindings to other languages (#2995). + * Fix L-BFGS convergence when starting from a minimum + ([#201](https://github.com/mlpack/ensmallen/pull/201)). - * Fix double-free of model pointers in R bindings (#3034). +* Add optimizer summary report callback + ([#213](https://github.com/mlpack/ensmallen/pull/213)). - * Fix Julia, Python, R, and Go handling of categorical data for - `decision_tree()` and `hoeffding_tree()` (#2971). +### ensmallen 2.13.0: "Automatically Automated Automation" +###### 2020-07-15 + * Fix CMake package export + ([#198](https://github.com/mlpack/ensmallen/pull/198)). - * Depend on `pkgbuild` for R bindings (#3081). - - * Replaced Numpy deprecated code in Python bindings (#3126). + * Allow early stop callback to accept a lambda function + ([#165](https://github.com/mlpack/ensmallen/pull/165)). -### mlpack 3.4.2 -###### 2020-10-26 - * Added Mean Absolute Percentage Error. +### ensmallen 2.12.1: "Stir Crazy" +###### 2020-04-20 + * Fix total number of epochs and time estimation for ProgressBar callback + ([#181](https://github.com/mlpack/ensmallen/pull/181)). - * Added Softmin activation function as layer in ann/layer. + * Handle SpSubview_col and SpSubview_row in Armadillo 9.870 + ([#194](https://github.com/mlpack/ensmallen/pull/194)). - * Fix spurious ARMA_64BIT_WORD compilation warnings on 32-bit systems (#2665). + * Minor documentation fixes + ([#197](https://github.com/mlpack/ensmallen/pull/197)). -### mlpack 3.4.1 -###### 2020-09-07 - * Fix incorrect parsing of required matrix/model parameters for command-line - bindings (#2600). +### ensmallen 2.12.0: "Stir Crazy" +###### 2020-03-28 + * Correction in the formulation of sigma in CMA-ES + ([#183](https://github.com/mlpack/ensmallen/pull/183)). - * Add manual type specification support to `data::Load()` and `data::Save()` - (#2084, #2135, #2602). + * Remove deprecated methods from PrimalDualSolver implementation + ([#185](https://github.com/mlpack/ensmallen/pull/185). - * Remove use of internal Armadillo functionality (#2596, #2601, #2602). + * Update logo ([#186](https://github.com/mlpack/ensmallen/pull/186)). -### mlpack 3.4.0 -###### 2020-09-01 - * Issue warnings when metrics produce NaNs in KFoldCV (#2595). +### ensmallen 2.11.5: "The Poster Session Is Full" +###### 2020-03-11 + * Change "mathematical optimization" term to "numerical optimization" in the + documentation ([#177](https://github.com/mlpack/ensmallen/pull/177)). - * Added bindings for _R_ during Google Summer of Code (#2556). +### ensmallen 2.11.4: "The Poster Session Is Full" +###### 2020-03-03 + * Require new HISTORY.md entry for each PR. + ([#171](https://github.com/mlpack/ensmallen/pull/171), + [#172](https://github.com/mlpack/ensmallen/pull/172), + [#175](https://github.com/mlpack/ensmallen/pull/175)). - * Added common striptype function for all bindings (#2556). + * Update/fix example documentation + ([#174](https://github.com/mlpack/ensmallen/pull/174)). - * Refactored common utility function of bindings to bindings/util (#2556). +### ensmallen 2.11.3: "The Poster Session Is Full" +###### 2020-02-19 + * Prevent spurious compiler warnings + ([#161](https://github.com/mlpack/ensmallen/pull/161)). - * Renamed InformationGain to HoeffdingInformationGain in - methods/hoeffding_trees/information_gain.hpp (#2556). + * Fix minor memory leaks + ([#167](https://github.com/mlpack/ensmallen/pull/167)). - * Added macro for changing stream of printing and warnings/errors (#2556). + * Revamp CMake configuration + ([#152](https://github.com/mlpack/ensmallen/pull/152)). - * Added Spatial Dropout layer (#2564). +### ensmallen 2.11.2: "The Poster Session Is Full" +###### 2020-01-16 + * Allow callback instantiation for SGD based optimizer + ([#138](https://github.com/mlpack/ensmallen/pull/155)). - * Force CMake to show error when it didn't find Python/modules (#2568). + * Minor test stability fixes on i386 + ([#156](https://github.com/mlpack/ensmallen/pull/156)). - * Refactor `ProgramInfo()` to separate out all the different - information (#2558). + * Fix Lookahead MaxIterations() check. + ([#159](https://github.com/mlpack/ensmallen/pull/159)). - * Add bindings for one-hot encoding (#2325). +### ensmallen 2.11.1: "The Poster Session Is Full" +###### 2019-12-28 + * Fix Lookahead Synchronization period type + ([#153](https://github.com/mlpack/ensmallen/pull/153)). - * Added Soft Actor-Critic to RL methods (#2487). +### ensmallen 2.11.0: "The Poster Session Is Full" +###### 2019-12-24 + * Add Lookahead + ([#138](https://github.com/mlpack/ensmallen/pull/138)). - * Added Categorical DQN to q_networks (#2454). + * Add AdaBound and AMSBound + ([#137](https://github.com/mlpack/ensmallen/pull/137)). - * Added N-step DQN to q_networks (#2461). +### ensmallen 2.10.5: "Fried Chicken" +###### 2019-12-13 + * SGD callback test 32-bit safety (big number) + ([#143](https://github.com/mlpack/ensmallen/pull/143)). - * Add Silhoutte Score metric and Pairwise Distances (#2406). + * Use "arbitrary" and "separable" terms in static function type checks + ([#145](https://github.com/mlpack/ensmallen/pull/145)). - * Add Go bindings for some missed models (#2460). + * Remove 'using namespace std' from `problems/` files + ([#147](https://github.com/mlpack/ensmallen/pull/147)). - * Replace boost program_options dependency with CLI11 (#2459). +### ensmallen 2.10.4: "Fried Chicken" +###### 2019-11-18 + * Add optional tests building. + ([#141](https://github.com/mlpack/ensmallen/pull/141)). - * Additional functionality for the ARFF loader (#2486); use case sensitive - categories (#2516). + * Make code samples collapsible in the documentation. + ([#140](https://github.com/mlpack/ensmallen/pull/140)). - * Add `bayesian_linear_regression` binding for the command-line, Python, - Julia, and Go. Also called "Bayesian Ridge", this is equivalent to a - version of linear regression where the regularization parameter is - automatically tuned (#2030). +### ensmallen 2.10.3: "Fried Chicken" +###### 2019-09-26 + * Fix ParallelSGD runtime bug. + ([#135](https://github.com/mlpack/ensmallen/pull/135)). - * Fix defeatist search for spill tree traversals (#2566, #1269). + * Add additional L-BFGS convergence check + ([#136](https://github.com/mlpack/ensmallen/pull/136)). - * Fix incremental training of logistic regression models (#2560). +### ensmallen 2.10.2: "Fried Chicken" +###### 2019-09-11 + * Add release script to rel/ for maintainers + ([#128](https://github.com/mlpack/ensmallen/pull/128)). - * Change default configuration of `BUILD_PYTHON_BINDINGS` to `OFF` (#2575). + * Fix Armadillo version check + ([#133](https://github.com/mlpack/ensmallen/pull/133)). -### mlpack 3.3.2 -###### 2020-06-18 - * Added Noisy DQN to q_networks (#2446). +### ensmallen 2.10.1: "Fried Chicken" +###### 2019-09-10 + * Documentation fix for callbacks + ([#129](https://github.com/mlpack/ensmallen/pull/129). - * Add Go bindings (#1884). + * Compatibility fixes for ensmallen 1.x + ([#131](https://github.com/mlpack/ensmallen/pull/131)). - * Added Dueling DQN to q_networks, Noisy linear layer to ann/layer - and Empty loss to ann/loss_functions (#2414). +### ensmallen 2.10.0: "Fried Chicken" +###### 2019-09-07 + * All `Optimize()` functions now take any matrix type; so, e.g., `arma::fmat` + or `arma::sp_mat` can be used for optimization. See the documentation for + more details ([#113](https://github.com/mlpack/ensmallen/pull/113), + [#119](https://github.com/mlpack/ensmallen/pull/119)). - * Storing and adding accessor method for action in q_learning (#2413). + * Introduce callback support. Callbacks can be appended as the last arguments + of an `Optimize()` call, and can perform custom behavior at different points + during the optimization. See the documentation for more details + ([#119](https://github.com/mlpack/ensmallen/pull/119)). - * Added accessor methods for ANN layers (#2321). + * Slight speedups for `FrankWolfe` optimizer + ([#127](https://github.com/mlpack/ensmallen/pull/127)). - * Addition of `Elliot` activation function (#2268). +### ensmallen 1.16.2: "Loud Alarm Clock" +###### 2019-08-12 + * Fix PSO return type bug + ([#126](https://github.com/mlpack/ensmallen/pull/126)). - * Add adaptive max pooling and adaptive mean pooling layers (#2195). +### ensmallen 1.16.1: "Loud Alarm Clock" +###### 2019-08-11 + * Update HISTORY.md to use Markdown links to the PR and add release names. - * Add parameter to avoid shuffling of data in preprocess_split (#2293). + * Fix PSO return type bug + ([#124](https://github.com/mlpack/ensmallen/pull/124)). - * Add `MatType` parameter to `LSHSearch`, allowing sparse matrices to be used - for search (#2395). +### ensmallen 1.16.0: "Loud Alarm Clock" +###### 2019-08-09 + * Add option to avoid computing exact objective at the end of the optimization + ([#109](https://github.com/mlpack/ensmallen/pull/109)). - * Documentation fixes to resolve Doxygen warnings and issues (#2400). + * Fix handling of curvature for BigBatchSGD + ([#118](https://github.com/mlpack/ensmallen/pull/118)). - * Add Load and Save of Sparse Matrix (#2344). + * Reduce runtime of tests + ([#118](https://github.com/mlpack/ensmallen/pull/118)). - * Add Intersection over Union (IoU) metric for bounding boxes (#2402). + * Introduce local-best particle swarm optimization, `LBestPSO`, for + unconstrained optimization problems + ([#86](https://github.com/mlpack/ensmallen/pull/86)). - * Add Non Maximal Supression (NMS) metric for bounding boxes (#2410). +### ensmallen 1.15.1: "Wrong Side Of The Road" +###### 2019-05-22 + * Fix `-Wreorder` in `qhadam` warning + ([#115](https://github.com/mlpack/ensmallen/pull/115)). - * Fix `no_intercept` and probability computation for linear SVM bindings - (#2419). + * Fix `-Wunused-private-field` warning in `spsa` + ([#115](https://github.com/mlpack/ensmallen/pull/115)). - * Fix incorrect neighbors for `k > 1` searches in `approx_kfn` binding, for - the `QDAFN` algorithm (#2448). + * Add more warning output for gcc/clang + ([#116](https://github.com/mlpack/ensmallen/pull/116)). - * Fix serialization of kernels with state for FastMKS (#2452). +### ensmallen 1.15.0: "Wrong Side Of The Road" +###### 2019-05-14 + * Added QHAdam and QHSGD optimizers + ([#81](https://github.com/mlpack/ensmallen/pull/81)). - * Add `RBF` layer in ann module to make `RBFN` architecture (#2261). +### ensmallen 1.14.4: "Difficult Crimp" +###### 2019-05-12 + * Fixes for BigBatchSGD ([#91](https://github.com/mlpack/ensmallen/pull/91)). -### mlpack 3.3.1 -###### 2020-04-29 - * Minor Julia and Python documentation fixes (#2373). +### ensmallen 1.14.3: "Difficult Crimp" +###### 2019-05-06 + * Handle `eig_sym()` failures correctly + ([#100](https://github.com/mlpack/ensmallen/pull/100)). - * Updated terminal state and fixed bugs for Pendulum environment (#2354, - #2369). +### ensmallen 1.14.2: "Difficult Crimp" +###### 2019-03-14 + * SPSA test tolerance fix + ([#97](https://github.com/mlpack/ensmallen/pull/97)). - * Added `EliSH` activation function (#2323). + * Minor documentation fixes (#95, #98). - * Add L1 Loss function (#2203). + * Fix newlines at end of file + ([#92](https://github.com/mlpack/ensmallen/pull/92)). - * Pass CMAKE_CXX_FLAGS (compilation options) correctly to Python build - (#2367). +### ensmallen 1.14.1: "Difficult Crimp" +###### 2019-03-09 + * Fixes for SPSA ([#87](https://github.com/mlpack/ensmallen/pull/87)). - * Expose ensmallen Callbacks for sparseautoencoder (#2198). + * Optimized CNE and DE ([#90](https://github.com/mlpack/ensmallen/pull/90)). + Changed initial population generation in CNE to be a normal distribution + about the given starting point, which should accelerate convergence. - * Bugfix for LARS class causing invalid read (#2374). +### ensmallen 1.14.0: "Difficult Crimp" +###### 2019-02-20 + * Add DE optimizer ([#77](https://github.com/mlpack/ensmallen/pull/77)). - * Add serialization support from Julia; use `mlpack.serialize()` and - `mlpack.deserialize()` to save and load from `IOBuffer`s. + * Fix for Cholesky decomposition in CMAES + ([#83](https://github.com/mlpack/ensmallen/pull/83)). -### mlpack 3.3.0 -###### 2020-04-07 - * Added `Normal Distribution` to `ann/dists` (#2382). +### ensmallen 1.13.2: "Coronavirus Invasion" +###### 2019-02-18 + * Minor documentation fixes ([#82](https://github.com/mlpack/ensmallen/pull/82)). - * Templated return type of `Forward function` of loss functions (#2339). +### ensmallen 1.13.1: "Coronavirus Invasion" +###### 2019-01-24 + * Fix -Wreorder warning ([#75](https://github.com/mlpack/ensmallen/pull/75)). - * Added `R2 Score` regression metric (#2323). +### ensmallen 1.13.0: "Coronavirus Invasion" +###### 2019-01-14 + * Enhance options for AugLagrangian optimizer + ([#66](https://github.com/mlpack/ensmallen/pull/66)). - * Added `poisson negative log likelihood` loss function (#2196). + * Add SPSA optimizer ([#69](https://github.com/mlpack/ensmallen/pull/69)). - * Added `huber` loss function (#2199). +### ensmallen 1.12.2: "New Year's Party" +###### 2019-01-05 + * Fix list of contributors. - * Added `mean squared logarithmic error` loss function for neural networks - (#2210). +### ensmallen 1.12.1: "New Year's Party" +###### 2019-01-03 + * Make sure all files end with newlines. - * Added `mean bias loss function` for neural networks (#2210). +### ensmallen 1.12.0: "New Year's Party" +###### 2018-12-30 + * Add link to ensmallen PDF to README.md. - * The DecisionStump class has been marked deprecated; use the `DecisionTree` - class with `NoRecursion=true` or use `ID3DecisionStump` instead (#2099). + * Minor documentation fixes. Remove too-verbose documentation from source for + each optimizer ([#61](https://github.com/mlpack/ensmallen/pull/61)). - * Added `probabilities_file` parameter to get the probabilities matrix of - AdaBoost classifier (#2050). + * Add FTML optimizer ([#48](https://github.com/mlpack/ensmallen/pull/48)). - * Fix STB header search paths (#2104). + * Add SWATS optimizer ([#42](https://github.com/mlpack/ensmallen/pull/42)). - * Add `DISABLE_DOWNLOADS` CMake configuration option (#2104). + * Add Padam optimizer ([#46](https://github.com/mlpack/ensmallen/pull/46)). - * Add padding layer in TransposedConvolutionLayer (#2082). + * Add Eve optimizer ([#45](https://github.com/mlpack/ensmallen/pull/45)). - * Fix pkgconfig generation on non-Linux systems (#2101). + * Add ResetPolicy() to SGD-like optimizers + ([#60](https://github.com/mlpack/ensmallen/pull/60)). - * Use log-space to represent HMM initial state and transition probabilities - (#2081). +### ensmallen 1.11.1: "Jet Lag" +###### 2018-11-29 + * Minor documentation fixes. - * Add functions to access parameters of `Convolution` and `AtrousConvolution` - layers (#1985). +### ensmallen 1.11.0: "Jet Lag" +###### 2018-11-28 + * Add WNGrad optimizer. - * Add Compute Error function in lars regression and changing Train function to - return computed error (#2139). + * Fix header name in documentation samples. - * Add Julia bindings (#1949). Build settings can be controlled with the - `BUILD_JULIA_BINDINGS=(ON/OFF)` and `JULIA_EXECUTABLE=/path/to/julia` CMake - parameters. +### ensmallen 1.10.1: "Corporate Catabolism" +###### 2018-11-16 + * Fixes for GridSearch optimizer. - * CMake fix for finding STB include directory (#2145). + * Include documentation with release. - * Add bindings for loading and saving images (#2019); `mlpack_image_converter` - from the command-line, `mlpack.image_converter()` from Python. +### ensmallen 1.10.0: "Corporate Catabolism" +###### 2018-10-20 + * Initial release. - * Add normalization support for CF binding (#2136). - - * Add Mish activation function (#2158). - - * Update `init_rules` in AMF to allow users to merge two initialization - rules (#2151). - - * Add GELU activation function (#2183). - - * Better error handling of eigendecompositions and Cholesky decompositions - (#2088, #1840). - - * Add LiSHT activation function (#2182). - - * Add Valid and Same Padding for Transposed Convolution layer (#2163). - - * Add CELU activation function (#2191) - - * Add Log-Hyperbolic-Cosine Loss function (#2207). - - * Change neural network types to avoid unnecessary use of rvalue references - (#2259). - - * Bump minimum Boost version to 1.58 (#2305). - - * Refactor STB support so `HAS_STB` macro is not needed when compiling against - mlpack (#2312). - - * Add Hard Shrink Activation Function (#2186). - - * Add Soft Shrink Activation Function (#2174). - - * Add Hinge Embedding Loss Function (#2229). - - * Add Cosine Embedding Loss Function (#2209). - - * Add Margin Ranking Loss Function (#2264). - - * Bugfix for incorrect parameter vector sizes in logistic regression and - softmax regression (#2359). - -### mlpack 3.2.2 -###### 2019-11-26 - * Add `valid` and `same` padding option in `Convolution` and `Atrous - Convolution` layer (#1988). - - * Add Model() to the FFN class to access individual layers (#2043). - - * Update documentation for pip and conda installation packages (#2044). - - * Add bindings for linear SVM (#1935); `mlpack_linear_svm` from the - command-line, `linear_svm()` from Python. - - * Add support to return the layer name as `std::string` (#1987). - - * Speed and memory improvements for the Transposed Convolution layer (#1493). - - * Fix Windows Python build configuration (#1885). - - * Validate md5 of STB library after download (#2087). - - * Add `__version__` to `__init__.py` (#2092). - - * Correctly handle RNN sequences that are shorter than the value of rho (#2102). - -### mlpack 3.2.1 -###### 2019-10-01 - * Enforce CMake version check for ensmallen (#2032). - - * Fix CMake check for Armadillo version (#2029). - - * Better handling of when STB is not installed (#2033). - - * Fix Naive Bayes classifier computations in high dimensions (#2022). - -### mlpack 3.2.0 -###### 2019-09-25 - * Fix some potential infinity errors in Naive Bayes Classifier (#2022). - - * Fix occasionally-failing RADICAL test (#1924). - - * Fix gcc 9 OpenMP compilation issue (#1970). - - * Added support for loading and saving of images (#1903). - - * Add Multiple Pole Balancing Environment (#1901, #1951). - - * Added functionality for scaling of data (#1876); see the command-line - binding `mlpack_preprocess_scale` or Python binding `preprocess_scale()`. - - * Add new parameter `maximum_depth` to decision tree and random forest - bindings (#1916). - - * Fix prediction output of softmax regression when test set accuracy is - calculated (#1922). - - * Pendulum environment now checks for termination. All RL environments now - have an option to terminate after a set number of time steps (no limit - by default) (#1941). - - * Add support for probabilistic KDE (kernel density estimation) error bounds - when using the Gaussian kernel (#1934). - - * Fix negative distances for cover tree computation (#1979). - - * Fix cover tree building when all pairwise distances are 0 (#1986). - - * Improve KDE pruning by reclaiming not used error tolerance (#1954, #1984). - - * Optimizations for sparse matrix accesses in z-score normalization for CF - (#1989). - - * Add `kmeans_max_iterations` option to GMM training binding `gmm_train_main`. - - * Bump minimum Armadillo version to 8.400.0 due to ensmallen dependency - requirement (#2015). - -### mlpack 3.1.1 -###### 2019-05-26 - * Fix random forest bug for numerical-only data (#1887). - - * Significant speedups for random forest (#1887). - - * Random forest now has `minimum_gain_split` and `subspace_dim` parameters - (#1887). - - * Decision tree parameter `print_training_error` deprecated in favor of - `print_training_accuracy`. - - * `output` option changed to `predictions` for adaboost and perceptron - binding. Old options are now deprecated and will be preserved until mlpack - 4.0.0 (#1882). - - * Concatenated ReLU layer (#1843). - - * Accelerate NormalizeLabels function using hashing instead of linear search - (see `src/mlpack/core/data/normalize_labels_impl.hpp`) (#1780). - - * Add `ConfusionMatrix()` function for checking performance of classifiers - (#1798). - - * Install ensmallen headers when it is downloaded during build (#1900). - -### mlpack 3.1.0 -###### 2019-04-25 - * Add DiagonalGaussianDistribution and DiagonalGMM classes to speed up the - diagonal covariance computation and deprecate DiagonalConstraint (#1666). - - * Add kernel density estimation (KDE) implementation with bindings to other - languages (#1301). - - * Where relevant, all models with a `Train()` method now return a `double` - value representing the goodness of fit (i.e. final objective value, error, - etc.) (#1678). - - * Add implementation for linear support vector machine (see - `src/mlpack/methods/linear_svm`). - - * Change DBSCAN to use PointSelectionPolicy and add OrderedPointSelection (#1625). - - * Residual block support (#1594). - - * Bidirectional RNN (#1626). - - * Dice loss layer (#1674, #1714) and hard sigmoid layer (#1776). - - * `output` option changed to `predictions` and `output_probabilities` to - `probabilities` for Naive Bayes binding (`mlpack_nbc`/`nbc()`). Old options - are now deprecated and will be preserved until mlpack 4.0.0 (#1616). - - * Add support for Diagonal GMMs to HMM code (#1658, #1666). This can provide - large speedup when a diagonal GMM is acceptable as an emission probability - distribution. - - * Python binding improvements: check parameter type (#1717), avoid copying - Pandas dataframes (#1711), handle Pandas Series objects (#1700). - -### mlpack 3.0.4 -###### 2018-11-13 - * Bump minimum CMake version to 3.3.2. - - * CMake fixes for Ninja generator by Marc Espie. - -### mlpack 3.0.3 -###### 2018-07-27 - * Fix Visual Studio compilation issue (#1443). - - * Allow running local_coordinate_coding binding with no initial_dictionary - parameter when input_model is not specified (#1457). - - * Make use of OpenMP optional via the CMake 'USE_OPENMP' configuration - variable (#1474). - - * Accelerate FNN training by 20-30% by avoiding redundant calculations - (#1467). - - * Fix math::RandomSeed() usage in tests (#1462, #1440). - - * Generate better Python setup.py with documentation (#1460). - -### mlpack 3.0.2 -###### 2018-06-08 - * Documentation generation fixes for Python bindings (#1421). - - * Fix build error for man pages if command-line bindings are not being built - (#1424). - - * Add 'shuffle' parameter and Shuffle() method to KFoldCV (#1412). This will - shuffle the data when the object is constructed, or when Shuffle() is - called. - - * Added neural network layers: AtrousConvolution (#1390), Embedding (#1401), - and LayerNorm (layer normalization) (#1389). - - * Add Pendulum environment for reinforcement learning (#1388) and update - Mountain Car environment (#1394). - -### mlpack 3.0.1 -###### 2018-05-10 - * Fix intermittently failing tests (#1387). - - * Add big-batch SGD (BBSGD) optimizer in - src/mlpack/core/optimizers/bigbatch_sgd/ (#1131). - - * Fix simple compiler warnings (#1380, #1373). - - * Simplify NeighborSearch constructor and Train() overloads (#1378). - - * Add warning for OpenMP setting differences (#1358/#1382). When mlpack is - compiled with OpenMP but another application is not (or vice versa), a - compilation warning will now be issued. - - * Restructured loss functions in src/mlpack/methods/ann/ (#1365). - - * Add environments for reinforcement learning tests (#1368, #1370, #1329). - - * Allow single outputs for multiple timestep inputs for recurrent neural - networks (#1348). - - * Add He and LeCun normal initializations for neural networks (#1342). - Neural networks: add He and LeCun normal initializations (#1342), add FReLU - and SELU activation functions (#1346, #1341), add alpha-dropout (#1349). - -### mlpack 3.0.0 -###### 2018-03-30 - * Speed and memory improvements for DBSCAN. --single_mode can now be used for - situations where previously RAM usage was too high. - - * Bump minimum required version of Armadillo to 6.500.0. - - * Add automatically generated Python bindings. These have the same interface - as the command-line programs. - - * Add deep learning infrastructure in src/mlpack/methods/ann/. - - * Add reinforcement learning infrastructure in - src/mlpack/methods/reinforcement_learning/. - - * Add optimizers: AdaGrad, CMAES, CNE, FrankeWolfe, GradientDescent, - GridSearch, IQN, Katyusha, LineSearch, ParallelSGD, SARAH, SCD, SGDR, - SMORMS3, SPALeRA, SVRG. - - * Add hyperparameter tuning infrastructure and cross-validation infrastructure - in src/mlpack/core/cv/ and src/mlpack/core/hpt/. - - * Fix bug in mean shift. - - * Add random forests (see src/mlpack/methods/random_forest). - - * Numerous other bugfixes and testing improvements. - - * Add randomized Krylov SVD and Block Krylov SVD. - -### mlpack 2.2.5 -###### 2017-08-25 - * Compilation fix for some systems (#1082). - - * Fix PARAM_INT_OUT() (#1100). - -### mlpack 2.2.4 -###### 2017-07-18 - * Speed and memory improvements for DBSCAN. --single_mode can now be used for - situations where previously RAM usage was too high. - - * Fix bug in CF causing incorrect recommendations. - -### mlpack 2.2.3 -###### 2017-05-24 - * Bug fix for --predictions_file in mlpack_decision_tree program. - -### mlpack 2.2.2 -###### 2017-05-04 - * Install backwards-compatibility mlpack_allknn and mlpack_allkfn programs; - note they are deprecated and will be removed in mlpack 3.0.0 (#992). - - * Fix RStarTree bug that surfaced on OS X only (#964). - - * Small fixes for MiniBatchSGD and SGD and tests. - -### mlpack 2.2.1 -###### 2017-04-13 - * Compilation fix for mlpack_nca and mlpack_test on older Armadillo versions - (#984). - -### mlpack 2.2.0 -###### 2017-03-21 - * Bugfix for mlpack_knn program (#816). - - * Add decision tree implementation in methods/decision_tree/. This is very - similar to a C4.5 tree learner. - - * Add DBSCAN implementation in methods/dbscan/. - - * Add support for multidimensional discrete distributions (#810, #830). - - * Better output for Log::Debug/Log::Info/Log::Warn/Log::Fatal for Armadillo - objects (#895, #928). - - * Refactor categorical CSV loading with boost::spirit for faster loading - (#681). - -### mlpack 2.1.1 -###### 2016-12-22 - * HMMs now use random initialization; this should fix some convergence issues - (#828). - - * HMMs now initialize emissions according to the distribution of observations - (#833). - - * Minor fix for formatted output (#814). - - * Fix DecisionStump to properly work with any input type. - -### mlpack 2.1.0 -###### 2016-10-31 - * Fixed CoverTree to properly handle single-point datasets. - - * Fixed a bug in CosineTree (and thus QUIC-SVD) that caused split failures for - some datasets (#717). - - * Added mlpack_preprocess_describe program, which can be used to print - statistics on a given dataset (#742). - - * Fix prioritized recursion for k-furthest-neighbor search (mlpack_kfn and the - KFN class), leading to orders-of-magnitude speedups in some cases. - - * Bump minimum required version of Armadillo to 4.200.0. - - * Added simple Gradient Descent optimizer, found in - src/mlpack/core/optimizers/gradient_descent/ (#792). - - * Added approximate furthest neighbor search algorithms QDAFN and - DrusillaSelect in src/mlpack/methods/approx_kfn/, with command-line program - mlpack_approx_kfn. - -### mlpack 2.0.3 -###### 2016-07-21 - * Added multiprobe LSH (#691). The parameter 'T' to LSHSearch::Search() can - now be used to control the number of extra bins that are probed, as can the - -T (--num_probes) option to mlpack_lsh. - - * Added the Hilbert R tree to src/mlpack/core/tree/rectangle_tree/ (#664). It - can be used as the typedef HilbertRTree, and it is now an option in the - mlpack_knn, mlpack_kfn, mlpack_range_search, and mlpack_krann command-line - programs. - - * Added the mlpack_preprocess_split and mlpack_preprocess_binarize programs, - which can be used for preprocessing code (#650, #666). - - * Added OpenMP support to LSHSearch and mlpack_lsh (#700). - -### mlpack 2.0.2 -###### 2016-06-20 - * Added the function LSHSearch::Projections(), which returns an arma::cube - with each projection table in a slice (#663). Instead of Projection(i), you - should now use Projections().slice(i). - - * A new constructor has been added to LSHSearch that creates objects using - projection tables provided in an arma::cube (#663). - - * Handle zero-variance dimensions in DET (#515). - - * Add MiniBatchSGD optimizer (src/mlpack/core/optimizers/minibatch_sgd/) and - allow its use in mlpack_logistic_regression and mlpack_nca programs. - - * Add better backtrace support from Grzegorz Krajewski for Log::Fatal messages - when compiled with debugging and profiling symbols. This requires libbfd - and libdl to be present during compilation. - - * CosineTree test fix from Mikhail Lozhnikov (#358). - - * Fixed HMM initial state estimation (#600). - - * Changed versioning macros __MLPACK_VERSION_MAJOR, __MLPACK_VERSION_MINOR, - and __MLPACK_VERSION_PATCH to MLPACK_VERSION_MAJOR, MLPACK_VERSION_MINOR, - and MLPACK_VERSION_PATCH. The old names will remain in place until - mlpack 3.0.0. - - * Renamed mlpack_allknn, mlpack_allkfn, and mlpack_allkrann to mlpack_knn, - mlpack_kfn, and mlpack_krann. The mlpack_allknn, mlpack_allkfn, and - mlpack_allkrann programs will remain as copies until mlpack 3.0.0. - - * Add --random_initialization option to mlpack_hmm_train, for use when no - labels are provided. - - * Add --kill_empty_clusters option to mlpack_kmeans and KillEmptyClusters - policy for the KMeans class (#595, #596). - -### mlpack 2.0.1 -###### 2016-02-04 - * Fix CMake to properly detect when MKL is being used with Armadillo. - - * Minor parameter handling fixes to mlpack_logistic_regression (#504, #505). - - * Properly install arma_config.hpp. - - * Memory handling fixes for Hoeffding tree code. - - * Add functions that allow changing training-time parameters to HoeffdingTree - class. - - * Fix infinite loop in sparse coding test. - - * Documentation spelling fixes (#501). - - * Properly handle covariances for Gaussians with large condition number - (#496), preventing GMMs from filling with NaNs during training (and also - HMMs that use GMMs). - - * CMake fixes for finding LAPACK and BLAS as Armadillo dependencies when ATLAS - is used. - - * CMake fix for projects using mlpack's CMake configuration from elsewhere - (#512). - -### mlpack 2.0.0 -###### 2015-12-24 - * Removed overclustering support from k-means because it is not well-tested, - may be buggy, and is (I think) unused. If this was support you were using, - open a bug or get in touch with us; it would not be hard for us to - reimplement it. - - * Refactored KMeans to allow different types of Lloyd iterations. - - * Added implementations of k-means: Elkan's algorithm, Hamerly's algorithm, - Pelleg-Moore's algorithm, and the DTNN (dual-tree nearest neighbor) - algorithm. - - * Significant acceleration of LRSDP via the use of accu(a % b) instead of - trace(a * b). - - * Added MatrixCompletion class (matrix_completion), which performs nuclear - norm minimization to fill unknown values of an input matrix. - - * No more dependence on Boost.Random; now we use C++11 STL random support. - - * Add softmax regression, contributed by Siddharth Agrawal and QiaoAn Chen. - - * Changed NeighborSearch, RangeSearch, FastMKS, LSH, and RASearch API; these - classes now take the query sets in the Search() method, instead of in the - constructor. - - * Use OpenMP, if available. For now OpenMP support is only available in the - DET training code. - - * Add support for predicting new test point values to LARS and the - command-line 'lars' program. - - * Add serialization support for Perceptron and LogisticRegression. - - * Refactor SoftmaxRegression to predict into an arma::Row object, and - add a softmax_regression program. - - * Refactor LSH to allow loading and saving of models. - - * ToString() is removed entirely (#487). - - * Add --input_model_file and --output_model_file options to appropriate - machine learning algorithms. - - * Rename all executables to start with an "mlpack" prefix (#229). - - * Add HoeffdingTree and mlpack_hoeffding_tree, an implementation of the - streaming decision tree methodology from Domingos and Hulten in 2000. - -### mlpack 1.0.12 -###### 2015-01-07 - * Switch to 3-clause BSD license (from LGPL). - -### mlpack 1.0.11 -###### 2014-12-11 - * Proper handling of dimension calculation in PCA. - - * Load parameter vectors properly for LinearRegression models. - - * Linker fixes for AugLagrangian specializations under Visual Studio. - - * Add support for observation weights to LinearRegression. - - * MahalanobisDistance<> now takes the root of the distance by default and - therefore satisfies the triangle inequality (TakeRoot now defaults to true). - - * Better handling of optional Armadillo HDF5 dependency. - - * Fixes for numerous intermittent test failures. - - * math::RandomSeed() now sets the random seed for recent (>=3.930) Armadillo - versions. - - * Handle Newton method convergence better for - SparseCoding::OptimizeDictionary() and make maximum iterations a parameter. - - * Known bug: CosineTree construction may fail in some cases on i386 systems - (#358). - -### mlpack 1.0.10 -###### 2014-08-29 - * Bugfix for NeighborSearch regression which caused very slow allknn/allkfn. - Speeds are now restored to approximately 1.0.8 speeds, with significant - improvement for the cover tree (#347). - - * Detect dependencies correctly when ARMA_USE_WRAPPER is not being defined - (i.e., libarmadillo.so does not exist). - - * Bugfix for compilation under Visual Studio (#348). - -### mlpack 1.0.9 -###### 2014-07-28 - * GMM initialization is now safer and provides a working GMM when constructed - with only the dimensionality and number of Gaussians (#301). - - * Check for division by 0 in Forward-Backward Algorithm in HMMs (#301). - - * Fix MaxVarianceNewCluster (used when re-initializing clusters for k-means) - (#301). - - * Fixed implementation of Viterbi algorithm in HMM::Predict() (#303). - - * Significant speedups for dual-tree algorithms using the cover tree (#235, - #314) including a faster implementation of FastMKS. - - * Fix for LRSDP optimizer so that it compiles and can be used (#312). - - * CF (collaborative filtering) now expects users and items to be zero-indexed, - not one-indexed (#311). - - * CF::GetRecommendations() API change: now requires the number of - recommendations as the first parameter. The number of users in the local - neighborhood should be specified with CF::NumUsersForSimilarity(). - - * Removed incorrect PeriodicHRectBound (#58). - - * Refactor LRSDP into LRSDP class and standalone function to be optimized - (#305). - - * Fix for centering in kernel PCA (#337). - - * Added simulated annealing (SA) optimizer, contributed by Zhihao Lou. - - * HMMs now support initial state probabilities; these can be set in the - constructor, trained, or set manually with HMM::Initial() (#302). - - * Added Nyström method for kernel matrix approximation by Marcus Edel. - - * Kernel PCA now supports using Nyström method for approximation. - - * Ball trees now work with dual-tree algorithms, via the BallBound<> bound - structure (#307); fixed by Yash Vadalia. - - * The NMF class is now AMF<>, and supports far more types of factorizations, - by Sumedh Ghaisas. - - * A QUIC-SVD implementation has returned, written by Siddharth Agrawal and - based on older code from Mudit Gupta. - - * Added perceptron and decision stump by Udit Saxena (these are weak learners - for an eventual AdaBoost class). - - * Sparse autoencoder added by Siddharth Agrawal. - -### mlpack 1.0.8 -###### 2014-01-06 - * Memory leak in NeighborSearch index-mapping code fixed (#298). - - * GMMs can be trained using the existing model as a starting point by - specifying an additional boolean parameter to GMM::Estimate() (#296). - - * Logistic regression implementation added in methods/logistic_regression (see - also #293). - - * L-BFGS optimizer now returns its function via Function(). - - * Version information is now obtainable via mlpack::util::GetVersion() or the - __MLPACK_VERSION_MAJOR, __MLPACK_VERSION_MINOR, and __MLPACK_VERSION_PATCH - macros (#297). - - * Fix typos in allkfn and allkrann output. - -### mlpack 1.0.7 -###### 2013-10-04 - * Cover tree support for range search (range_search), rank-approximate nearest - neighbors (allkrann), minimum spanning tree calculation (emst), and FastMKS - (fastmks). - - * Dual-tree FastMKS implementation added and tested. - - * Added collaborative filtering package (cf) that can provide recommendations - when given users and items. - - * Fix for correctness of Kernel PCA (kernel_pca) (#270). - - * Speedups for PCA and Kernel PCA (#198). - - * Fix for correctness of Neighborhood Components Analysis (NCA) (#279). - - * Minor speedups for dual-tree algorithms. - - * Fix for Naive Bayes Classifier (nbc) (#269). - - * Added a ridge regression option to LinearRegression (linear_regression) - (#286). - - * Gaussian Mixture Models (gmm::GMM<>) now support arbitrary covariance matrix - constraints (#283). - - * MVU (mvu) removed because it is known to not work (#183). - - * Minor updates and fixes for kernels (in mlpack::kernel). - -### mlpack 1.0.6 -###### 2013-06-13 - * Minor bugfix so that FastMKS gets built. - -### mlpack 1.0.5 -###### 2013-05-01 - * Speedups of cover tree traversers (#235). - - * Addition of rank-approximate nearest neighbors (RANN), found in - src/mlpack/methods/rann/. - - * Addition of fast exact max-kernel search (FastMKS), found in - src/mlpack/methods/fastmks/. - - * Fix for EM covariance estimation; this should improve GMM training time. - - * More parameters for GMM estimation. - - * Force GMM and GaussianDistribution covariance matrices to be positive - definite, so that training converges much more often. - - * Add parameter for the tolerance of the Baum-Welch algorithm for HMM - training. - - * Fix for compilation with clang compiler. - - * Fix for k-furthest-neighbor-search. - -### mlpack 1.0.4 -###### 2013-02-08 - * Force minimum Armadillo version to 2.4.2. - - * Better output of class types to streams; a class with a ToString() method - implemented can be sent to a stream with operator<<. - - * Change return type of GMM::Estimate() to double (#257). - - * Style fixes for k-means and RADICAL. - - * Handle size_t support correctly with Armadillo 3.6.2 (#258). - - * Add locality-sensitive hashing (LSH), found in src/mlpack/methods/lsh/. - - * Better tests for SGD (stochastic gradient descent) and NCA (neighborhood - components analysis). - -### mlpack 1.0.3 -###### 2012-09-16 - - * Remove internal sparse matrix support because Armadillo 3.4.0 now includes - it. When using Armadillo versions older than 3.4.0, sparse matrix support - is not available. - - * NCA (neighborhood components analysis) now support an arbitrary optimizer - (#245), including stochastic gradient descent (#249). - -### mlpack 1.0.2 -###### 2012-08-15 - * Added density estimation trees, found in src/mlpack/methods/det/. - - * Added non-negative matrix factorization, found in src/mlpack/methods/nmf/. - - * Added experimental cover tree implementation, found in - src/mlpack/core/tree/cover_tree/ (#157). - - * Better reporting of boost::program_options errors (#225). - - * Fix for timers on Windows (#212, #211). - - * Fix for allknn and allkfn output (#204). - - * Sparse coding dictionary initialization is now a template parameter (#220). - -### mlpack 1.0.1 -###### 2012-03-03 - * Added kernel principal components analysis (kernel PCA), found in - src/mlpack/methods/kernel_pca/ (#74). - - * Fix for Lovasz-Theta AugLagrangian tests (#182). - - * Fixes for allknn output (#185, #186). - - * Added range search executable (#192). - - * Adapted citations in documentation to BibTeX; no citations in -h output - (#195). - - * Stop use of 'const char*' and prefer 'std::string' (#176). - - * Support seeds for random numbers (#177). - -### mlpack 1.0.0 -###### 2011-12-17 - * Initial release. See any resolved tickets numbered less than #196 or - execute this query: - http://www.mlpack.org/trac/query?status=closed&milestone=mlpack+1.0.0 + * Includes the ported optimization framework from mlpack + (http://www.mlpack.org/). From d42f98546dc8cf2487a93094365bc02811e50c12 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 29 May 2023 11:53:42 -0400 Subject: [PATCH 08/34] Tiny style fix for name of function: initialStepSize -> InitialStepSize. --- include/ensmallen_bits/cmaes/cmaes_impl.hpp | 2 +- .../boundary_box_constraint.hpp | 2 +- .../empty_transformation.hpp | 31 +++++++++---------- 3 files changed, 17 insertions(+), 18 deletions(-) diff --git a/include/ensmallen_bits/cmaes/cmaes_impl.hpp b/include/ensmallen_bits/cmaes/cmaes_impl.hpp index f7dd38bae..037424448 100644 --- a/include/ensmallen_bits/cmaes/cmaes_impl.hpp +++ b/include/ensmallen_bits/cmaes/cmaes_impl.hpp @@ -125,7 +125,7 @@ typename MatType::elem_type CMAES class EmptyTransformation { -public: - - /** - * Transforms coordinates to themselves (effectively no transformation). - * - * @param x Input coordinates. - * @return Transformed coordinates (the coordinates themselves). - */ - MatType Transform(const MatType& x) { return x; } - - /** - * Return a suitable initial step size. - * - * @return initial step size. - */ - typename MatType::elem_type initialStepSize() { return 1; } + public: + /** + * Transforms coordinates to themselves (effectively no transformation). + * + * @param x Input coordinates. + * @return Transformed coordinates (the coordinates themselves). + */ + MatType Transform(const MatType& x) { return x; } + /** + * Return a suitable initial step size. + * + * @return initial step size. + */ + typename MatType::elem_type InitialStepSize() { return 1; } }; + } // namespace ens From c88e0c3395706f9ddbcc3664a92fa2dba7943891 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 29 May 2023 11:54:04 -0400 Subject: [PATCH 09/34] Remove some more tab characters I found... --- include/ensmallen_bits/nsga2/nsga2_impl.hpp | 6 +++--- include/ensmallen_bits/problems/zdt/zdt4_function.hpp | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/ensmallen_bits/nsga2/nsga2_impl.hpp b/include/ensmallen_bits/nsga2/nsga2_impl.hpp index d28e69c5a..acfb736d4 100644 --- a/include/ensmallen_bits/nsga2/nsga2_impl.hpp +++ b/include/ensmallen_bits/nsga2/nsga2_impl.hpp @@ -423,9 +423,9 @@ inline bool NSGA2::Dominates( //! Assign crowding distance to the population. template inline void NSGA2::CrowdingDistanceAssignment( - const std::vector& front, - std::vector>& calculatedObjectives, - std::vector& crowdingDistance) + const std::vector& front, + std::vector>& calculatedObjectives, + std::vector& crowdingDistance) { // Convenience typedefs. typedef typename MatType::elem_type ElemType; diff --git a/include/ensmallen_bits/problems/zdt/zdt4_function.hpp b/include/ensmallen_bits/problems/zdt/zdt4_function.hpp index b7d77f06d..fad2ba912 100644 --- a/include/ensmallen_bits/problems/zdt/zdt4_function.hpp +++ b/include/ensmallen_bits/problems/zdt/zdt4_function.hpp @@ -83,7 +83,7 @@ namespace test { 10. * arma::cos(4 * arma::datum::pi * truncatedCoords)); ElemType g = 1. + 10. * static_cast(numVariables - 1) + sum; ElemType objectiveRatio = objectives(0) / g; - objectives(1) = g * (1. - std::sqrt(objectiveRatio)); + objectives(1) = g * (1. - std::sqrt(objectiveRatio)); return objectives; } @@ -157,4 +157,4 @@ namespace test { }; } //namespace test } //namespace ens -#endif \ No newline at end of file +#endif From 46565e7e0a94a51a35fcd194efaae86680db5fe2 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 29 May 2023 11:54:21 -0400 Subject: [PATCH 10/34] Compilation fixes for tests. --- tests/callbacks_test.cpp | 4 ++-- tests/cmaes_test.cpp | 18 +++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/callbacks_test.cpp b/tests/callbacks_test.cpp index a40bbf480..ec87b8855 100644 --- a/tests/callbacks_test.cpp +++ b/tests/callbacks_test.cpp @@ -312,8 +312,8 @@ TEST_CASE("BigBatchSGDCallbacksFullFunctionTest", "[CallbacksTest]") */ TEST_CASE("CMAESCallbacksFullFunctionTest", "[CallbacksTest]") { - BoundaryBoxConstraint b(-1, 1); - CMAES optimizer(0, b, 32, 3, 1e-3); + BoundaryBoxConstraint<> b(-1, 1); + CMAES> optimizer(0, b, 32, 3, 1e-3); CallbacksFullFunctionTest(optimizer, true, false, false, false, true, true, false, false, true); } diff --git a/tests/cmaes_test.cpp b/tests/cmaes_test.cpp index 4aca40661..1dc19242d 100644 --- a/tests/cmaes_test.cpp +++ b/tests/cmaes_test.cpp @@ -35,8 +35,8 @@ TEST_CASE("CMAESDeprecatedConstructorLogisticRegressionTest", "[CMAESTest]") */ TEST_CASE("CMAESLogisticRegressionTest", "[CMAESTest]") { - BoundaryBoxConstraint b(-1, 1); - CMAES cmaes(0, b, 32, 200, 1e-3); + BoundaryBoxConstraint<> b(-1, 1); + CMAES> cmaes(0, b, 32, 200, 1e-3); LogisticRegressionFunctionTest(cmaes, 0.003, 0.006, 5); } @@ -46,7 +46,7 @@ TEST_CASE("CMAESLogisticRegressionTest", "[CMAESTest]") */ TEST_CASE("ApproxCMAESLogisticRegressionTest", "[CMAESTest]") { - BoundaryBoxConstraint b(-1, 1); + BoundaryBoxConstraint<> b(-1, 1); ApproxCMAES> cmaes(0, b, 32, 200, 1e-3); LogisticRegressionFunctionTest(cmaes, 0.003, 0.006, 5); } @@ -58,7 +58,7 @@ TEST_CASE("ApproxCMAESLogisticRegressionTest", "[CMAESTest]") TEST_CASE("CMAESLogisticRegressionFMatTest", "[CMAESTest]") { BoundaryBoxConstraint b(-1, 1); - CMAES cmaes(0, b, 32, 200, 1e-3); + CMAES> cmaes(0, b, 32, 200, 1e-3); LogisticRegressionFunctionTest(cmaes, 0.01, 0.02, 5); } @@ -74,14 +74,14 @@ TEST_CASE("ApproxCMAESLogisticRegressionFMatTest", "[CMAESTest]") } /** - * Run CMA-ES with the random selection and empty transformation policies - * on logistic regression and make sure the results are acceptable. + * Run CMA-ES with the random selection and empty transformation policies + * on logistic regression and make sure the results are acceptable. * Use arma::fmat. */ -TEST_CASE("ApproxCMAESEmptyTransformationLogisticRegressionFMatTest", - "[CMAESTest]") +TEST_CASE("ApproxCMAESEmptyTransformationLogisticRegressionFMatTest", + "[CMAESTest]") { - ApproxCMAES> + ApproxCMAES> cmaes(5000, EmptyTransformation(), 32, 5000, 1e-3); LogisticRegressionFunctionTest(cmaes, 0.01, 0.02, 5); } From 716cea6a77a36568f450a71bae16b4a7afc095c3 Mon Sep 17 00:00:00 2001 From: Suvarsha Chennareddy Date: Thu, 1 Jun 2023 00:49:14 +0530 Subject: [PATCH 11/34] fix bug and modify tests --- .../boundary_box_constraint.hpp | 4 +-- tests/cmaes_test.cpp | 26 +++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/include/ensmallen_bits/cmaes/transformation_policies/boundary_box_constraint.hpp b/include/ensmallen_bits/cmaes/transformation_policies/boundary_box_constraint.hpp index 2134b42e3..9583a0976 100644 --- a/include/ensmallen_bits/cmaes/transformation_policies/boundary_box_constraint.hpp +++ b/include/ensmallen_bits/cmaes/transformation_policies/boundary_box_constraint.hpp @@ -95,11 +95,11 @@ class BoundaryBoxConstraint // Shift y into feasible pre-image. if (y(i, j) < xlow) { - y(i,j) += (ElemType)(r * (1 + (int)(xlow - y(i, j)) / r)); + y(i,j) += (ElemType)(r * (1 + (int)((xlow - y(i, j)) / r))); } if (y(i, j) > xup) { - y(i, j) -= (ElemType)(r * (1 + (int)(y(i, j) - xup) / r)); + y(i, j) -= (ElemType)(r * (1 + (int)((y(i, j) - xup) / r))); } if (y(i, j) < lowerBound(Bi, Bj) - al) { diff --git a/tests/cmaes_test.cpp b/tests/cmaes_test.cpp index 1dc19242d..19639a1bd 100644 --- a/tests/cmaes_test.cpp +++ b/tests/cmaes_test.cpp @@ -20,12 +20,12 @@ using namespace ens::test; /** * Run CMA-ES with the full selection policy on logistic regression and * make sure the results are acceptable. - * This test uses the deprecated constructor and therefore can be removed + * This test uses the deprecated constructor and therefore can be removed * in a future version of ensmallen. */ TEST_CASE("CMAESDeprecatedConstructorLogisticRegressionTest", "[CMAESTest]") { - CMAES> cmaes(0, -1, 1, 32, 200, 1e-3); + CMAES> cmaes(0, -10, 10, 32, 500, 1e-3); LogisticRegressionFunctionTest(cmaes, 0.003, 0.006, 5); } @@ -35,8 +35,8 @@ TEST_CASE("CMAESDeprecatedConstructorLogisticRegressionTest", "[CMAESTest]") */ TEST_CASE("CMAESLogisticRegressionTest", "[CMAESTest]") { - BoundaryBoxConstraint<> b(-1, 1); - CMAES> cmaes(0, b, 32, 200, 1e-3); + BoundaryBoxConstraint<> b(-10, 10); + CMAES> cmaes(0, b, 32, 500, 1e-3); LogisticRegressionFunctionTest(cmaes, 0.003, 0.006, 5); } @@ -46,8 +46,8 @@ TEST_CASE("CMAESLogisticRegressionTest", "[CMAESTest]") */ TEST_CASE("ApproxCMAESLogisticRegressionTest", "[CMAESTest]") { - BoundaryBoxConstraint<> b(-1, 1); - ApproxCMAES> cmaes(0, b, 32, 200, 1e-3); + BoundaryBoxConstraint<> b(-10, 10); + ApproxCMAES> cmaes(0, b, 32, 500, 1e-3); LogisticRegressionFunctionTest(cmaes, 0.003, 0.006, 5); } @@ -57,8 +57,8 @@ TEST_CASE("ApproxCMAESLogisticRegressionTest", "[CMAESTest]") */ TEST_CASE("CMAESLogisticRegressionFMatTest", "[CMAESTest]") { - BoundaryBoxConstraint b(-1, 1); - CMAES> cmaes(0, b, 32, 200, 1e-3); + BoundaryBoxConstraint b(-10, 10); + CMAES> cmaes(0, b, 32, 500, 1e-3); LogisticRegressionFunctionTest(cmaes, 0.01, 0.02, 5); } @@ -68,8 +68,8 @@ TEST_CASE("CMAESLogisticRegressionFMatTest", "[CMAESTest]") */ TEST_CASE("ApproxCMAESLogisticRegressionFMatTest", "[CMAESTest]") { - BoundaryBoxConstraint b(-1, 1); - ApproxCMAES> cmaes(0, b, 32, 200, 1e-3); + BoundaryBoxConstraint b(-10, 10); + ApproxCMAES> cmaes(0, b, 32, 500, 1e-3); LogisticRegressionFunctionTest(cmaes, 0.01, 0.02, 5); } @@ -79,9 +79,9 @@ TEST_CASE("ApproxCMAESLogisticRegressionFMatTest", "[CMAESTest]") * Use arma::fmat. */ TEST_CASE("ApproxCMAESEmptyTransformationLogisticRegressionFMatTest", - "[CMAESTest]") + "[CMAESTest]") { ApproxCMAES> - cmaes(5000, EmptyTransformation(), 32, 5000, 1e-3); + cmaes(0, EmptyTransformation(), 32, 500, 1e-3); LogisticRegressionFunctionTest(cmaes, 0.01, 0.02, 5); -} +} \ No newline at end of file From 8baccf89901deeb54850ca16bc60da95eae29a1f Mon Sep 17 00:00:00 2001 From: Suvarsha Chennareddy Date: Fri, 9 Jun 2023 20:02:13 +0530 Subject: [PATCH 12/34] add implementation of active cmaes --- include/ensmallen.hpp | 1 + include/ensmallen_bits/cmaes/active_cmaes.hpp | 219 ++++++++++ .../cmaes/active_cmaes_impl.hpp | 382 ++++++++++++++++++ include/ensmallen_bits/cmaes/cmaes_impl.hpp | 5 + .../boundary_box_constraint.hpp | 4 + .../empty_transformation.hpp | 4 + 6 files changed, 615 insertions(+) create mode 100644 include/ensmallen_bits/cmaes/active_cmaes.hpp create mode 100644 include/ensmallen_bits/cmaes/active_cmaes_impl.hpp diff --git a/include/ensmallen.hpp b/include/ensmallen.hpp index a1338e819..be91cc23f 100644 --- a/include/ensmallen.hpp +++ b/include/ensmallen.hpp @@ -97,6 +97,7 @@ #include "ensmallen_bits/aug_lagrangian/aug_lagrangian.hpp" #include "ensmallen_bits/bigbatch_sgd/bigbatch_sgd.hpp" #include "ensmallen_bits/cmaes/cmaes.hpp" +#include "ensmallen_bits/cmaes/active_cmaes.hpp" #include "ensmallen_bits/cne/cne.hpp" #include "ensmallen_bits/de/de.hpp" #include "ensmallen_bits/eve/eve.hpp" diff --git a/include/ensmallen_bits/cmaes/active_cmaes.hpp b/include/ensmallen_bits/cmaes/active_cmaes.hpp new file mode 100644 index 000000000..76e369954 --- /dev/null +++ b/include/ensmallen_bits/cmaes/active_cmaes.hpp @@ -0,0 +1,219 @@ +/** + * @file active_cmaes.hpp + * @author Marcus Edel + * @author Suvarsha Chennareddy + * + * Definition of the Active Covariance Matrix Adaptation Evolution Strategy + * as proposed by G.A Jastrebski and D.V Arnold in "Improving Evolution + * Strategies through Active Covariance Matrix Adaptation". + * + * ensmallen is free software; you may redistribute it and/or modify it under + * the terms of the 3-clause BSD license. You should have received a copy of + * the 3-clause BSD license along with ensmallen. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef ENSMALLEN_ACTIVE_CMAES_CMAES_HPP +#define ENSMALLEN_ACTIVE_CMAES_CMAES_HPP + +#include "full_selection.hpp" +#include "random_selection.hpp" +#include "transformation_policies/empty_transformation.hpp" +#include "transformation_policies/boundary_box_constraint.hpp" + +namespace ens { + +/** + * Active CMA-ES is a variant of the stochastic search algorithm + * CMA-ES - Covariance Matrix Adaptation Evolution Strategy. + * Active CMA-ES actively reduces the uncertainty in unfavourable directions by + * exploiting the information about bad mutations in the covariance matrix + * update step. This isn't for the purpose of accelerating progress, but + * instead for speeding up the adaptation of the covariance matrix (which, in + * turn, will lead to faster progress). + * + * For more information, please refer to: + * + * @code + * @INPROCEEDINGS{1688662, + * author={Jastrebski, G.A. and Arnold, D.V.}, + * booktitle={2006 IEEE International Conference on Evolutionary + Computation}, + * title={Improving Evolution Strategies through Active Covariance + Matrix Adaptation}, + * year={2006}, + * volume={}, + * number={}, + * pages={2814-2821}, + * doi={10.1109/CEC.2006.1688662}} + * @endcode + * + * Active CMA-ES can optimize separable functions. For more details, see the + * documentation on function types included with this distribution or on the + * ensmallen website. + * + * @tparam SelectionPolicy The selection strategy used for the evaluation step. + * @tparam TransformationPolicy The transformation strategy used to + * map decision variables to the desired domain during fitness evaluation + * and termination. Use EmptyTransformation if the domain isn't bounded. + */ +template> +class ActiveCMAES +{ + public: + /** + * Construct the Active CMA-ES optimizer with the given function and parameters. The + * defaults here are not necessarily good for the given problem, so it is + * suggested that the values used be tailored to the task at hand. The + * maximum number of iterations refers to the maximum number of points that + * are processed (i.e., one iteration equals one point; one iteration does not + * equal one pass over the dataset). + * + * @param lambda The population size (0 use the default size). + * @param transformationPolicy Instantiated transformation policy used to + * map the coordinates to the desired domain. + * @param batchSize Batch size to use for the objective calculation. + * @param maxIterations Maximum number of iterations allowed (0 means no + * limit). + * @param tolerance Maximum absolute tolerance to terminate algorithm. + * @param selectionPolicy Instantiated selection policy used to calculate the + * objective. + * @param stepSize Starting sigma/step size (will be modified). + */ + ActiveCMAES(const size_t lambda = 0, + const TransformationPolicyType& + transformationPolicy = TransformationPolicyType(), + const size_t batchSize = 32, + const size_t maxIterations = 1000, + const double tolerance = 1e-5, + const SelectionPolicyType& selectionPolicy = SelectionPolicyType(), + double stepSize = 0); + + /** + * Construct the CMA-ES optimizer with the given function and parameters + * (including lower and upper bounds). The defaults here are not necessarily + * good for the given problem, so it is suggested that the values used be + * tailored to the task at hand. The maximum number of iterations refers to + * the maximum number of points that are processed (i.e., one iteration + * equals one point; one iteration does not equal one pass over the dataset). + * This constructor is deprecated. + * + * @param lambda The population size(0 use the default size). + * @param lowerBound Lower bound of decision variables. + * @param upperBound Upper bound of decision variables. + * @param batchSize Batch size to use for the objective calculation. + * @param maxIterations Maximum number of iterations allowed(0 means no + limit). + * @param tolerance Maximum absolute tolerance to terminate algorithm. + * @param selectionPolicy Instantiated selection policy used to calculate the + * objective. + * @param stepSize Starting sigma/step size (will be modified). + */ + ens_deprecated ActiveCMAES(const size_t lambda = 0, + const double lowerBound = -10, + const double upperBound = 10, + const size_t batchSize = 32, + const size_t maxIterations = 1000, + const double tolerance = 1e-5, + const SelectionPolicyType& selectionPolicy = SelectionPolicyType(), + double stepSize = 1); + + /** + * Optimize the given function using Active CMA-ES. The given starting point will be + * modified to store the finishing point of the algorithm, and the final + * objective value is returned. + * + * @tparam SeparableFunctionType Type of the function to be optimized. + * @tparam MatType Type of matrix to optimize. + * @tparam CallbackTypes Types of callback functions. + * @param function Function to optimize. + * @param iterate Starting point (will be modified). + * @param callbacks Callback functions. + * @return Objective value of the final point. + */ + template + typename MatType::elem_type Optimize(SeparableFunctionType& function, + MatType& iterate, + CallbackTypes&&... callbacks); + + //typedef typename TransformationPolicyType::BaseMatType BMatType; + + //! Get the population size. + size_t PopulationSize() const { return lambda; } + //! Modify the population size. + size_t& PopulationSize() { return lambda; } + + //! Get the batch size. + size_t BatchSize() const { return batchSize; } + //! Modify the batch size. + size_t& BatchSize() { return batchSize; } + + //! Get the maximum number of iterations (0 indicates no limit). + size_t MaxIterations() const { return maxIterations; } + //! Modify the maximum number of iterations (0 indicates no limit). + size_t& MaxIterations() { return maxIterations; } + + //! Get the tolerance for termination. + double Tolerance() const { return tolerance; } + //! Modify the tolerance for termination. + double& Tolerance() { return tolerance; } + + //! Get the selection policy. + const SelectionPolicyType& SelectionPolicy() const { return selectionPolicy; } + //! Modify the selection policy. + SelectionPolicyType& SelectionPolicy() { return selectionPolicy; } + + //! Get the transformation policy. + const TransformationPolicyType& TransformationPolicy() const + { return transformationPolicy; } + //! Modify the transformation policy. + TransformationPolicyType& TransformationPolicy() + { return transformationPolicy; } + + //! Get the step size. + double StepSize() const + { return stepSize; } + //! Modify the step size. + double& StepSize() + { return stepSize; } + + private: + //! Population size. + size_t lambda; + + //! The batch size for processing. + size_t batchSize; + + //! The maximum number of allowed iterations. + size_t maxIterations; + + //! The tolerance for termination. + double tolerance; + + //! The selection policy used to calculate the objective. + SelectionPolicyType selectionPolicy; + + //! The transformationPolicy used to map coordinates to the suitable domain + //! while evaluating fitness. This mapping is also done after optimization + //! has completed. + TransformationPolicyType transformationPolicy; + + //! The step size. + double stepSize; +}; + +/** + * Convenient typedef for CMAES approximation. + */ +template, + typename SelectionPolicyType = RandomSelection> +using ApproxActiveCMAES = ActiveCMAES; + +} // namespace ens + +// Include implementation. +#include "active_cmaes_impl.hpp" + +#endif diff --git a/include/ensmallen_bits/cmaes/active_cmaes_impl.hpp b/include/ensmallen_bits/cmaes/active_cmaes_impl.hpp new file mode 100644 index 000000000..77aaae482 --- /dev/null +++ b/include/ensmallen_bits/cmaes/active_cmaes_impl.hpp @@ -0,0 +1,382 @@ +/** + * @file active_cmaes_impl.hpp + * @author Marcus Edel + * @author Suvarsha Chennareddy + * + * Implementation of the Active Covariance Matrix Adaptation Evolution Strategy + * as proposed by G.A Jastrebski and D.V Arnold in "Improving Evolution + * Strategies through Active Covariance Matrix Adaptation". + * + * ensmallen is free software; you may redistribute it and/or modify it under + * the terms of the 3-clause BSD license. You should have received a copy of + * the 3-clause BSD license along with ensmallen. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef ENSMALLEN_ACTIVE_CMAES_CMAES_IMPL_HPP +#define ENSMALLEN_ACTIVE_CMAES_CMAES_IMPL_HPP + +// In case it hasn't been included yet. +#include "active_cmaes.hpp" + +#include + +#ifndef NOT_EMPTY_TRANSFORMATION +#define NOT_EMPTY_TRANSFORMATION + +/* + This partial specialization is used to throw an exception when the + TransformationPolicyType is EmptyTransformation and call a + constructor with parameters 'lowerBound' and 'upperBound' otherwise. + This shall be removed when the deprecated constructor is removed in + the next major version of ensmallen. +*/ +template +struct NotEmptyTransformation : std::true_type { + void Assign(T1& obj, double lowerBound, double upperBound) { + obj = T1(lowerBound, upperBound); + } +}; + +template class T, typename... A, typename... B> +struct NotEmptyTransformation, T> : std::false_type { + void Assign(T& obj, double lowerBound, double upperBound) { + throw std::logic_error("TransformationPolicyType is EmptyTransformation"); + } +}; + +#endif + +namespace ens { + +template +ActiveCMAES::ActiveCMAES( + const size_t lambda, + const TransformationPolicyType& + transformationPolicy, + const size_t batchSize, + const size_t maxIterations, + const double tolerance, + const SelectionPolicyType& selectionPolicy, + double stepSizeIn) : + lambda(lambda), + batchSize(batchSize), + maxIterations(maxIterations), + tolerance(tolerance), + selectionPolicy(selectionPolicy), + transformationPolicy(transformationPolicy), + stepSize(stepSizeIn) +{ /* Nothing to do. */ } + +template +ActiveCMAES::ActiveCMAES( + const size_t lambda, + const double lowerBound, + const double upperBound, + const size_t batchSize, + const size_t maxIterations, + const double tolerance, + const SelectionPolicyType& selectionPolicy, + double stepSizeIn) : + lambda(lambda), + batchSize(batchSize), + maxIterations(maxIterations), + tolerance(tolerance), + selectionPolicy(selectionPolicy), + stepSize(stepSizeIn) +{ + Warn << "This is a deprecated constructor and will be removed in a " + "future version of ensmallen" << std::endl; + NotEmptyTransformation> d; + d.Assign(transformationPolicy, lowerBound, upperBound); +} + + +//! Optimize the function (minimize). +template +template +typename MatType::elem_type ActiveCMAES::Optimize( + SeparableFunctionType& function, + MatType& iterateIn, + CallbackTypes&&... callbacks) +{ + // Convenience typedefs. + typedef typename MatType::elem_type ElemType; + typedef typename MatTypeTraits::BaseMatType BaseMatType; + + // Make sure that we have the methods that we need. Long name... + traits::CheckArbitrarySeparableFunctionTypeAPI< + SeparableFunctionType, BaseMatType>(); + RequireDenseFloatingPointType(); + + BaseMatType& iterate = (BaseMatType&) iterateIn; + + // Find the number of functions to use. + const size_t numFunctions = function.NumFunctions(); + + // Population size. + if (lambda == 0) + lambda = (4 + std::round(3 * std::log(iterate.n_elem))) * 10; + + // Parent weights. + const size_t mu = std::round(lambda / 4); + + /* + BaseMatType w = std::log(mu + 0.5) - arma::log( + arma::linspace(0, mu - 1, mu) + 1.0); + */ + const double w = 1.0 / mu; + + // Number of effective solutions. + const double muEffective = mu; + + // Step size control parameters. + BaseMatType sigma(2, 1); // sigma is vector-shaped. + if (stepSize == 0) + sigma(0) = transformationPolicy.InitialStepSize(); + else + sigma(0) = stepSize; + + const double cs = 4.0 / (iterate.n_elem + 4); + const double ds = 1 + cs + 2 * std::max(std::sqrt((muEffective - 1) / + (iterate.n_elem + 1)) - 1, 0.0); + const double enn = std::sqrt(iterate.n_elem) * (1.0 - 1.0 / + (4.0 * iterate.n_elem) + 1.0 / (21 * std::pow(iterate.n_elem, 2))); + + // Covariance update parameters. + // + // Cumulation for distribution. + const double cc = cs; + const double ccov = 2.0 / std::pow((iterate.n_elem + std::sqrt(2)), 2); + const double beta = (4.0 * mu - 2.0) / (std::pow((iterate.n_elem + 12), 2) + + 4 * mu); + + std::vector mPosition(2, BaseMatType(iterate.n_rows, + iterate.n_cols)); + mPosition[0] = iterate; + + BaseMatType step(iterate.n_rows, iterate.n_cols); + step.zeros(); + + BaseMatType transformedIterate = transformationPolicy.Transform(iterate); + + // Calculate the first objective function. + ElemType currentObjective = 0; + for (size_t f = 0; f < numFunctions; f += batchSize) + { + const size_t effectiveBatchSize = std::min(batchSize, numFunctions - f); + const ElemType objective = function.Evaluate(transformedIterate, f, + effectiveBatchSize); + currentObjective += objective; + + Callback::Evaluate(*this, function, transformedIterate, objective, + callbacks...); + } + + ElemType overallObjective = currentObjective; + ElemType lastObjective = std::numeric_limits::max(); + + // Population parameters. + std::vector pStep(lambda, BaseMatType(iterate.n_rows, + iterate.n_cols)); + std::vector pPosition(lambda, BaseMatType(iterate.n_rows, + iterate.n_cols)); + BaseMatType pObjective(lambda, 1); // pObjective is vector-shaped. + std::vector ps(2, BaseMatType(iterate.n_rows, iterate.n_cols)); + ps[0].zeros(); + ps[1].zeros(); + std::vector pc = ps; + std::vector C(2, BaseMatType(iterate.n_elem, iterate.n_elem)); + C[0].eye(); + + // Covariance matrix parameters. + arma::Col eigval; // TODO: might need a more general type. + BaseMatType eigvec; + BaseMatType eigvalZero(iterate.n_elem, 1); // eigvalZero is vector-shaped. + eigvalZero.zeros(); + + // The current visitation order (sorted by population objectives). + arma::uvec idx = arma::linspace(0, lambda - 1, lambda); + + // Controls early termination of the optimization process. + bool terminate = false; + + // Now iterate! + terminate |= Callback::BeginOptimization(*this, function, + transformedIterate, callbacks...); + + size_t idx0, idx1; + + for (size_t i = 1; (i != maxIterations) && !terminate; ++i) + { + // To keep track of where we are. + idx0 = (i - 1) % 2; + idx1 = i % 2; + + // Perform Cholesky decomposition. If the matrix is not positive definite, + // add a small value and try again. + BaseMatType covLower; + while (!arma::chol(covLower, C[idx0], "lower")) + C[idx0].diag() += std::numeric_limits::epsilon(); + + arma::eig_sym(eigval, eigvec, C[idx0]); + + for (size_t j = 0; j < lambda; ++j) + { + if (iterate.n_rows > iterate.n_cols) + { + pStep[idx(j)] = covLower * + arma::randn(iterate.n_rows, iterate.n_cols); + } + else + { + pStep[idx(j)] = arma::randn(iterate.n_rows, iterate.n_cols) + * covLower.t(); + } + + pPosition[idx(j)] = mPosition[idx0] + sigma(idx0) * pStep[idx(j)]; + + // Calculate the objective function. + pObjective(idx(j)) = selectionPolicy.Select(function, batchSize, + transformationPolicy.Transform(pPosition[idx(j)]), callbacks...); + } + + // Sort population. + idx = arma::sort_index(pObjective); + + step = w * pStep[idx(0)]; + for (size_t j = 1; j < mu; ++j) + step += w * pStep[idx(j)]; + + mPosition[idx1] = mPosition[idx0] + sigma(idx0) * step; + + // Calculate the objective function. + currentObjective = selectionPolicy.Select(function, batchSize, + transformationPolicy.Transform(mPosition[idx1]), callbacks...); + + // Update best parameters. + if (currentObjective < overallObjective) + { + overallObjective = currentObjective; + iterate = mPosition[idx1]; + + transformedIterate = transformationPolicy.Transform(iterate); + terminate |= Callback::StepTaken(*this, function, + transformedIterate, callbacks...); + } + + // Update Step Size. + if (iterate.n_rows > iterate.n_cols) + { + ps[idx1] = (1 - cs) * ps[idx0] + std::sqrt( + cs * (2 - cs) * muEffective) * + eigvec * diagmat(1 / eigval) * eigvec.t() * step; + } + else + { + ps[idx1] = (1 - cs) * ps[idx0] + std::sqrt( + cs * (2 - cs) * muEffective) * step * + eigvec * diagmat(1 / eigval) * eigvec.t(); + } + + const ElemType psNorm = arma::norm(ps[idx1]); + sigma(idx1) = sigma(idx0) * std::exp(cs / ds * ( psNorm / enn - 1)); + + + pc[idx1] = (1 - cc) * pc[idx0] + std::sqrt(cc * (2 - cc) * + muEffective) * step; + + + if (iterate.n_rows > iterate.n_cols) + { + C[idx1] = (1 - ccov) * C[idx0] + ccov * + (pc[idx1] * pc[idx1].t()); + + for (size_t j = 0; j < mu; ++j) + { + C[idx1] = C[idx1] + beta * w * + pStep[idx(j)] * pStep[idx(j)].t(); + } + + for (size_t j = mu; j < lambda; ++j) + { + C[idx1] = C[idx1] - beta * w * + pStep[idx(j)] * pStep[idx(j)].t(); + } + + } + else + { + C[idx1] = (1 - ccov) * C[idx0] + ccov * + (pc[idx1].t() * pc[idx1]); + + for (size_t j = 0; j < mu; ++j) + { + C[idx1] = C[idx1] + beta * w * + pStep[idx(j)].t() * pStep[idx(j)]; + } + + for (size_t j = mu; j < lambda; ++j) + { + C[idx1] = C[idx1] - beta * w * + pStep[idx(j)].t() * pStep[idx(j)]; + } + + } + + arma::eig_sym(eigval, eigvec, C[idx1]); + const arma::uvec negativeEigval = arma::find(eigval < 0, 1); + if (!negativeEigval.is_empty()) + { + if (negativeEigval(0) == 0) + { + C[idx1].zeros(); + } + else + { + C[idx1] = eigvec.cols(0, negativeEigval(0) - 1) * + arma::diagmat(eigval.subvec(0, negativeEigval(0) - 1)) * + eigvec.cols(0, negativeEigval(0) - 1).t(); + } + } + + // Output current objective function. + Info << "CMA-ES: iteration " << i << ", objective " << overallObjective + << "." << std::endl; + + if (std::isnan(overallObjective) || std::isinf(overallObjective)) + { + Warn << "CMA-ES: converged to " << overallObjective << "; " + << "terminating with failure. Try a smaller step size?" << std::endl; + + iterate = transformationPolicy.Transform(iterate); + + Callback::EndOptimization(*this, function, iterate, callbacks...); + return overallObjective; + } + + if (std::abs(lastObjective - overallObjective) < tolerance) + { + Info << "CMA-ES: minimized within tolerance " << tolerance << "; " + << "terminating optimization." << std::endl; + + iterate = transformationPolicy.Transform(iterate); + + Callback::EndOptimization(*this, function, iterate, callbacks...); + return overallObjective; + } + + lastObjective = overallObjective; + } + + iterate = transformationPolicy.Transform(iterate); + + Callback::EndOptimization(*this, function, iterate, callbacks...); + return overallObjective; +} + +} // namespace ens + +#endif diff --git a/include/ensmallen_bits/cmaes/cmaes_impl.hpp b/include/ensmallen_bits/cmaes/cmaes_impl.hpp index 037424448..2009cfc2a 100644 --- a/include/ensmallen_bits/cmaes/cmaes_impl.hpp +++ b/include/ensmallen_bits/cmaes/cmaes_impl.hpp @@ -20,6 +20,9 @@ #include +#ifndef NOT_EMPTY_TRANSFORMATION +#define NOT_EMPTY_TRANSFORMATION + /* This partial specialization is used to throw an exception when the TransformationPolicyType is EmptyTransformation and call a @@ -41,6 +44,8 @@ struct NotEmptyTransformation, T> : std::false_type { } }; +#endif + namespace ens { template diff --git a/include/ensmallen_bits/cmaes/transformation_policies/boundary_box_constraint.hpp b/include/ensmallen_bits/cmaes/transformation_policies/boundary_box_constraint.hpp index 9583a0976..fdc1d82e9 100644 --- a/include/ensmallen_bits/cmaes/transformation_policies/boundary_box_constraint.hpp +++ b/include/ensmallen_bits/cmaes/transformation_policies/boundary_box_constraint.hpp @@ -10,6 +10,8 @@ * the 3-clause BSD license along with ensmallen. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ +#ifndef ENSMALLEN_CMAES_BOUNDARY_BOX_TRANSFORMATION_HPP +#define ENSMALLEN_CMAES_BOUNDARY_BOX_TRANSFORMATION_HPP namespace ens { @@ -156,3 +158,5 @@ class BoundaryBoxConstraint }; } // namespace ens + +#endif diff --git a/include/ensmallen_bits/cmaes/transformation_policies/empty_transformation.hpp b/include/ensmallen_bits/cmaes/transformation_policies/empty_transformation.hpp index f81083a13..eb8618116 100644 --- a/include/ensmallen_bits/cmaes/transformation_policies/empty_transformation.hpp +++ b/include/ensmallen_bits/cmaes/transformation_policies/empty_transformation.hpp @@ -9,6 +9,8 @@ * the 3-clause BSD license along with ensmallen. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ +#ifndef ENSMALLEN_CMAES_EMPTY_TRANSFORMATION_HPP +#define ENSMALLEN_CMAES_EMPTY_TRANSFORMATION_HPP namespace ens { @@ -41,3 +43,5 @@ class EmptyTransformation }; } // namespace ens + +#endif From 52c4d63b890d8c631ec6acf93f1bf0db62d19ead Mon Sep 17 00:00:00 2001 From: Suvarsha Chennareddy Date: Mon, 12 Jun 2023 19:03:32 +0530 Subject: [PATCH 13/34] add tests --- include/ensmallen_bits/cmaes/active_cmaes.hpp | 2 - .../cmaes/active_cmaes_impl.hpp | 13 +-- tests/CMakeLists.txt | 1 + tests/active_cmaes_test.cpp | 96 +++++++++++++++++++ tests/callbacks_test.cpp | 13 ++- 5 files changed, 114 insertions(+), 11 deletions(-) create mode 100644 tests/active_cmaes_test.cpp diff --git a/include/ensmallen_bits/cmaes/active_cmaes.hpp b/include/ensmallen_bits/cmaes/active_cmaes.hpp index 76e369954..8f3b17882 100644 --- a/include/ensmallen_bits/cmaes/active_cmaes.hpp +++ b/include/ensmallen_bits/cmaes/active_cmaes.hpp @@ -138,8 +138,6 @@ class ActiveCMAES MatType& iterate, CallbackTypes&&... callbacks); - //typedef typename TransformationPolicyType::BaseMatType BMatType; - //! Get the population size. size_t PopulationSize() const { return lambda; } //! Modify the population size. diff --git a/include/ensmallen_bits/cmaes/active_cmaes_impl.hpp b/include/ensmallen_bits/cmaes/active_cmaes_impl.hpp index 77aaae482..6176ab12f 100644 --- a/include/ensmallen_bits/cmaes/active_cmaes_impl.hpp +++ b/include/ensmallen_bits/cmaes/active_cmaes_impl.hpp @@ -120,13 +120,10 @@ typename MatType::elem_type ActiveCMAES(0, mu - 1, mu) + 1.0); - */ + //Recombination weight (w = 1/ (parent number)); const double w = 1.0 / mu; // Number of effective solutions. @@ -343,12 +340,12 @@ typename MatType::elem_type ActiveCMAES +#include "catch.hpp" +#include "test_function_tools.hpp" + +using namespace ens; +using namespace ens::test; + +/** + * Run Active CMA-ES with the full selection policy on Rosenbrock function and + * make sure the results are acceptable. + * This test uses the deprecated constructor and therefore can be removed + * in a future version of ensmallen. + */ + +TEST_CASE("ActiveCMAESDeprecatedConstructorRosenbrockFunctionTest", "[ActiveCMAESTest]") +{ + ActiveCMAES> + activecmaes(0, 0, 2, 32, 0, 1e-3); + activecmaes.StepSize() = 0.075; + FunctionTest(activecmaes, 0.1, 0.15); +} + +/** + * Run Active CMA-ES with the full selection policy on Rosenbrock function and + * make sure the results are acceptable. + */ +TEST_CASE("ActiveCMAESRosenbrockFunctionTest", "[ActiveCMAESTest]") +{ + BoundaryBoxConstraint<> b(0, 2); + ActiveCMAES> + activecmaes(0, b, 32, 0, 1e-3); + activecmaes.StepSize() = 0.075; + FunctionTest(activecmaes, 0.1, 0.15); +} + +/** + * Run Active CMA-ES with the random selection policy on Rosenbrock function and + * make sure the results are acceptable. + */ +TEST_CASE("ApproxActiveCMAESRosenbrockFunctionTest", "[ActiveCMAESTest]") +{ + BoundaryBoxConstraint<> b(0, 2); + ApproxActiveCMAES> + activecmaes(0, b, 32, 0, 1e-3); + activecmaes.StepSize() = 0.075; + FunctionTest(activecmaes, 0.1, 0.15); +} + +/** + * Run Active CMA-ES with the full selection policy on Rosenbrock function and + * make sure the results are acceptable. Use arma::fmat. + */ +TEST_CASE("ActiveCMAESRosenbrockFunctionFMatTest", "[ActiveCMAESTest]") +{ + BoundaryBoxConstraint b(0, 2); + ActiveCMAES> + activecmaes(0, b, 32, 0, 1e-3); + activecmaes.StepSize() = 0.075; + FunctionTest(activecmaes, 0.1, 0.15); +} + +/** + * Run Active CMA-ES with the random selection policy on Rosenbrock function and + * make sure the results are acceptable. Use arma::fmat. + */ +TEST_CASE("ApproxActiveCMAESRosenbrockFunctionFMatTest", "[ActiveCMAESTest]") +{ + BoundaryBoxConstraint b(0, 2); + ApproxActiveCMAES> + activecmaes(0, b, 32, 0, 1e-3); + activecmaes.StepSize() = 0.075; + FunctionTest(activecmaes, 0.1, 0.15); +} + +/** + * Run Active CMA-ES with the random selection and empty transformation policies + * on Rosenbrock function and make sure the results are acceptable. + * Use arma::fmat. + */ +TEST_CASE("ApproxActiveCMAESEmptyTransformationRosenbrockFunctionFMatTest", + "[ActiveCMAESTest]") +{ + ApproxActiveCMAES> + activecmaes(0, EmptyTransformation(), 32, 0, 1e-3); + LogisticRegressionFunctionTest(activecmaes, 0.01, 0.02, 5); +} \ No newline at end of file diff --git a/tests/callbacks_test.cpp b/tests/callbacks_test.cpp index ec87b8855..03c04dfff 100644 --- a/tests/callbacks_test.cpp +++ b/tests/callbacks_test.cpp @@ -256,6 +256,17 @@ TEST_CASE("EarlyStopAtMinLossCustomLambdaTest", "[CallbacksTest]") REQUIRE(std::abs(coordinates[i]) >= 3.0); } +/** + * Make sure we invoke all callbacks (CMAES). + */ +TEST_CASE("ActiveCMAESCallbacksFullFunctionTest", "[CallbacksTest]") +{ + BoundaryBoxConstraint<> b(-1, 1); + ActiveCMAES> optimizer(0, b, 32, 3, 1e-3); + CallbacksFullFunctionTest(optimizer, true, false, false, false, true, true, + false, false, true); +} + /** * Make sure we invoke all callbacks (AdaBound). */ @@ -315,7 +326,7 @@ TEST_CASE("CMAESCallbacksFullFunctionTest", "[CallbacksTest]") BoundaryBoxConstraint<> b(-1, 1); CMAES> optimizer(0, b, 32, 3, 1e-3); CallbacksFullFunctionTest(optimizer, true, false, false, false, true, true, - false, false, true); + false, false, true); } /** From 3c9bcfb9470122e196848278f10bcf5b0a54acb8 Mon Sep 17 00:00:00 2001 From: Suvarsha Chennareddy Date: Mon, 12 Jun 2023 23:47:00 +0530 Subject: [PATCH 14/34] fix implementation error and update tests --- include/ensmallen_bits/cmaes/active_cmaes_impl.hpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/include/ensmallen_bits/cmaes/active_cmaes_impl.hpp b/include/ensmallen_bits/cmaes/active_cmaes_impl.hpp index 6176ab12f..94d309f45 100644 --- a/include/ensmallen_bits/cmaes/active_cmaes_impl.hpp +++ b/include/ensmallen_bits/cmaes/active_cmaes_impl.hpp @@ -123,7 +123,7 @@ typename MatType::elem_type ActiveCMAES Date: Tue, 13 Jun 2023 00:00:10 +0530 Subject: [PATCH 15/34] remove empty line --- tests/active_cmaes_test.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/active_cmaes_test.cpp b/tests/active_cmaes_test.cpp index c37d838ad..8c212501e 100644 --- a/tests/active_cmaes_test.cpp +++ b/tests/active_cmaes_test.cpp @@ -21,7 +21,6 @@ using namespace ens::test; * This test uses the deprecated constructor and therefore can be removed * in a future version of ensmallen. */ - TEST_CASE("ActiveCMAESDeprecatedConstructorRosenbrockFunctionTest", "[ActiveCMAESTest]") { ActiveCMAES> @@ -93,4 +92,4 @@ TEST_CASE("ApproxActiveCMAESEmptyTransformationRosenbrockFunctionFMatTest", ApproxActiveCMAES> activecmaes(0, EmptyTransformation(), 32, 0, 1e-3); LogisticRegressionFunctionTest(activecmaes, 0.01, 0.02, 5); -} \ No newline at end of file +} From 9ae0e6d9793047cc090b5867c96956e10aee64af Mon Sep 17 00:00:00 2001 From: Suvarsha Chennareddy Date: Tue, 13 Jun 2023 00:06:18 +0530 Subject: [PATCH 16/34] update cmaes tests --- tests/cmaes_test.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/cmaes_test.cpp b/tests/cmaes_test.cpp index 19639a1bd..8b60f080e 100644 --- a/tests/cmaes_test.cpp +++ b/tests/cmaes_test.cpp @@ -25,7 +25,7 @@ using namespace ens::test; */ TEST_CASE("CMAESDeprecatedConstructorLogisticRegressionTest", "[CMAESTest]") { - CMAES> cmaes(0, -10, 10, 32, 500, 1e-3); + CMAES> cmaes(0, -10, 10, 16, 500, 1e-3); LogisticRegressionFunctionTest(cmaes, 0.003, 0.006, 5); } @@ -36,7 +36,7 @@ TEST_CASE("CMAESDeprecatedConstructorLogisticRegressionTest", "[CMAESTest]") TEST_CASE("CMAESLogisticRegressionTest", "[CMAESTest]") { BoundaryBoxConstraint<> b(-10, 10); - CMAES> cmaes(0, b, 32, 500, 1e-3); + CMAES> cmaes(0, b, 16, 500, 1e-3); LogisticRegressionFunctionTest(cmaes, 0.003, 0.006, 5); } @@ -47,7 +47,7 @@ TEST_CASE("CMAESLogisticRegressionTest", "[CMAESTest]") TEST_CASE("ApproxCMAESLogisticRegressionTest", "[CMAESTest]") { BoundaryBoxConstraint<> b(-10, 10); - ApproxCMAES> cmaes(0, b, 32, 500, 1e-3); + ApproxCMAES> cmaes(0, b, 16, 500, 1e-3); LogisticRegressionFunctionTest(cmaes, 0.003, 0.006, 5); } @@ -58,7 +58,7 @@ TEST_CASE("ApproxCMAESLogisticRegressionTest", "[CMAESTest]") TEST_CASE("CMAESLogisticRegressionFMatTest", "[CMAESTest]") { BoundaryBoxConstraint b(-10, 10); - CMAES> cmaes(0, b, 32, 500, 1e-3); + CMAES> cmaes(0, b, 16, 500, 1e-3); LogisticRegressionFunctionTest(cmaes, 0.01, 0.02, 5); } @@ -69,7 +69,7 @@ TEST_CASE("CMAESLogisticRegressionFMatTest", "[CMAESTest]") TEST_CASE("ApproxCMAESLogisticRegressionFMatTest", "[CMAESTest]") { BoundaryBoxConstraint b(-10, 10); - ApproxCMAES> cmaes(0, b, 32, 500, 1e-3); + ApproxCMAES> cmaes(0, b, 16, 500, 1e-3); LogisticRegressionFunctionTest(cmaes, 0.01, 0.02, 5); } @@ -82,6 +82,6 @@ TEST_CASE("ApproxCMAESEmptyTransformationLogisticRegressionFMatTest", "[CMAESTest]") { ApproxCMAES> - cmaes(0, EmptyTransformation(), 32, 500, 1e-3); + cmaes(0, EmptyTransformation(), 16, 500, 1e-3); LogisticRegressionFunctionTest(cmaes, 0.01, 0.02, 5); } \ No newline at end of file From a0843dc2f42adff050cabf2b4b53950f15f8b5c9 Mon Sep 17 00:00:00 2001 From: Suvarsha Chennareddy Date: Tue, 13 Jun 2023 00:20:36 +0530 Subject: [PATCH 17/34] update active cmaes tests --- tests/active_cmaes_test.cpp | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/tests/active_cmaes_test.cpp b/tests/active_cmaes_test.cpp index 8c212501e..0199ce896 100644 --- a/tests/active_cmaes_test.cpp +++ b/tests/active_cmaes_test.cpp @@ -21,12 +21,13 @@ using namespace ens::test; * This test uses the deprecated constructor and therefore can be removed * in a future version of ensmallen. */ + TEST_CASE("ActiveCMAESDeprecatedConstructorRosenbrockFunctionTest", "[ActiveCMAESTest]") { ActiveCMAES> - activecmaes(0, 0, 2, 32, 0, 1e-3); + activecmaes(0, 0, 2, 16, 0, 1e-3); activecmaes.StepSize() = 0.075; - FunctionTest(activecmaes, 0.1, 0.15); + FunctionTest(activecmaes, 0.1, 0.1); } /** @@ -37,9 +38,9 @@ TEST_CASE("ActiveCMAESRosenbrockFunctionTest", "[ActiveCMAESTest]") { BoundaryBoxConstraint<> b(0, 2); ActiveCMAES> - activecmaes(0, b, 32, 0, 1e-3); + activecmaes(0, b, 16, 0, 1e-3); activecmaes.StepSize() = 0.075; - FunctionTest(activecmaes, 0.1, 0.15); + FunctionTest(activecmaes, 0.1, 0.1); } /** @@ -50,9 +51,9 @@ TEST_CASE("ApproxActiveCMAESRosenbrockFunctionTest", "[ActiveCMAESTest]") { BoundaryBoxConstraint<> b(0, 2); ApproxActiveCMAES> - activecmaes(0, b, 32, 0, 1e-3); + activecmaes(0, b, 16, 0, 1e-3); activecmaes.StepSize() = 0.075; - FunctionTest(activecmaes, 0.1, 0.15); + FunctionTest(activecmaes, 0.1, 0.1); } /** @@ -63,9 +64,9 @@ TEST_CASE("ActiveCMAESRosenbrockFunctionFMatTest", "[ActiveCMAESTest]") { BoundaryBoxConstraint b(0, 2); ActiveCMAES> - activecmaes(0, b, 32, 0, 1e-3); + activecmaes(0, b, 16, 0, 1e-3); activecmaes.StepSize() = 0.075; - FunctionTest(activecmaes, 0.1, 0.15); + FunctionTest(activecmaes, 0.1, 0.1); } /** @@ -76,9 +77,9 @@ TEST_CASE("ApproxActiveCMAESRosenbrockFunctionFMatTest", "[ActiveCMAESTest]") { BoundaryBoxConstraint b(0, 2); ApproxActiveCMAES> - activecmaes(0, b, 32, 0, 1e-3); + activecmaes(0, b, 16, 0, 1e-3); activecmaes.StepSize() = 0.075; - FunctionTest(activecmaes, 0.1, 0.15); + FunctionTest(activecmaes, 0.1, 0.1); } /** @@ -86,10 +87,11 @@ TEST_CASE("ApproxActiveCMAESRosenbrockFunctionFMatTest", "[ActiveCMAESTest]") * on Rosenbrock function and make sure the results are acceptable. * Use arma::fmat. */ -TEST_CASE("ApproxActiveCMAESEmptyTransformationRosenbrockFunctionFMatTest", +TEST_CASE("ApproxActiveCMAESEmptyTransformationLogisticRegressionFMatTest", "[ActiveCMAESTest]") { ApproxActiveCMAES> - activecmaes(0, EmptyTransformation(), 32, 0, 1e-3); + activecmaes(0, EmptyTransformation(), 16, 0, 1e-3); + activecmaes.StepSize() = 0.55; LogisticRegressionFunctionTest(activecmaes, 0.01, 0.02, 5); -} +} \ No newline at end of file From 0cc8d0c6f54beaab59f39f37574fd2aa4fd72dae Mon Sep 17 00:00:00 2001 From: Suvarsha Chennareddy Date: Tue, 13 Jun 2023 13:43:01 +0530 Subject: [PATCH 18/34] update tests --- tests/active_cmaes_test.cpp | 1 - tests/cmaes_test.cpp | 10 +++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/active_cmaes_test.cpp b/tests/active_cmaes_test.cpp index 0199ce896..5a0b17083 100644 --- a/tests/active_cmaes_test.cpp +++ b/tests/active_cmaes_test.cpp @@ -21,7 +21,6 @@ using namespace ens::test; * This test uses the deprecated constructor and therefore can be removed * in a future version of ensmallen. */ - TEST_CASE("ActiveCMAESDeprecatedConstructorRosenbrockFunctionTest", "[ActiveCMAESTest]") { ActiveCMAES> diff --git a/tests/cmaes_test.cpp b/tests/cmaes_test.cpp index 8b60f080e..369599466 100644 --- a/tests/cmaes_test.cpp +++ b/tests/cmaes_test.cpp @@ -25,7 +25,7 @@ using namespace ens::test; */ TEST_CASE("CMAESDeprecatedConstructorLogisticRegressionTest", "[CMAESTest]") { - CMAES> cmaes(0, -10, 10, 16, 500, 1e-3); + CMAES> cmaes(0, -10, 10, 32, 500, 1e-3); LogisticRegressionFunctionTest(cmaes, 0.003, 0.006, 5); } @@ -36,7 +36,7 @@ TEST_CASE("CMAESDeprecatedConstructorLogisticRegressionTest", "[CMAESTest]") TEST_CASE("CMAESLogisticRegressionTest", "[CMAESTest]") { BoundaryBoxConstraint<> b(-10, 10); - CMAES> cmaes(0, b, 16, 500, 1e-3); + CMAES> cmaes(0, b, 32, 500, 1e-3); LogisticRegressionFunctionTest(cmaes, 0.003, 0.006, 5); } @@ -47,7 +47,7 @@ TEST_CASE("CMAESLogisticRegressionTest", "[CMAESTest]") TEST_CASE("ApproxCMAESLogisticRegressionTest", "[CMAESTest]") { BoundaryBoxConstraint<> b(-10, 10); - ApproxCMAES> cmaes(0, b, 16, 500, 1e-3); + ApproxCMAES> cmaes(256, b, 16, 500, 1e-3); LogisticRegressionFunctionTest(cmaes, 0.003, 0.006, 5); } @@ -58,7 +58,7 @@ TEST_CASE("ApproxCMAESLogisticRegressionTest", "[CMAESTest]") TEST_CASE("CMAESLogisticRegressionFMatTest", "[CMAESTest]") { BoundaryBoxConstraint b(-10, 10); - CMAES> cmaes(0, b, 16, 500, 1e-3); + CMAES> cmaes(0, b, 32, 500, 1e-3); LogisticRegressionFunctionTest(cmaes, 0.01, 0.02, 5); } @@ -69,7 +69,7 @@ TEST_CASE("CMAESLogisticRegressionFMatTest", "[CMAESTest]") TEST_CASE("ApproxCMAESLogisticRegressionFMatTest", "[CMAESTest]") { BoundaryBoxConstraint b(-10, 10); - ApproxCMAES> cmaes(0, b, 16, 500, 1e-3); + ApproxCMAES> cmaes(256, b, 16, 500, 1e-3); LogisticRegressionFunctionTest(cmaes, 0.01, 0.02, 5); } From 37b342c83ccb5b4df73d3440460d45a08ed9b2ae Mon Sep 17 00:00:00 2001 From: Suvarsha Chennareddy Date: Wed, 14 Jun 2023 00:27:25 +0530 Subject: [PATCH 19/34] update comments --- include/ensmallen_bits/cmaes/active_cmaes.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/ensmallen_bits/cmaes/active_cmaes.hpp b/include/ensmallen_bits/cmaes/active_cmaes.hpp index 8f3b17882..0c29b2b9c 100644 --- a/include/ensmallen_bits/cmaes/active_cmaes.hpp +++ b/include/ensmallen_bits/cmaes/active_cmaes.hpp @@ -90,7 +90,7 @@ class ActiveCMAES double stepSize = 0); /** - * Construct the CMA-ES optimizer with the given function and parameters + * Construct the Active CMA-ES optimizer with the given function and parameters * (including lower and upper bounds). The defaults here are not necessarily * good for the given problem, so it is suggested that the values used be * tailored to the task at hand. The maximum number of iterations refers to @@ -203,7 +203,7 @@ class ActiveCMAES }; /** - * Convenient typedef for CMAES approximation. + * Convenient typedef for Active CMAES approximation. */ template, typename SelectionPolicyType = RandomSelection> From ef6396641f2c0599957262180e250d133fae77ca Mon Sep 17 00:00:00 2001 From: Suvarsha Chennareddy Date: Wed, 14 Jun 2023 16:29:40 +0530 Subject: [PATCH 20/34] Update hyperparameters used in tests --- tests/active_cmaes_test.cpp | 4 ++-- tests/cmaes_test.cpp | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/active_cmaes_test.cpp b/tests/active_cmaes_test.cpp index 5a0b17083..85301661a 100644 --- a/tests/active_cmaes_test.cpp +++ b/tests/active_cmaes_test.cpp @@ -50,7 +50,7 @@ TEST_CASE("ApproxActiveCMAESRosenbrockFunctionTest", "[ActiveCMAESTest]") { BoundaryBoxConstraint<> b(0, 2); ApproxActiveCMAES> - activecmaes(0, b, 16, 0, 1e-3); + activecmaes(256, b, 16, 0, 1e-3); activecmaes.StepSize() = 0.075; FunctionTest(activecmaes, 0.1, 0.1); } @@ -76,7 +76,7 @@ TEST_CASE("ApproxActiveCMAESRosenbrockFunctionFMatTest", "[ActiveCMAESTest]") { BoundaryBoxConstraint b(0, 2); ApproxActiveCMAES> - activecmaes(0, b, 16, 0, 1e-3); + activecmaes(256, b, 16, 0, 1e-3); activecmaes.StepSize() = 0.075; FunctionTest(activecmaes, 0.1, 0.1); } diff --git a/tests/cmaes_test.cpp b/tests/cmaes_test.cpp index 369599466..0e3e5df9b 100644 --- a/tests/cmaes_test.cpp +++ b/tests/cmaes_test.cpp @@ -37,6 +37,7 @@ TEST_CASE("CMAESLogisticRegressionTest", "[CMAESTest]") { BoundaryBoxConstraint<> b(-10, 10); CMAES> cmaes(0, b, 32, 500, 1e-3); + cmaes.StepSize() = 0.6; LogisticRegressionFunctionTest(cmaes, 0.003, 0.006, 5); } @@ -48,6 +49,7 @@ TEST_CASE("ApproxCMAESLogisticRegressionTest", "[CMAESTest]") { BoundaryBoxConstraint<> b(-10, 10); ApproxCMAES> cmaes(256, b, 16, 500, 1e-3); + cmaes.StepSize() = 0.6; LogisticRegressionFunctionTest(cmaes, 0.003, 0.006, 5); } @@ -69,7 +71,7 @@ TEST_CASE("CMAESLogisticRegressionFMatTest", "[CMAESTest]") TEST_CASE("ApproxCMAESLogisticRegressionFMatTest", "[CMAESTest]") { BoundaryBoxConstraint b(-10, 10); - ApproxCMAES> cmaes(256, b, 16, 500, 1e-3); + ApproxCMAES> cmaes(0, b, 16, 500, 1e-3); LogisticRegressionFunctionTest(cmaes, 0.01, 0.02, 5); } From 0785776adc041d92b6d52c4a82d3c5ac5f6a11b5 Mon Sep 17 00:00:00 2001 From: Suvarsha Chennareddy Date: Mon, 10 Jul 2023 20:02:07 +0530 Subject: [PATCH 21/34] add missing comment --- include/ensmallen_bits/cmaes/cmaes.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/include/ensmallen_bits/cmaes/cmaes.hpp b/include/ensmallen_bits/cmaes/cmaes.hpp index 934235c3b..bf40fa083 100644 --- a/include/ensmallen_bits/cmaes/cmaes.hpp +++ b/include/ensmallen_bits/cmaes/cmaes.hpp @@ -194,6 +194,7 @@ class CMAES //! has completed. TransformationPolicyType transformationPolicy; + //! The step size. double stepSize; }; From c8719fb7ad9e2b4ee6193336f56840a561dd2dfd Mon Sep 17 00:00:00 2001 From: Suvarsha Chennareddy Date: Mon, 10 Jul 2023 21:02:59 +0530 Subject: [PATCH 22/34] minor comment fix in callbacks_test.cpp --- tests/callbacks_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/callbacks_test.cpp b/tests/callbacks_test.cpp index 03c04dfff..f5a5205eb 100644 --- a/tests/callbacks_test.cpp +++ b/tests/callbacks_test.cpp @@ -257,7 +257,7 @@ TEST_CASE("EarlyStopAtMinLossCustomLambdaTest", "[CallbacksTest]") } /** - * Make sure we invoke all callbacks (CMAES). + * Make sure we invoke all callbacks (ActiveCMAES). */ TEST_CASE("ActiveCMAESCallbacksFullFunctionTest", "[CallbacksTest]") { From ee7b695d4c11ec6226098b316e7c67259df87895 Mon Sep 17 00:00:00 2001 From: Suvarsha Chennareddy Date: Wed, 12 Jul 2023 23:44:44 +0530 Subject: [PATCH 23/34] add patience and update termination conditions --- .../cmaes/active_cmaes_impl.hpp | 76 ++++++++++++------- include/ensmallen_bits/cmaes/cmaes_impl.hpp | 68 +++++++++++------ 2 files changed, 94 insertions(+), 50 deletions(-) diff --git a/include/ensmallen_bits/cmaes/active_cmaes_impl.hpp b/include/ensmallen_bits/cmaes/active_cmaes_impl.hpp index 94d309f45..2314cdab4 100644 --- a/include/ensmallen_bits/cmaes/active_cmaes_impl.hpp +++ b/include/ensmallen_bits/cmaes/active_cmaes_impl.hpp @@ -205,6 +205,11 @@ typename MatType::elem_type ActiveCMAES iterate.n_cols) { pStep[idx(j)] = covLower * - arma::randn(iterate.n_rows, iterate.n_cols); + arma::randn(iterate.n_rows, iterate.n_cols); } else { pStep[idx(j)] = arma::randn(iterate.n_rows, iterate.n_cols) - * covLower.t(); + * covLower.t(); } pPosition[idx(j)] = mPosition[idx0] + sigma(idx0) * pStep[idx(j)]; // Calculate the objective function. pObjective(idx(j)) = selectionPolicy.Select(function, batchSize, - transformationPolicy.Transform(pPosition[idx(j)]), callbacks...); + transformationPolicy.Transform(pPosition[idx(j)]), callbacks...); + } // Sort population. idx = arma::sort_index(pObjective); - step = w * pStep[idx(0)]; + step = w * pStep[idx(0)]; for (size_t j = 1; j < mu; ++j) step += w * pStep[idx(j)]; @@ -250,7 +256,7 @@ typename MatType::elem_type ActiveCMAES iterate.n_cols) { ps[idx1] = (1 - cs) * ps[idx0] + std::sqrt( - cs * (2 - cs) * muEffective) * - eigvec * diagmat(1 / eigval) * eigvec.t() * step; + cs * (2 - cs) * muEffective) * + eigvec * diagmat(1 / eigval) * eigvec.t() * step; } else { ps[idx1] = (1 - cs) * ps[idx0] + std::sqrt( - cs * (2 - cs) * muEffective) * step * + cs * (2 - cs) * muEffective) * step * eigvec * diagmat(1 / eigval) * eigvec.t(); } const ElemType psNorm = arma::norm(ps[idx1]); - sigma(idx1) = sigma(idx0) * std::exp(cs / ds * ( psNorm / enn - 1)); + sigma(idx1) = sigma(idx0) * std::exp(cs / ds * (psNorm / enn - 1)); + + if (std::isnan(sigma(idx1)) || sigma(idx1) > 1e14) + { + Warn << "The step size diverged to " << sigma(idx1) << "; " + << "terminating with failure. Try a smaller step size?" << std::endl; + + iterate = transformationPolicy.Transform(iterate); + Callback::EndOptimization(*this, function, iterate, callbacks...); + return overallObjective; + } pc[idx1] = (1 - cc) * pc[idx0] + std::sqrt(cc * (2 - cc) * muEffective) * step; @@ -293,15 +309,15 @@ typename MatType::elem_type ActiveCMAES patience) { + Info << "Active CMA-ES: minimized within tolerance " << tolerance << "; " << "terminating optimization." << std::endl; - iterate = transformationPolicy.Transform(iterate); - - Callback::EndOptimization(*this, function, iterate, callbacks...); - return overallObjective; + iterate = transformationPolicy.Transform(iterate); + Callback::EndOptimization(*this, function, iterate, callbacks...); + return overallObjective; + } } + else { + steps = 0; + } + + steps++; lastObjective = overallObjective; } iterate = transformationPolicy.Transform(iterate); - Callback::EndOptimization(*this, function, iterate, callbacks...); return overallObjective; } diff --git a/include/ensmallen_bits/cmaes/cmaes_impl.hpp b/include/ensmallen_bits/cmaes/cmaes_impl.hpp index 2009cfc2a..fa07972a7 100644 --- a/include/ensmallen_bits/cmaes/cmaes_impl.hpp +++ b/include/ensmallen_bits/cmaes/cmaes_impl.hpp @@ -205,6 +205,12 @@ typename MatType::elem_type CMAES iterate.n_cols) { pStep[idx(j)] = covLower * - arma::randn(iterate.n_rows, iterate.n_cols); + arma::randn(iterate.n_rows, iterate.n_cols); } else { pStep[idx(j)] = arma::randn(iterate.n_rows, iterate.n_cols) - * covLower.t(); + * covLower.t(); } pPosition[idx(j)] = mPosition[idx0] + sigma(idx0) * pStep[idx(j)]; // Calculate the objective function. pObjective(idx(j)) = selectionPolicy.Select(function, batchSize, - transformationPolicy.Transform(pPosition[idx(j)]), callbacks...); + transformationPolicy.Transform(pPosition[idx(j)]), callbacks...); } // Sort population. @@ -250,7 +256,7 @@ typename MatType::elem_type CMAES iterate.n_cols) { ps[idx1] = (1 - cs) * ps[idx0] + std::sqrt( - cs * (2 - cs) * muEffective) * - eigvec * diagmat(1 / eigval) * eigvec.t() * step; + cs * (2 - cs) * muEffective) * + eigvec * diagmat(1 / eigval) * eigvec.t() * step; } else { ps[idx1] = (1 - cs) * ps[idx0] + std::sqrt( - cs * (2 - cs) * muEffective) * step * + cs * (2 - cs) * muEffective) * step * eigvec * diagmat(1 / eigval) * eigvec.t(); } const ElemType psNorm = arma::norm(ps[idx1]); - sigma(idx1) = sigma(idx0) * std::exp(cs / ds * ( psNorm / enn - 1)); + sigma(idx1) = sigma(idx0) * std::exp(cs / ds * (psNorm / enn - 1)); + + if (std::isnan(sigma(idx1)) || sigma(idx1) > 1e14) + { + Warn << "The step size diverged to " << sigma(idx1) << "; " + << "terminating with failure. Try a smaller step size?" << std::endl; + + iterate = transformationPolicy.Transform(iterate); + + Callback::EndOptimization(*this, function, iterate, callbacks...); + return overallObjective; + } // Update covariance matrix. if ((psNorm / sqrt(1 - std::pow(1 - cs, 2 * i))) < h) @@ -304,12 +321,12 @@ typename MatType::elem_type CMAES iterate.n_cols) { C[idx1] = (1 - c1 - cmu) * C[idx0] + c1 * (pc[idx1] * - pc[idx1].t() + (cc * (2 - cc)) * C[idx0]); + pc[idx1].t() + (cc * (2 - cc)) * C[idx0]); } else { C[idx1] = (1 - c1 - cmu) * C[idx0] + c1 * - (pc[idx1].t() * pc[idx1] + (cc * (2 - cc)) * C[idx0]); + (pc[idx1].t() * pc[idx1] + (cc * (2 - cc)) * C[idx0]); } } @@ -318,7 +335,7 @@ typename MatType::elem_type CMAES patience) { + Info << "CMA-ES: minimized within tolerance " << tolerance << "; " << "terminating optimization." << std::endl; - iterate = transformationPolicy.Transform(iterate); - Callback::EndOptimization(*this, function, iterate, callbacks...); - return overallObjective; + iterate = transformationPolicy.Transform(iterate); + Callback::EndOptimization(*this, function, iterate, callbacks...); + return overallObjective; + } } + else { + steps = 0; + } + + steps++; lastObjective = overallObjective; } From caaa9263f417286263e359fcc39a8b4b37619a50 Mon Sep 17 00:00:00 2001 From: Suvarsha Chennareddy Date: Mon, 7 Aug 2023 22:17:30 +0530 Subject: [PATCH 24/34] move NotEmptyTransformation to its own file --- include/ensmallen_bits/cmaes/active_cmaes.hpp | 2 +- .../cmaes/active_cmaes_impl.hpp | 31 ++-------------- include/ensmallen_bits/cmaes/cmaes.hpp | 2 +- include/ensmallen_bits/cmaes/cmaes_impl.hpp | 27 +------------- .../cmaes/not_empty_transformation.hpp | 36 +++++++++++++++++++ 5 files changed, 42 insertions(+), 56 deletions(-) create mode 100644 include/ensmallen_bits/cmaes/not_empty_transformation.hpp diff --git a/include/ensmallen_bits/cmaes/active_cmaes.hpp b/include/ensmallen_bits/cmaes/active_cmaes.hpp index 0c29b2b9c..6641b3048 100644 --- a/include/ensmallen_bits/cmaes/active_cmaes.hpp +++ b/include/ensmallen_bits/cmaes/active_cmaes.hpp @@ -116,7 +116,7 @@ class ActiveCMAES const size_t maxIterations = 1000, const double tolerance = 1e-5, const SelectionPolicyType& selectionPolicy = SelectionPolicyType(), - double stepSize = 1); + double stepSize = 0); /** * Optimize the given function using Active CMA-ES. The given starting point will be diff --git a/include/ensmallen_bits/cmaes/active_cmaes_impl.hpp b/include/ensmallen_bits/cmaes/active_cmaes_impl.hpp index 2314cdab4..332e1ec0d 100644 --- a/include/ensmallen_bits/cmaes/active_cmaes_impl.hpp +++ b/include/ensmallen_bits/cmaes/active_cmaes_impl.hpp @@ -12,40 +12,15 @@ * the 3-clause BSD license along with ensmallen. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#ifndef ENSMALLEN_ACTIVE_CMAES_CMAES_IMPL_HPP -#define ENSMALLEN_ACTIVE_CMAES_CMAES_IMPL_HPP +#ifndef ENSMALLEN_CMAES_ACTIVE_CMAES_IMPL_HPP +#define ENSMALLEN_CMAES_ACTIVE_CMAES_IMPL_HPP // In case it hasn't been included yet. #include "active_cmaes.hpp" +#include "not_empty_transformation.hpp" #include -#ifndef NOT_EMPTY_TRANSFORMATION -#define NOT_EMPTY_TRANSFORMATION - -/* - This partial specialization is used to throw an exception when the - TransformationPolicyType is EmptyTransformation and call a - constructor with parameters 'lowerBound' and 'upperBound' otherwise. - This shall be removed when the deprecated constructor is removed in - the next major version of ensmallen. -*/ -template -struct NotEmptyTransformation : std::true_type { - void Assign(T1& obj, double lowerBound, double upperBound) { - obj = T1(lowerBound, upperBound); - } -}; - -template class T, typename... A, typename... B> -struct NotEmptyTransformation, T> : std::false_type { - void Assign(T& obj, double lowerBound, double upperBound) { - throw std::logic_error("TransformationPolicyType is EmptyTransformation"); - } -}; - -#endif - namespace ens { template diff --git a/include/ensmallen_bits/cmaes/cmaes.hpp b/include/ensmallen_bits/cmaes/cmaes.hpp index bf40fa083..eddf655e4 100644 --- a/include/ensmallen_bits/cmaes/cmaes.hpp +++ b/include/ensmallen_bits/cmaes/cmaes.hpp @@ -112,7 +112,7 @@ class CMAES const size_t maxIterations = 1000, const double tolerance = 1e-5, const SelectionPolicyType& selectionPolicy = SelectionPolicyType(), - double stepSize = 1); + double stepSize = 0); /** * Optimize the given function using CMA-ES. The given starting point will be diff --git a/include/ensmallen_bits/cmaes/cmaes_impl.hpp b/include/ensmallen_bits/cmaes/cmaes_impl.hpp index fa07972a7..4d875ee38 100644 --- a/include/ensmallen_bits/cmaes/cmaes_impl.hpp +++ b/include/ensmallen_bits/cmaes/cmaes_impl.hpp @@ -18,34 +18,9 @@ // In case it hasn't been included yet. #include "cmaes.hpp" +#include "not_empty_transformation.hpp" #include -#ifndef NOT_EMPTY_TRANSFORMATION -#define NOT_EMPTY_TRANSFORMATION - -/* - This partial specialization is used to throw an exception when the - TransformationPolicyType is EmptyTransformation and call a - constructor with parameters 'lowerBound' and 'upperBound' otherwise. - This shall be removed when the deprecated constructor is removed in - the next major version of ensmallen. -*/ -template -struct NotEmptyTransformation : std::true_type { - void Assign(T1& obj, double lowerBound, double upperBound) { - obj = T1(lowerBound, upperBound); - } -}; - -template class T, typename... A, typename... B> -struct NotEmptyTransformation, T> : std::false_type { - void Assign(T& obj, double lowerBound, double upperBound) { - throw std::logic_error("TransformationPolicyType is EmptyTransformation"); - } -}; - -#endif - namespace ens { template diff --git a/include/ensmallen_bits/cmaes/not_empty_transformation.hpp b/include/ensmallen_bits/cmaes/not_empty_transformation.hpp new file mode 100644 index 000000000..cdf18c10d --- /dev/null +++ b/include/ensmallen_bits/cmaes/not_empty_transformation.hpp @@ -0,0 +1,36 @@ +/** + * @file not_empty_transformation.hpp + * @author Suvarsha Chennareddy + * + * Check whether TransformationPolicyType is EmptyTransformation. + * + * ensmallen is free software; you may redistribute it and/or modify it under + * the terms of the 3-clause BSD license. You should have received a copy of + * the 3-clause BSD license along with ensmallen. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef NOT_EMPTY_TRANSFORMATION +#define NOT_EMPTY_TRANSFORMATION + + /* + This partial specialization is used to throw an exception when the + TransformationPolicyType is EmptyTransformation and call a + constructor with parameters 'lowerBound' and 'upperBound' otherwise. + This shall be removed when the deprecated constructor is removed in + the next major version of ensmallen. + */ +template +struct NotEmptyTransformation : std::true_type { + void Assign(T1& obj, double lowerBound, double upperBound) { + obj = T1(lowerBound, upperBound); + } +}; + +template class T, typename... A, typename... B> +struct NotEmptyTransformation, T> : std::false_type { + void Assign(T& obj, double lowerBound, double upperBound) { + throw std::logic_error("TransformationPolicyType is EmptyTransformation"); + } +}; + +#endif \ No newline at end of file From 5a145e8a4f2bb00dc98ca138b4c30bfe17baf2d1 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Wed, 16 Aug 2023 15:57:37 -0400 Subject: [PATCH 25/34] Fix style issues and keep the constructor. --- include/ensmallen_bits/cmaes/active_cmaes.hpp | 40 ++++++++++--------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/include/ensmallen_bits/cmaes/active_cmaes.hpp b/include/ensmallen_bits/cmaes/active_cmaes.hpp index 6641b3048..defdfc7aa 100644 --- a/include/ensmallen_bits/cmaes/active_cmaes.hpp +++ b/include/ensmallen_bits/cmaes/active_cmaes.hpp @@ -80,14 +80,15 @@ class ActiveCMAES * objective. * @param stepSize Starting sigma/step size (will be modified). */ - ActiveCMAES(const size_t lambda = 0, - const TransformationPolicyType& - transformationPolicy = TransformationPolicyType(), - const size_t batchSize = 32, - const size_t maxIterations = 1000, - const double tolerance = 1e-5, - const SelectionPolicyType& selectionPolicy = SelectionPolicyType(), - double stepSize = 0); + ActiveCMAES( + const size_t lambda = 0, + const TransformationPolicyType& + transformationPolicy = TransformationPolicyType(), + const size_t batchSize = 32, + const size_t maxIterations = 1000, + const double tolerance = 1e-5, + const SelectionPolicyType& selectionPolicy = SelectionPolicyType(), + double stepSize = 0); /** * Construct the Active CMA-ES optimizer with the given function and parameters @@ -96,8 +97,7 @@ class ActiveCMAES * tailored to the task at hand. The maximum number of iterations refers to * the maximum number of points that are processed (i.e., one iteration * equals one point; one iteration does not equal one pass over the dataset). - * This constructor is deprecated. - * + * * @param lambda The population size(0 use the default size). * @param lowerBound Lower bound of decision variables. * @param upperBound Upper bound of decision variables. @@ -109,14 +109,15 @@ class ActiveCMAES * objective. * @param stepSize Starting sigma/step size (will be modified). */ - ens_deprecated ActiveCMAES(const size_t lambda = 0, - const double lowerBound = -10, - const double upperBound = 10, - const size_t batchSize = 32, - const size_t maxIterations = 1000, - const double tolerance = 1e-5, - const SelectionPolicyType& selectionPolicy = SelectionPolicyType(), - double stepSize = 0); + ActiveCMAES( + const size_t lambda = 0, + const double lowerBound = -10, + const double upperBound = 10, + const size_t batchSize = 32, + const size_t maxIterations = 1000, + const double tolerance = 1e-5, + const SelectionPolicyType& selectionPolicy = SelectionPolicyType(), + double stepSize = 0); /** * Optimize the given function using Active CMA-ES. The given starting point will be @@ -134,7 +135,8 @@ class ActiveCMAES template - typename MatType::elem_type Optimize(SeparableFunctionType& function, + typename MatType::elem_type Optimize( + SeparableFunctionType& function, MatType& iterate, CallbackTypes&&... callbacks); From 4f6e31226fc08a7b181e9880cff5220d34dc0012 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Wed, 16 Aug 2023 15:58:42 -0400 Subject: [PATCH 26/34] Use the correct element type and style fixes. --- .../cmaes/active_cmaes_impl.hpp | 70 +++++++++---------- 1 file changed, 32 insertions(+), 38 deletions(-) diff --git a/include/ensmallen_bits/cmaes/active_cmaes_impl.hpp b/include/ensmallen_bits/cmaes/active_cmaes_impl.hpp index 332e1ec0d..8975038dd 100644 --- a/include/ensmallen_bits/cmaes/active_cmaes_impl.hpp +++ b/include/ensmallen_bits/cmaes/active_cmaes_impl.hpp @@ -65,7 +65,6 @@ ActiveCMAES::ActiveCMAES( d.Assign(transformationPolicy, lowerBound, upperBound); } - //! Optimize the function (minimize). template template mPosition(2, BaseMatType(iterate.n_rows, @@ -163,7 +160,7 @@ typename MatType::elem_type ActiveCMAES eigval; // TODO: might need a more general type. + arma::Col eigval; BaseMatType eigvec; BaseMatType eigvalZero(iterate.n_elem, 1); // eigvalZero is vector-shaped. eigvalZero.zeros(); @@ -216,8 +213,7 @@ typename MatType::elem_type ActiveCMAES 1e14) { Warn << "The step size diverged to " << sigma(idx1) << "; " - << "terminating with failure. Try a smaller step size?" << std::endl; + << "terminating with failure. Try a smaller step size?" << std::endl; iterate = transformationPolicy.Transform(iterate); @@ -273,44 +269,41 @@ typename MatType::elem_type ActiveCMAES iterate.n_cols) { C[idx1] = (1 - ccov) * C[idx0] + ccov * - (pc[idx1] * pc[idx1].t()); + (pc[idx1] * pc[idx1].t()); for (size_t j = 0; j < mu; ++j) { C[idx1] = C[idx1] + beta * w * - pStep[idx(j)] * pStep[idx(j)].t(); + pStep[idx(j)] * pStep[idx(j)].t(); } for (size_t j = lambda - mu; j < lambda; ++j) { C[idx1] = C[idx1] - beta * w * - pStep[idx(j)] * pStep[idx(j)].t(); + pStep[idx(j)] * pStep[idx(j)].t(); } - } else { C[idx1] = (1 - ccov) * C[idx0] + ccov * - (pc[idx1].t() * pc[idx1]); + (pc[idx1].t() * pc[idx1]); for (size_t j = 0; j < mu; ++j) { C[idx1] = C[idx1] + beta * w * - pStep[idx(j)].t() * pStep[idx(j)]; + pStep[idx(j)].t() * pStep[idx(j)]; } for (size_t j = lambda - mu; j < lambda; ++j) { C[idx1] = C[idx1] - beta * w * - pStep[idx(j)].t() * pStep[idx(j)]; + pStep[idx(j)].t() * pStep[idx(j)]; } - } arma::eig_sym(eigval, eigvec, C[idx1]); @@ -324,19 +317,19 @@ typename MatType::elem_type ActiveCMAES patience) { + if (steps > patience) + { Info << "Active CMA-ES: minimized within tolerance " << tolerance << "; " - << "terminating optimization." << std::endl; + << "terminating optimization." << std::endl; iterate = transformationPolicy.Transform(iterate); Callback::EndOptimization(*this, function, iterate, callbacks...); return overallObjective; } } - else { + else + { steps = 0; } steps++; - lastObjective = overallObjective; } From c92b48e8a8a02631e1f1ff224008b3a4d5eb426e Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Wed, 16 Aug 2023 16:00:31 -0400 Subject: [PATCH 27/34] Indentation fixes. --- include/ensmallen_bits/cmaes/cmaes.hpp | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/include/ensmallen_bits/cmaes/cmaes.hpp b/include/ensmallen_bits/cmaes/cmaes.hpp index eddf655e4..0ae323490 100644 --- a/include/ensmallen_bits/cmaes/cmaes.hpp +++ b/include/ensmallen_bits/cmaes/cmaes.hpp @@ -91,9 +91,8 @@ class CMAES * good for the given problem, so it is suggested that the values used be * tailored to the task at hand. The maximum number of iterations refers to * the maximum number of points that are processed (i.e., one iteration - * equals one point; one iteration does not equal one pass over the dataset). - * This constructor is deprecated. - * + * equals one point; one iteration does not equal one pass over the dataset). + * * @param lambda The population size(0 use the default size). * @param lowerBound Lower bound of decision variables. * @param upperBound Upper bound of decision variables. @@ -105,14 +104,14 @@ class CMAES * objective. * @param stepSize Starting sigma/step size (will be modified). */ - ens_deprecated CMAES(const size_t lambda = 0, - const double lowerBound = -10, - const double upperBound = 10, - const size_t batchSize = 32, - const size_t maxIterations = 1000, - const double tolerance = 1e-5, - const SelectionPolicyType& selectionPolicy = SelectionPolicyType(), - double stepSize = 0); + CMAES(const size_t lambda = 0, + const double lowerBound = -10, + const double upperBound = 10, + const size_t batchSize = 32, + const size_t maxIterations = 1000, + const double tolerance = 1e-5, + const SelectionPolicyType& selectionPolicy = SelectionPolicyType(), + double stepSize = 0); /** * Optimize the given function using CMA-ES. The given starting point will be From e4a843b9eeb271a930d6a11d3f19a5206312ce98 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Wed, 16 Aug 2023 16:01:49 -0400 Subject: [PATCH 28/34] Use the correct element type and style fixes. --- .../boundary_box_constraint.hpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/include/ensmallen_bits/cmaes/transformation_policies/boundary_box_constraint.hpp b/include/ensmallen_bits/cmaes/transformation_policies/boundary_box_constraint.hpp index fdc1d82e9..bc8f937ae 100644 --- a/include/ensmallen_bits/cmaes/transformation_policies/boundary_box_constraint.hpp +++ b/include/ensmallen_bits/cmaes/transformation_policies/boundary_box_constraint.hpp @@ -21,10 +21,10 @@ namespace ens { * by boundaries. * The implemented transformation transforms given coordinates into a region * bounded by the given lower and upper bounds (a box). First, the - * coordinates are shifted into a feasible preimage bounded by lowerBound - al + * coordinates are shifted into a feasible preimage bounded by lowerBound - al * and upperBound + au where al and au and calculated internally. - * These shifted coordinates are then transformed into coordinates bounded by - * lower_bound and upper_bound. It is an identity transformation in between + * These shifted coordinates are then transformed into coordinates bounded by + * lower_bound and upper_bound. It is an identity transformation in between * the lower and upper bounds. * * For more information, check the original implementation in C by N. Hansen: @@ -61,8 +61,8 @@ class BoundaryBoxConstraint * @param lowerBound The lower bound (for every dimension) of the coordinates. * @param upperBound The upper bound (for every dimension) of the coordinates. */ - BoundaryBoxConstraint(const double lowerBound, - const double upperBound) : + BoundaryBoxConstraint(const typename MatType::elem_type lowerBound, + const typename MatType::elem_type upperBound) : lowerBound({ (typename MatType::elem_type) lowerBound }), upperBound({ (typename MatType::elem_type) upperBound }) {} @@ -150,11 +150,11 @@ class BoundaryBoxConstraint MatType& UpperBound() { return upperBound; } private: - //! Upper bound of decision variables. - MatType upperBound; - //! Lower bound of decision variables. MatType lowerBound; + + //! Upper bound of decision variables. + MatType upperBound; }; } // namespace ens From a0a9400dfe0c13501df7147111675cbdfbd6b2b6 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Wed, 16 Aug 2023 16:03:06 -0400 Subject: [PATCH 29/34] Make the test more stable by testing multiple times before reporting a failure. --- tests/active_cmaes_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/active_cmaes_test.cpp b/tests/active_cmaes_test.cpp index 85301661a..93cffe6c7 100644 --- a/tests/active_cmaes_test.cpp +++ b/tests/active_cmaes_test.cpp @@ -78,7 +78,7 @@ TEST_CASE("ApproxActiveCMAESRosenbrockFunctionFMatTest", "[ActiveCMAESTest]") ApproxActiveCMAES> activecmaes(256, b, 16, 0, 1e-3); activecmaes.StepSize() = 0.075; - FunctionTest(activecmaes, 0.1, 0.1); + FunctionTest(activecmaes, 0.1, 0.1, 5); } /** @@ -93,4 +93,4 @@ TEST_CASE("ApproxActiveCMAESEmptyTransformationLogisticRegressionFMatTest", activecmaes(0, EmptyTransformation(), 16, 0, 1e-3); activecmaes.StepSize() = 0.55; LogisticRegressionFunctionTest(activecmaes, 0.01, 0.02, 5); -} \ No newline at end of file +} From 25ef5ff01a024b9df13d1ecf543fe78e8e3228ef Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Wed, 16 Aug 2023 16:04:56 -0400 Subject: [PATCH 30/34] Update HISTORY.md. --- HISTORY.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 958b7d55f..6ba087b76 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -2,9 +2,6 @@ ###### ????-??-?? * Implementation of Active CMAES ([#367](https://github.com/mlpack/ensmallen/pull/367)). - - * Fix the current CMAES inconsistencies - ([#351](https://github.com/mlpack/ensmallen/pull/351)). * LBFGS: avoid generation of NaNs, and add checks for finite values ([#368](https://github.com/mlpack/ensmallen/pull/368)). From 7bdf98224509c89188180edfeb9c8b1b6cf6bdad Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Thu, 17 Aug 2023 13:24:48 -0400 Subject: [PATCH 31/34] Increase number of steps to make the tests more stable. --- tests/active_cmaes_test.cpp | 48 +++++++++++++------------------------ 1 file changed, 17 insertions(+), 31 deletions(-) diff --git a/tests/active_cmaes_test.cpp b/tests/active_cmaes_test.cpp index 93cffe6c7..be1a6db16 100644 --- a/tests/active_cmaes_test.cpp +++ b/tests/active_cmaes_test.cpp @@ -15,31 +15,17 @@ using namespace ens; using namespace ens::test; -/** - * Run Active CMA-ES with the full selection policy on Rosenbrock function and - * make sure the results are acceptable. - * This test uses the deprecated constructor and therefore can be removed - * in a future version of ensmallen. - */ -TEST_CASE("ActiveCMAESDeprecatedConstructorRosenbrockFunctionTest", "[ActiveCMAESTest]") -{ - ActiveCMAES> - activecmaes(0, 0, 2, 16, 0, 1e-3); - activecmaes.StepSize() = 0.075; - FunctionTest(activecmaes, 0.1, 0.1); -} - /** * Run Active CMA-ES with the full selection policy on Rosenbrock function and * make sure the results are acceptable. */ TEST_CASE("ActiveCMAESRosenbrockFunctionTest", "[ActiveCMAESTest]") { - BoundaryBoxConstraint<> b(0, 2); + BoundaryBoxConstraint<> b(0, 10); ActiveCMAES> - activecmaes(0, b, 16, 0, 1e-3); - activecmaes.StepSize() = 0.075; - FunctionTest(activecmaes, 0.1, 0.1); + activecmaes(0, b, 1, 0, 1e-8); + activecmaes.StepSize() = 0.01; + FunctionTest(activecmaes, 0.1, 0.1, 10); } /** @@ -48,11 +34,11 @@ TEST_CASE("ActiveCMAESRosenbrockFunctionTest", "[ActiveCMAESTest]") */ TEST_CASE("ApproxActiveCMAESRosenbrockFunctionTest", "[ActiveCMAESTest]") { - BoundaryBoxConstraint<> b(0, 2); + BoundaryBoxConstraint<> b(0, 10); ApproxActiveCMAES> - activecmaes(256, b, 16, 0, 1e-3); - activecmaes.StepSize() = 0.075; - FunctionTest(activecmaes, 0.1, 0.1); + activecmaes(2048, b, 1, 0, 1e-13); + activecmaes.StepSize() = 0.01; + FunctionTest(activecmaes, 0.1, 0.1, 10); } /** @@ -61,11 +47,11 @@ TEST_CASE("ApproxActiveCMAESRosenbrockFunctionTest", "[ActiveCMAESTest]") */ TEST_CASE("ActiveCMAESRosenbrockFunctionFMatTest", "[ActiveCMAESTest]") { - BoundaryBoxConstraint b(0, 2); + BoundaryBoxConstraint b(0, 10); ActiveCMAES> - activecmaes(0, b, 16, 0, 1e-3); - activecmaes.StepSize() = 0.075; - FunctionTest(activecmaes, 0.1, 0.1); + activecmaes(0, b, 1, 0, 1e-8); + activecmaes.StepSize() = 0.01; + FunctionTest(activecmaes, 0.1, 0.1, 5); } /** @@ -74,11 +60,11 @@ TEST_CASE("ActiveCMAESRosenbrockFunctionFMatTest", "[ActiveCMAESTest]") */ TEST_CASE("ApproxActiveCMAESRosenbrockFunctionFMatTest", "[ActiveCMAESTest]") { - BoundaryBoxConstraint b(0, 2); + BoundaryBoxConstraint b(0, 10); ApproxActiveCMAES> - activecmaes(256, b, 16, 0, 1e-3); - activecmaes.StepSize() = 0.075; - FunctionTest(activecmaes, 0.1, 0.1, 5); + activecmaes(20248, b, 1, 0, 1e-5); + activecmaes.StepSize() = 0.01; + FunctionTest(activecmaes, 0.1, 0.1, 10); } /** @@ -90,7 +76,7 @@ TEST_CASE("ApproxActiveCMAESEmptyTransformationLogisticRegressionFMatTest", "[ActiveCMAESTest]") { ApproxActiveCMAES> - activecmaes(0, EmptyTransformation(), 16, 0, 1e-3); + activecmaes(0, EmptyTransformation(), 16, 0, 1e-3); activecmaes.StepSize() = 0.55; LogisticRegressionFunctionTest(activecmaes, 0.01, 0.02, 5); } From e8284423f10728e895b8347c05e698ecb344b481 Mon Sep 17 00:00:00 2001 From: Suvarsha Chennareddy Date: Fri, 18 Aug 2023 20:02:38 +0530 Subject: [PATCH 32/34] add documentation --- doc/optimizers.md | 93 +++++++++++++++++++ include/ensmallen_bits/cmaes/active_cmaes.hpp | 4 +- 2 files changed, 95 insertions(+), 2 deletions(-) diff --git a/doc/optimizers.md b/doc/optimizers.md index 8123039e5..2acb5a6ed 100644 --- a/doc/optimizers.md +++ b/doc/optimizers.md @@ -1,3 +1,95 @@ +## ActiveCMAES + +*An optimizer for [separable functions](#separable-functions).* + +Active CMA-ES is a variant of the stochastic search algorithm +CMA-ES - Covariance Matrix Adaptation Evolution Strategy. +Active CMA-ES actively reduces the uncertainty in unfavourable directions by +exploiting the information about bad mutations in the covariance matrix +update step. This isn't for the purpose of accelerating progress, but +instead for speeding up the adaptation of the covariance matrix (which, in +turn, will lead to faster progress). + +#### Constructors + + * `ActiveCMAES<`_`SelectionPolicyType, TransformationPolicyType`_`>()` + * `ActiveCMAES<`_`SelectionPolicyType, TransformationPolicyType`_`>(`_`lambda, transformationPolicy`_`)` + * `ActiveCMAES<`_`SelectionPolicyType, TransformationPolicyType`_`>(`_`lambda, transformationPolicy, batchSize`_`)` + * `ActiveCMAES<`_`SelectionPolicyType, TransformationPolicyType`_`>(`_`lambda, transformationPolicy, batchSize, maxIterations, tolerance, selectionPolicy`_`)` + * `ActiveCMAES<`_`SelectionPolicyType, TransformationPolicyType`_`>(`_`lambda, transformationPolicy, batchSize, maxIterations, tolerance, selectionPolicy, stepSize`_`)` + +The _`SelectionPolicyType`_ template parameter refers to the strategy used to +compute the (approximate) objective function. The `FullSelection` and +`RandomSelection` classes are available for use; custom behavior can be achieved +by implementing a class with the same method signatures. +The _`TransformationPolicyType`_ template parameter refers to transformation +strategy used to map decision variables to the desired domain during fitness +evaluation and optimization termination. The `EmptyTransformation` and +`BoundaryBoxConstraint` classes are available for use; custom behavior can be +achieved by implementing a class with the same method signatures. + +For convenience the following types can be used: + + * **`ActiveCMAES<>`** (equivalent to `ActiveCMAES>`): uses all separable functions to compute objective + * **`ApproxActiveCMAES<>`** (equivalent to `ActiveCMAES>`): uses a small amount of separable functions to compute approximate objective + +#### Attributes + +| **type** | **name** | **description** | **default** | +|----------|----------|-----------------|-------------| +| `size_t` | **`lambda`** | The population size (0 uses a default size). | `0` | +| `TransformationPolicyType` | **`transformationPolicy`** | Instantiated transformation policy used to map the coordinates to the desired domain. | `TransformationPolicyType()` | +| `size_t` | **`batchSize`** | Batch size to use for the objective calculation. | `32` | +| `size_t` | **`maxIterations`** | Maximum number of iterations. | `1000` | +| `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` | +| `SelectionPolicyType` | **`selectionPolicy`** | Instantiated selection policy used to calculate the objective. | `SelectionPolicyType()` | +| `size_t` | **`stepSize`** | Initial step size | `0` | + +Attributes of the optimizer may also be changed via the member methods +`Lambda()`, `TransformationPolicy()`, `BatchSize()`, `MaxIterations()`, +`Tolerance()`, and `SelectionPolicy()`. + +The `selectionPolicy` attribute allows an instantiated `SelectionPolicyType` to +be given. The `FullSelection` policy has no need to be instantiated and thus +the option is not relevant when the `ActiveCMAES<>` optimizer type is being used; the +`RandomSelection` policy has the constructor `RandomSelection(`_`fraction`_`)` +where _`fraction`_ specifies the percentage of separable functions to use to +estimate the objective function. +The `transformationPolicy` attribute allows an instantiated +`TransformationPolicyType` to be given. The `EmptyTransformation<`_`MatType`_`>` +has no need to be instantiated. `BoundaryBoxConstraint<`_`MatType`_`>` policy has +the constructor `BoundaryBoxConstraint(`_`lowerBound, upperBound`_`)` +where _`lowerBound`_ and _`lowerBound`_ are the lower bound and upper bound of +the coordinates respectively. + +#### Examples: + +
+Click to collapse/expand example code. + + +```c++ +RosenbrockFunction f; +arma::mat coordinates = f.GetInitialPoint(); + +// ActiveCMAES with the FullSelection and BoundaryBoxConstraint policies. +BoundaryBoxConstraint b(-1, 1); +ActiveCMAES optimizer(0, b, 32, 200, 1e-4); +optimizer.Optimize(f, coordinates); + +// ActiveCMAES with the RandomSelection and BoundaryBoxConstraint policies. +ApproxActiveCMAES> cmaes(0, b, 32, 200, 1e-4); +approxOptimizer.Optimize(f, coordinates); +``` + +
+ +#### See also: + + * [CMAES](#cmaes) + * [Improving Evolution Strategies through Active Covariance Matrix Adaptation](https://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.114.4239&rep=rep1&type=pdfn) + * [Evolution strategy in Wikipedia](https://en.wikipedia.org/wiki/Evolution_strategy) + ## AdaBelief *An optimizer for [differentiable separable functions](#differentiable-separable-functions).* @@ -727,6 +819,7 @@ For convenience the following types can be used: | `size_t` | **`maxIterations`** | Maximum number of iterations. | `1000` | | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` | | `SelectionPolicyType` | **`selectionPolicy`** | Instantiated selection policy used to calculate the objective. | `SelectionPolicyType()` | +| `size_t` | **`stepSize`** | Initial step size | `0` | Attributes of the optimizer may also be changed via the member methods `Lambda()`, `TransformationPolicy()`, `BatchSize()`, `MaxIterations()`, diff --git a/include/ensmallen_bits/cmaes/active_cmaes.hpp b/include/ensmallen_bits/cmaes/active_cmaes.hpp index defdfc7aa..1edec07c2 100644 --- a/include/ensmallen_bits/cmaes/active_cmaes.hpp +++ b/include/ensmallen_bits/cmaes/active_cmaes.hpp @@ -12,8 +12,8 @@ * the 3-clause BSD license along with ensmallen. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#ifndef ENSMALLEN_ACTIVE_CMAES_CMAES_HPP -#define ENSMALLEN_ACTIVE_CMAES_CMAES_HPP +#ifndef ENSMALLEN_CMAES_ACTIVE_CMAES_HPP +#define ENSMALLEN_CMAES_ACTIVE_CMAES_HPP #include "full_selection.hpp" #include "random_selection.hpp" From 9f71c1cb16d460b0a9cc39139d114fe4e957c472 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Mon, 21 Aug 2023 10:40:46 -0400 Subject: [PATCH 33/34] No need to depreciate the CMAES constructor. --- tests/cmaes_test.cpp | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/tests/cmaes_test.cpp b/tests/cmaes_test.cpp index 0e3e5df9b..75a9fe6b8 100644 --- a/tests/cmaes_test.cpp +++ b/tests/cmaes_test.cpp @@ -17,18 +17,6 @@ using namespace ens; using namespace ens::test; -/** - * Run CMA-ES with the full selection policy on logistic regression and - * make sure the results are acceptable. - * This test uses the deprecated constructor and therefore can be removed - * in a future version of ensmallen. - */ -TEST_CASE("CMAESDeprecatedConstructorLogisticRegressionTest", "[CMAESTest]") -{ - CMAES> cmaes(0, -10, 10, 32, 500, 1e-3); - LogisticRegressionFunctionTest(cmaes, 0.003, 0.006, 5); -} - /** * Run CMA-ES with the full selection policy on logistic regression and * make sure the results are acceptable. @@ -86,4 +74,4 @@ TEST_CASE("ApproxCMAESEmptyTransformationLogisticRegressionFMatTest", ApproxCMAES> cmaes(0, EmptyTransformation(), 16, 500, 1e-3); LogisticRegressionFunctionTest(cmaes, 0.01, 0.02, 5); -} \ No newline at end of file +} From 43c0cb38bd96be8003688ce3cbf415fbfeac97be Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Mon, 21 Aug 2023 13:38:23 -0400 Subject: [PATCH 34/34] Use the correct population size. --- tests/active_cmaes_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/active_cmaes_test.cpp b/tests/active_cmaes_test.cpp index be1a6db16..24c2fd1e3 100644 --- a/tests/active_cmaes_test.cpp +++ b/tests/active_cmaes_test.cpp @@ -62,7 +62,7 @@ TEST_CASE("ApproxActiveCMAESRosenbrockFunctionFMatTest", "[ActiveCMAESTest]") { BoundaryBoxConstraint b(0, 10); ApproxActiveCMAES> - activecmaes(20248, b, 1, 0, 1e-5); + activecmaes(2048, b, 1, 0, 1e-5); activecmaes.StepSize() = 0.01; FunctionTest(activecmaes, 0.1, 0.1, 10); }