An attempt to fix the current CMAES inconsistencies

HISTORY.md

@@ -1,9 +1,12 @@
 ### ensmallen ?.??.?: "???"
 ###### ????-??-??
-* LBFGS: avoid generation of NaNs, and add checks for finite values
+ * Fix the current CMAES inconsistencies
+   ([#351](https://github.com/mlpack/ensmallen/pull/351)).
+
+ * LBFGS: avoid generation of NaNs, and add checks for finite values
    ([#368](https://github.com/mlpack/ensmallen/pull/368)).
 
-* Fix CNE test tolerances
+ * Fix CNE test tolerances
    ([#360](https://github.com/mlpack/ensmallen/pull/360)).
 
 ### ensmallen 2.19.1: "Eight Ball Deluxe"

doc/optimizers.md

@@ -696,35 +696,40 @@ matrix within an iterative procedure using the covariance matrix.
 
 #### Constructors
 
- * `CMAES<`_`SelectionPolicyType`_`>()`
- * `CMAES<`_`SelectionPolicyType`_`>(`_`lambda, lowerBound, upperBound`_`)`
- * `CMAES<`_`SelectionPolicyType`_`>(`_`lambda, lowerBound, upperBound, batchSize`_`)`
- * `CMAES<`_`SelectionPolicyType`_`>(`_`lambda, lowerBound, upperBound, batchSize, maxIterations, tolerance, selectionPolicy`_`)`
+ * `CMAES<`_`SelectionPolicyType, TransformationPolicyType`_`>()`
+ * `CMAES<`_`SelectionPolicyType, TransformationPolicyType`_`>(`_`lambda, transformationPolicy`_`)`
+ * `CMAES<`_`SelectionPolicyType, TransformationPolicyType`_`>(`_`lambda, transformationPolicy, batchSize`_`)`
+ * `CMAES<`_`SelectionPolicyType, TransformationPolicyType`_`>(`_`lambda, transformationPolicy, batchSize, maxIterations, tolerance, selectionPolicy`_`)`
+ * `CMAES<`_`SelectionPolicyType, TransformationPolicyType`_`>(`_`lambda, transformationPolicy, batchSize, maxIterations, tolerance, selectionPolicy, stepSize`_`)`
 
 The _`SelectionPolicyType`_ template parameter refers to the strategy used to
 compute the (approximate) objective function.  The `FullSelection` and
 `RandomSelection` classes are available for use; custom behavior can be achieved
 by implementing a class with the same method signatures.
+The _`TransformationPolicyType`_  template parameter refers to transformation 
+strategy used to map decision variables to the desired domain during fitness 
+evaluation and optimization termination. The `EmptyTransformation` and 
+`BoundaryBoxConstraint` classes are available for use; custom behavior can be 
+achieved by implementing a class with the same method signatures.
 
 For convenience the following types can be used:
 
- * **`CMAES<>`** (equivalent to `CMAES<FullSelection>`): uses all separable functions to compute objective
- * **`ApproxCMAES`** (equivalent to `CMAES<RandomSelection>`): uses a small amount of separable functions to compute approximate objective
+ * **`CMAES<>`** (equivalent to `CMAES<FullSelection, EmptyTransformation<>>`): uses all separable functions to compute objective
+ * **`ApproxCMAES<>`** (equivalent to `CMAES<RandomSelection, EmptyTransformation<>>`): uses a small amount of separable functions to compute approximate objective
 
 #### Attributes
 
 | **type** | **name** | **description** | **default** |
 |----------|----------|-----------------|-------------|
 | `size_t` | **`lambda`** | The population size (0 uses a default size). | `0` |
-| `double` | **`lowerBound`** | Lower bound of decision variables. | `-10.0` |
-| `double` | **`upperBound`** | Upper bound of decision variables. | `10.0` |
+| `TransformationPolicyType` | **`transformationPolicy`** | Instantiated transformation policy used to map the coordinates to the desired domain. | `TransformationPolicyType()` |
 | `size_t` | **`batchSize`** | Batch size to use for the objective calculation. | `32` |
 | `size_t` | **`maxIterations`** | Maximum number of iterations. | `1000` |
 | `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
 | `SelectionPolicyType` | **`selectionPolicy`** | Instantiated selection policy used to calculate the objective. | `SelectionPolicyType()` |
 
 Attributes of the optimizer may also be changed via the member methods
-`Lambda()`, `LowerBound()`, `UpperBound()`, `BatchSize()`, `MaxIterations()`,
+`Lambda()`, `TransformationPolicy()`, `BatchSize()`, `MaxIterations()`,
 `Tolerance()`, and `SelectionPolicy()`.
 
 The `selectionPolicy` attribute allows an instantiated `SelectionPolicyType` to
@@ -733,6 +738,12 @@ the option is not relevant when the `CMAES<>` optimizer type is being used; the
 `RandomSelection` policy has the constructor `RandomSelection(`_`fraction`_`)`
 where _`fraction`_ specifies the percentage of separable functions to use to
 estimate the objective function.
+The `transformationPolicy` attribute allows an instantiated 
+`TransformationPolicyType` to be given. The `EmptyTransformation<`_`MatType`_`>` 
+has no need to be instantiated. `BoundaryBoxConstraint<`_`MatType`_`>` policy has
+the constructor `BoundaryBoxConstraint(`_`lowerBound, upperBound`_`)`
+where  _`lowerBound`_ and _`lowerBound`_ are the lower bound and upper bound of 
+the coordinates respectively.
 
 #### Examples:
 
@@ -744,12 +755,13 @@ estimate the objective function.
 RosenbrockFunction f;
 arma::mat coordinates = f.GetInitialPoint();
 
-// CMAES with the FullSelection policy.
-CMAES<> optimizer(0, -1, 1, 32, 200, 1e-4);
+// CMAES with the FullSelection and BoundaryBoxConstraint policies.
+BoundaryBoxConstraint b(-1, 1);
+CMAES optimizer(0, b, 32, 200, 1e-4);
 optimizer.Optimize(f, coordinates);
 
-// CMAES with the RandomSelection policy.
-ApproxCMAES<> approxOptimizer(0, -1, 1. 32, 200, 1e-4);
+// CMAES with the RandomSelection and BoundaryBoxConstraint policies.
+ApproxCMAES<BoundaryBoxConstraint<>> cmaes(0, b, 32, 200, 1e-4);
 approxOptimizer.Optimize(f, coordinates);
 ```
 

include/ensmallen_bits/cmaes/cmaes.hpp

@@ -17,6 +17,8 @@
 
 #include "full_selection.hpp"
 #include "random_selection.hpp"
+#include "transformation_policies/empty_transformation.hpp"
+#include "transformation_policies/boundary_box_constraint.hpp"
 
 namespace ens {
 
@@ -46,8 +48,12 @@ namespace ens {
  * ensmallen website.
  *
  * @tparam SelectionPolicy The selection strategy used for the evaluation step.
+ * @tparam TransformationPolicy The transformation strategy used to 
+ *       map decision variables to the desired domain during fitness evaluation
+ *       and termination. Use EmptyTransformation if the domain isn't bounded.
  */
-template<typename SelectionPolicyType = FullSelection>
+template<typename SelectionPolicyType = FullSelection,
+         typename TransformationPolicyType = EmptyTransformation<>>
 class CMAES
 {
  public:
@@ -60,22 +66,53 @@ class CMAES
    * equal one pass over the dataset).
    *
    * @param lambda The population size (0 use the default size).
-   * @param lowerBound Lower bound of decision variables.
-   * @param upperBound Upper bound of decision variables.
+   * @param transformationPolicy Instantiated transformation policy used to 
+   *     map the coordinates to the desired domain.
    * @param batchSize Batch size to use for the objective calculation.
    * @param maxIterations Maximum number of iterations allowed (0 means no
    *     limit).
    * @param tolerance Maximum absolute tolerance to terminate algorithm.
    * @param selectionPolicy Instantiated selection policy used to calculate the
    *     objective.
+   * @param stepSize Starting sigma/step size (will be modified).
    */
   CMAES(const size_t lambda = 0,
-        const double lowerBound = -10,
-        const double upperBound = 10,
+        const TransformationPolicyType& 
+              transformationPolicy = TransformationPolicyType(),
         const size_t batchSize = 32,
         const size_t maxIterations = 1000,
         const double tolerance = 1e-5,
-        const SelectionPolicyType& selectionPolicy = SelectionPolicyType());
+        const SelectionPolicyType& selectionPolicy = SelectionPolicyType(),
+        double stepSize = 0);
+
+  /**
+   * Construct the CMA-ES optimizer with the given function and parameters 
+   * (including lower and upper bounds). The defaults here are not necessarily 
+   * good for the given problem, so it is suggested that the values used be 
+   * tailored to the task at hand.  The maximum number of iterations refers to 
+   * the maximum number of points that are processed (i.e., one iteration 
+   * equals one point; one iteration does not equal one pass over the dataset). 
+   * This constructor is deprecated.
+   * 
+   * @param lambda The population size(0 use the default size).
+   * @param lowerBound Lower bound of decision variables.
+   * @param upperBound Upper bound of decision variables.
+   * @param batchSize Batch size to use for the objective calculation.
+   * @param maxIterations Maximum number of iterations allowed(0 means no
+      limit).
+   * @param tolerance Maximum absolute tolerance to terminate algorithm.
+   * @param selectionPolicy Instantiated selection policy used to calculate the
+   * objective.
+   * @param stepSize Starting sigma/step size (will be modified).
+   */
+  ens_deprecated CMAES(const size_t lambda = 0,
+                       const double lowerBound = -10,
+                       const double upperBound = 10,
+                       const size_t batchSize = 32,
+                       const size_t maxIterations = 1000,
+                       const double tolerance = 1e-5,
+                       const SelectionPolicyType& selectionPolicy = SelectionPolicyType(),
+                       double stepSize = 1);
 
   /**
    * Optimize the given function using CMA-ES. The given starting point will be
@@ -91,27 +128,17 @@ class CMAES
    * @return Objective value of the final point.
    */
   template<typename SeparableFunctionType,
-           typename MatType,
-           typename... CallbackTypes>
-  typename MatType::elem_type Optimize(SeparableFunctionType& function,
-                                       MatType& iterate,
-                                       CallbackTypes&&... callbacks);
+      typename MatType,
+      typename... CallbackTypes>
+      typename MatType::elem_type Optimize(SeparableFunctionType& function,
+          MatType& iterate,
+          CallbackTypes&&... callbacks);
 
   //! Get the population size.
   size_t PopulationSize() const { return lambda; }
   //! Modify the population size.
   size_t& PopulationSize() { return lambda; }
 
-  //! Get the lower bound of decision variables.
-  double LowerBound() const { return lowerBound; }
-  //! Modify the lower bound of decision variables.
-  double& LowerBound() { return lowerBound; }
-
-  //! Get the upper bound of decision variables
-  double UpperBound() const { return upperBound; }
-  //! Modify the upper bound of decision variables
-  double& UpperBound() { return upperBound; }
-
   //! Get the batch size.
   size_t BatchSize() const { return batchSize; }
   //! Modify the batch size.
@@ -132,16 +159,24 @@ class CMAES
   //! Modify the selection policy.
   SelectionPolicyType& SelectionPolicy() { return selectionPolicy; }
 
+  //! Get the transformation policy.
+  const TransformationPolicyType& TransformationPolicy() const
+  { return transformationPolicy; }
+  //! Modify the transformation policy.
+  TransformationPolicyType& TransformationPolicy() 
+  { return transformationPolicy; }
+
+  //! Get the step size.
+  double StepSize() const
+  { return stepSize; }
+  //! Modify the step size.
+  double& StepSize()
+  { return stepSize; }
+
  private:
   //! Population size.
   size_t lambda;
 
-  //! Lower bound of decision variables.
-  double lowerBound;
-
-  //! Upper bound of decision variables
-  double upperBound;
-
   //! The batch size for processing.
   size_t batchSize;
 
@@ -153,13 +188,22 @@ class CMAES
 
   //! The selection policy used to calculate the objective.
   SelectionPolicyType selectionPolicy;
+
+  //! The transformationPolicy used to map coordinates to the suitable domain
+  //! while evaluating fitness. This mapping is also done after optimization 
+  //! has completed.
+  TransformationPolicyType transformationPolicy;
+
+  //! The step size.
+  double stepSize;
 };
 
 /**
  * Convenient typedef for CMAES approximation.
  */
-template<typename SelectionPolicyType = RandomSelection>
-using ApproxCMAES = CMAES<SelectionPolicyType>;
+template<typename TransformationPolicyType = EmptyTransformation<>,
+         typename SelectionPolicyType = RandomSelection>
+using ApproxCMAES = CMAES<SelectionPolicyType, TransformationPolicyType>;
 
 } // namespace ens