Merge pull request #1078 from jennloe/gmresPrec

Add (right) preconditioning to GMRES.

example/gmres/CMakeLists.txt

@@ -16,3 +16,8 @@ KOKKOSKERNELS_ADD_EXECUTABLE(
   SOURCES test_cmplx_A.cpp
   )
 
+KOKKOSKERNELS_ADD_EXECUTABLE_AND_TEST(
+  gmres_test_prec
+  SOURCES test_prec.cpp
+  )
+

example/gmres/KokkosSparse_MatrixPrec.hpp

@@ -0,0 +1,138 @@
+/*
+//@HEADER
+// ************************************************************************
+//
+//                        Kokkos v. 3.0
+//       Copyright (2020) National Technology & Engineering
+//               Solutions of Sandia, LLC (NTESS).
+//
+// Under the terms of Contract DE-NA0003525 with NTESS,
+// the U.S. Government retains certain rights in this software.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+//
+// 3. Neither the name of the Corporation nor the names of the
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Questions? Contact Jennifer Loe (jloe@sandia.gov)
+//
+// ************************************************************************
+//@HEADER
+*/
+/// @file KokkosKernels_MatrixPrec.hpp
+
+#ifndef KK_MATRIX_PREC_HPP
+#define KK_MATRIX_PREC_HPP
+
+#include<KokkosSparse_Preconditioner.hpp>
+#include<Kokkos_Core.hpp>
+#include<KokkosBlas.hpp>
+#include<KokkosSparse_spmv.hpp>
+
+namespace KokkosSparse{
+
+namespace Experimental{
+
+/// \class MatrixPrec
+/// \brief  This is a simple class to use if one 
+///         already has a matrix representation of their 
+///         preconditioner M.  The class applies an
+///         SpMV with M as the preconditioning step. 
+/// \tparam ScalarType Type of the matrix's entries
+/// \tparam Layout Kokkos layout of vectors X and Y to which 
+///         the preconditioner is applied
+/// \tparam EXSP Execution space for the preconditioner apply
+/// \tparam Ordinal Type of the matrix's indices; 
+///
+/// Preconditioner provides the following methods
+///   - initialize() Does nothing; Matrix initialized upon object construction. 
+///   - isInitialized() returns true 
+///   - compute() Does nothing; Matrix initialized upon object construction.
+///   - isComputed() returns true 
+///
+template< class ScalarType, class Layout, class EXSP, class OrdinalType = int > 
+class MatrixPrec : virtual public KokkosSparse::Experimental::Preconditioner<ScalarType, Layout, EXSP, OrdinalType>
+{ 
+private:
+    using crsMat_t = KokkosSparse::CrsMatrix<ScalarType, OrdinalType, EXSP>;
+    crsMat_t A;
+
+    bool isInitialized_ = true;
+    bool isComputed_ = true;
+
+public:
+  //! Constructor:
+    MatrixPrec<ScalarType, Layout, EXSP, OrdinalType> (const KokkosSparse::CrsMatrix<ScalarType, OrdinalType, EXSP> &mat) 
+     : A(mat) {}
+
+  //! Destructor.
+  virtual ~MatrixPrec(){}
+
+  ///// \brief Apply the preconditioner to X, putting the result in Y.
+  /////
+  ///// \tparam XViewType Input vector, as a 1-D Kokkos::View
+  ///// \tparam YViewType Output vector, as a nonconst 1-D Kokkos::View
+  /////
+  ///// \param transM [in] "N" for non-transpose, "T" for transpose, "C"
+  /////   for conjugate transpose.  All characters after the first are
+  /////   ignored.  This works just like the BLAS routines.
+  ///// \param alpha [in] Input coefficient of M*x
+  ///// \param beta [in] Input coefficient of Y
+  /////
+  ///// If the result of applying this preconditioner to a vector X is
+  ///// \f$M \cdot X\f$, then this method computes \f$Y = \beta Y + \alpha M \cdot X\f$.
+  ///// The typical case is \f$\beta = 0\f$ and \f$\alpha = 1\f$.
+  //
+  void apply (const Kokkos::View<ScalarType*, Layout, EXSP> &X, 
+      Kokkos::View<ScalarType*, Layout, EXSP> &Y, 
+      const char transM[] = "N",
+      ScalarType alpha = Kokkos::Details::ArithTraits<ScalarType>::one(),
+      ScalarType beta = Kokkos::Details::ArithTraits<ScalarType>::zero()) const 
+  {
+    KokkosSparse::spmv(transM, alpha, A, X, beta, Y);
+  };
+  //@}
+
+  //! Set this preconditioner's parameters.
+  void setParameters () {}
+
+  void initialize() { }
+
+  //! True if the preconditioner has been successfully initialized, else false.
+  bool isInitialized() const { return isInitialized_;}
+
+  void compute(){ }
+
+  //! True if the preconditioner has been successfully computed, else false.
+  bool isComputed() const {return isComputed_;}
+
+  //! True if the preconditioner implements a transpose operator apply. 
+  bool hasTransposeApply() const { return true; } 
+
+};
+} //End Experimental
+} //End namespace KokkosSparse
+
+#endif

example/gmres/KokkosSparse_Preconditioner.hpp

@@ -0,0 +1,155 @@
+/*
+//@HEADER
+// ************************************************************************
+//
+//                        Kokkos v. 3.0
+//       Copyright (2020) National Technology & Engineering
+//               Solutions of Sandia, LLC (NTESS).
+//
+// Under the terms of Contract DE-NA0003525 with NTESS,
+// the U.S. Government retains certain rights in this software.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+//
+// 3. Neither the name of the Corporation nor the names of the
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Questions? Contact Jennifer Loe (jloe@sandia.gov)
+//
+// ************************************************************************
+//@HEADER
+*/
+/// @file KokkosKernels_Preconditioner.hpp
+//
+#ifndef KK_PREC_HPP
+#define KK_PREC_HPP
+
+#include<Kokkos_Core.hpp>
+#include<KokkosKernels_Controls.hpp>
+#include<Kokkos_ArithTraits.hpp>
+
+namespace KokkosSparse{
+
+namespace Experimental{
+
+/// \class Preconditioner
+/// \brief Interface for KokkosKernels preconditioners
+/// \tparam ScalarType Type of the matrix's entries
+/// \tparam Layout Kokkos layout of vectors X and Y to which 
+///         the preconditioner is applied
+/// \tparam EXSP Execution space for the preconditioner apply
+/// \tparam Ordinal Type of the matrix's indices; 
+///
+/// Preconditioner provides the following methods
+///   - initialize() performs all operations based on the graph of the
+///     matrix (without considering the numerical values)
+///   - isInitialized() returns true if the preconditioner has been
+///     successfully initialized
+///   - compute() computes everything required to apply the
+///     preconditioner, using the matrix's values (and assuming that the
+///     graph structure of the matrix has not changed)
+///   - isComputed() returns true if the preconditioner has been
+///     successfully computed, false otherwise.
+///
+/// Implementations of compute() must internally call initialize() if
+/// isInitialized() returns false. The preconditioner is applied by
+/// apply(). 
+/// Every time that initialize() is called, the object destroys all the previously
+/// allocated information, and reinitializes the preconditioner. Every
+/// time compute() is called, the object recomputes the actual values of
+/// the preconditioner.
+template< class ScalarType, class Layout, class EXSP, class OrdinalType = int > 
+class Preconditioner{ 
+
+public:
+  //! Constructor:
+  Preconditioner(){}
+
+  //! Destructor.
+  virtual ~Preconditioner(){}
+
+  ///// \brief Apply the preconditioner to X, putting the result in Y.
+  /////
+  ///// \tparam XViewType Input vector, as a 1-D Kokkos::View
+  ///// \tparam YViewType Output vector, as a nonconst 1-D Kokkos::View
+  /////
+  ///// \param transM [in] "N" for non-transpose, "T" for transpose, "C"
+  /////   for conjugate transpose.  All characters after the first are
+  /////   ignored.  This works just like the BLAS routines.
+  ///// \param alpha [in] Input coefficient of M*x
+  ///// \param beta [in] Input coefficient of Y
+  /////
+  ///// If the result of applying this preconditioner to a vector X is
+  ///// \f$M \cdot X\f$, then this method computes \f$Y = \beta Y + \alpha M \cdot X\f$.
+  ///// The typical case is \f$\beta = 0\f$ and \f$\alpha = 1\f$.
+  //
+  virtual void
+  apply (const Kokkos::View<ScalarType*, Layout, EXSP> &X, 
+         Kokkos::View<ScalarType*, Layout, EXSP> &Y, 
+         const char transM[] = "N",
+         ScalarType alpha = Kokkos::Details::ArithTraits<ScalarType>::one(),
+         ScalarType beta = Kokkos::Details::ArithTraits<ScalarType>::zero()) const = 0;
+  //@}
+
+  //! Set this preconditioner's parameters.
+  virtual void setParameters () = 0;
+
+  /// @brief Set up the graph structure of this preconditioner.
+  ///
+  /// If the graph structure of the constructor's input matrix has
+  /// changed, or if you have not yet called initialize(), you must
+  /// call initialize() before you may call compute() or apply().
+  ///
+  /// Thus, initialize() corresponds to the "symbolic factorization"
+  /// step of a sparse factorization, whether or not the specific
+  /// preconditioner actually does a sparse factorization.
+  virtual void initialize() = 0;
+
+  //! True if the preconditioner has been successfully initialized, else false.
+  virtual bool isInitialized() const = 0;
+
+  /// @brief Set up the numerical values in this preconditioner.
+  ///
+  /// If the values of the constructor's input matrix have changed, or
+  /// if you have not yet called compute(), you must call compute()
+  /// before you may call apply().
+  ///
+  /// Thus, compute() corresponds to the "numeric factorization"
+  /// step of a sparse factorization, whether or not the specific
+  /// preconditioner actually does a sparse factorization.
+  virtual void compute() = 0;
+
+  //! True if the preconditioner has been successfully computed, else false.
+  virtual bool isComputed() const = 0;
+
+  //! True if the preconditioner implements a transpose operator apply. 
+  virtual bool hasTransposeApply() const { return false; } 
+
+};
+
+} // End Experimental
+} //End namespace KokkosSparse
+
+#endif

example/gmres/README.md

@@ -27,6 +27,7 @@ The gmres function takes the following input paramters:
 **B:** A Kokkos::View that is the system right-hand side. Must have B.extent(1)=1. (Currently only one right-hand side is supported.)   
 **X:** A Kokkos::View that is used as both the initial vector for the GMRES iteration and the output for the solution vector.  (Must have X.extent(1)=1.)   
 **opts:** A 'GmresOpts' struct as described below.   
+**M:** A pointer to a KokkosSparse::Experimental::Preconditioner. Only right preconditioning is supported at this time.  
 
 The solver has a 'GmresOpts' struct to pass in solver options.  Available options are:   
 **tol:** The convergence tolerance for GMRES.  Based upon the relative residual. The solver will terminate when norm(b-Ax)/norm(b) <= tol. (Default: 1e-8)   
@@ -54,12 +55,12 @@ The real-valued test uses a matrix generated directly by Kokkos Kernels.
 These measurements were taken on 7/23/21, running on an NVIDIA V100 GPU on Weaver7.  
 (Timings based upon the GMRES::TotalTime profiling region.)
 
-**ex\_real\_A:** Converges in 2270 iterations and 0.9629 seconds.
+**ex\_real\_A:** Converges in 2271 iterations and 0.9629 seconds.
 
 (The two following timings total the time for the CGS2 and MGS tests.)   
-**test\_real\_A:** Converges in 29 iterations (with a restart size of 15) and 0.2536 seconds.
+**test\_real\_A:** Converges in 30 iterations (with a restart size of 15) and 0.2536 seconds.
 
-**test\_cmplx\_A:** Converges in 651 iterations (to a tolerance of 1e-5) in 2.822 seconds.  
+**test\_cmplx\_A:** Converges in 652 iterations (to a tolerance of 1e-5) in 2.822 seconds.  
 
 ### Concerns, enhancements, or bug reporting:
 If you wish to suggest an enhancement or make a bug report for this solver code, please post an issue at https://github.com/kokkos/kokkos-kernels/issues or email jloe@sandia.gov.

example/gmres/READMEPreconditioners.md

@@ -0,0 +1,51 @@
+## KokkosSparse Preconditioner Interface:
+
+The `KokkosSparse_Preconditioner` class provides an abstract base class interface to use Kokkos-based preconditioners with iterative linear solvers. In particular, this class is designed to work with the Kokkos-based GMRES implementation in `examples/gmres`. It may also be useful for integrating Kokkos-based preconditioners into other solver codes and packages. (For Trilinos users: This class is loosely based upon the IfPack2::Preconditioner class.)  The Preconditioner class and derived classes sit in the `KokkosSparse::Experimental` namespace. 
+
+### Input parameters:
+
+##### Preconditioner template paramters:
+The KokkosSparse Preconditioner has the following template parameters:   
+**ScalarType:** Type of scalars in the preconditioner apply. (double, float, half, Kokkos::complex<double>, etc.)   
+**Layout:** Layout of vectors X to which the preconditioner will be applied. (Kokkos::LayoutLeft, or other Kokkos layouts.)   
+**EXSP:** A Kokkos execution space.   
+**OrdinalType:** The ordinal type from the Kokkos::CrsMatrix A.   
+**Note:** If using this preconditioner with the Kokkos-based GMRES example, these template parameters must match those used to initialize the GMRES solver.  
+
+### Preconditioner Base Class Functions (See code for full details.):
+
+**constructor**: Empty in the base class.   
+**apply**`( Kokkos::View<ScalarType*> &X, Kokkos::View<ScalarType*> &Y, const char transM[] = "N", ScalarType alpha = 1.0, ScalarType beta =  0):`
+Returns `y = beta * y + alpha * M * x` where `M` is the preconditioning operator.  (May apply `M` transposed if `transM = 'T' or 'C'` for 'transpose' and 'conjugate transpose' respectively.)   
+**setParameters():** Used to set preconditioner parameters.     
+**initialize():** Set up the graph structure of this preconditioner. If the graph structure of the constructor's input matrix has changed, or if you have not yet called `initialize()`, you must call `initialize()` before you may call `compute()` or `apply()`. Thus, `initialize()` corresponds to the "symbolic factorization" step of a sparse factorization, whether or not the specific preconditioner actually does a sparse factorization.   
+**compute():** Set up the numerical values in this preconditioner. If the values of the constructor's input matrix have changed, or if you have not yet called `compute()`, you must call `compute()` before you may call `apply()`. Thus, `compute()` corresponds to the "numeric factorization" step of a sparse factorization, whether or not the specific preconditioner actually does a sparse factorization.    
+**isInitialized()** and **isComputed()** return whether the preconditioner has been initialized or computed, respectively.  
+**hasTransposeApply()** Returns true if the transposed (or conjugate transposed) operator apply is available for this preconditioner.  Base class function returns `false`.
+
+### Derived Preconditioner Classes:
+
+#### Matrix Prec:
+A simple class that takes a `KokkosSparse::CrsMatrix` to apply as the preconditioner `M`.  This matrix is given to the preconditioner constructor function.  There are no parameters to set.  The functions `initialize` and `compute` do not perform any operations.   
+
+### Testing
+
+##### Command-Line parameters for test\_prec:
+Current solver parameters for the GMRES preconditioning test are as follows:
+
+"**--mat-size**   :   The size of the n x n test matrix. (Default: n=1000.)   
+"**--max-subsp**   :  The maximum size of the Kyrlov subspace before restarting (Default 50)."   
+"**--max-restarts:**  Maximum number of GMRES restarts (Default 50)."   
+"**--tol        :**  Convergence tolerance.  (Default 1e-10)."   
+"**--ortho       :**  Type of orthogonalization. Use 'CGS2' or 'MGS'. (Default 'CGS2')"   
+"**--rand\_rhs**    :  Generate a random right-hand side b.  (Without this option, the solver default generates b = vector of ones.)"   
+"**--help**         : Display a help message."
+
+#### Test Measurements:
+**test\_prec:** Tests the GMRES solver with a diagonal matrix and uses its inverse as the preconditioner.  Should always converge in 1 iteration, regardless of problem size.
+
+### Concerns, enhancements, or bug reporting:
+If you wish to suggest an enhancement or make a bug report for this preconditioner code, please post an issue at https://github.com/kokkos/kokkos-kernels/issues or email jloe@sandia.gov.
+
+SAND2021-9744 O
+