Zoltan2 : add Metis interface to ordering

packages/ifpack2/src/Ifpack2_Details_Filu_def.hpp

@@ -100,13 +100,12 @@ template<typename Scalar, typename LocalOrdinal, typename GlobalOrdinal, typenam
 void Filu<Scalar, LocalOrdinal, GlobalOrdinal, Node>::
 initLocalPrec()
 {
-  typedef Tpetra::CrsMatrix<Scalar, LocalOrdinal, GlobalOrdinal, Node> TCrsMatrix;
   auto nRows = this->mat_->getLocalNumRows();
   auto& p = this->params_;
   auto matCrs = Ifpack2::Details::getCrsMatrix(this->mat_);
 
   bool skipSortMatrix = !matCrs.is_null() && matCrs->getCrsGraph()->isSorted() &&
-                       !p.use_metis;
+                        !p.use_metis;
   localPrec_ = Teuchos::rcp(new LocalFILU(skipSortMatrix, this->localRowPtrs_, this->localColInds_, this->localValues_, nRows, p.sptrsv_algo,
                                           p.nFact, p.nTrisol, p.level, p.omega, p.shift, p.guessFlag ? 1 : 0, p.blockSizeILU, p.blockSize));
   #ifdef HAVE_IFPACK2_METIS

packages/zoltan2/core/src/algorithms/order/Zoltan2_AlgMetis.hpp

@@ -0,0 +1,177 @@
+// @HEADER
+//
+// ***********************************************************************
+//
+//   Zoltan2: A package of combinatorial algorithms for scientific computing
+//                  Copyright 2012 Sandia Corporation
+//
+// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
+// the U.S. Government retains certain rights in this software.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+//
+// 3. Neither the name of the Corporation nor the names of the
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Questions? Contact Karen Devine      (kddevin@sandia.gov)
+//                    Erik Boman        (egboman@sandia.gov)
+//                    Siva Rajamanickam (srajama@sandia.gov)
+//
+// ***********************************************************************
+//
+// @HEADER
+
+/*! \file Zoltan2_AlgMetis.hpp
+    \brief The ND ordering algorithm uses Metis.
+*/
+
+#ifndef _ZOLTAN2_ALGMETIS_HPP_
+#define _ZOLTAN2_ALGMETIS_HPP_
+
+#include <Zoltan2_Algorithm.hpp>
+#include <Zoltan2_GraphModel.hpp>
+#include <Zoltan2_OrderingSolution.hpp>
+#include <Zoltan2_TPLTraits.hpp>
+#ifdef HAVE_ZOLTAN2_METIS
+#include "metis.h"
+#endif
+
+namespace Zoltan2{
+
+template <typename Adapter>
+class AlgMetis : public Algorithm<Adapter>
+{
+    private:
+
+    const RCP<GraphModel<Adapter> > model;
+    const RCP<Teuchos::ParameterList> pl;
+    const RCP<const Teuchos::Comm<int> > comm;
+
+    public:
+
+    AlgMetis(
+      const RCP<GraphModel<Adapter> > &model__,
+      const RCP<Teuchos::ParameterList> &pl__,
+      const RCP<const Teuchos::Comm<int> > &comm__
+    ) : model(model__), pl(pl__), comm(comm__)
+    { }
+
+    int globalOrder(
+      const RCP<GlobalOrderingSolution<typename Adapter::gno_t> > &/* solution */) {
+        throw std::logic_error("AlgMetis does not yet support global ordering.");
+    }
+
+    int localOrder(
+      const RCP<LocalOrderingSolution<typename Adapter::lno_t> > &solution)
+    {
+#ifndef HAVE_ZOLTAN2_METIS
+      (void)solution; // remove unused parameter warning
+  throw std::runtime_error(
+        "BUILD ERROR: Metis requested but not compiled into Zoltan2.\n"
+        "Please set CMake flag Zoltan2_ENABLE_METIS:BOOL=ON.");
+#else
+      typedef typename Adapter::gno_t gno_t;
+      typedef typename Adapter::lno_t lno_t;
+      typedef typename Adapter::offset_t offset_t;
+      typedef typename Adapter::scalar_t scalar_t;
+
+      int ierr= 0;
+
+      // Get EdgeList
+      const size_t nVtx = model->getLocalNumVertices();
+      const size_t nNnz = model->getLocalNumEdges();
+      lno_t *perm = (lno_t *) (solution->getPermutationRCP().getRawPtr());
+
+      if (nVtx > 0 && nNnz > 0) {
+        ArrayView<const gno_t> edgeIds;
+        ArrayView<const offset_t> offsets;
+        ArrayView<StridedData<lno_t, scalar_t> > wgts; // wgts are ignored in NodeND
+        model->getEdgeList(edgeIds, offsets, wgts);
+
+        // Prepare for calling metis
+        using Zoltan2OffsetView = typename Kokkos::View<offset_t*, Kokkos::HostSpace>;
+        using Zoltan2EdgeView = typename Kokkos::View<gno_t*, Kokkos::HostSpace>;
+        Zoltan2OffsetView zoltan2_rowptr (const_cast<offset_t*>(offsets.data()), nVtx+1);
+        Zoltan2EdgeView   zoltan2_colidx (const_cast<gno_t*>(edgeIds.data()), nNnz);
+
+        using MetisIdxView = typename Kokkos::View<idx_t*, Kokkos::HostSpace>;
+        MetisIdxView metis_rowptr;
+        MetisIdxView metis_colidx;
+
+        // Symmetrize (always for now)
+        KokkosKernels::Impl::symmetrize_graph_symbolic_hashmap<
+          Zoltan2OffsetView, Zoltan2EdgeView, MetisIdxView, MetisIdxView, Kokkos::HostSpace::execution_space>
+          (nVtx, zoltan2_rowptr, zoltan2_colidx, metis_rowptr, metis_colidx);
+
+        // Remove diagonals
+        idx_t metis_nVtx=0;
+        TPL_Traits<idx_t, size_t>::ASSIGN(metis_nVtx, nVtx);
+
+        idx_t nnz = metis_rowptr(0);
+        idx_t old_nnz = nnz;
+        for (idx_t i = 0; i < metis_nVtx; i++) {
+          for (idx_t k = old_nnz; k < metis_rowptr(i+1); k++) {
+            if (metis_colidx(k) != i) {
+              metis_colidx(nnz) = metis_colidx(k);
+              nnz++;
+            }
+          }
+          old_nnz = metis_rowptr(i+1);
+          metis_rowptr(i+1) = nnz;
+        }
+
+        // Allocate Metis perm/iperm
+        idx_t *metis_perm = new idx_t[nVtx];
+        idx_t *metis_iperm = new idx_t[nVtx];
+
+        // Call metis
+        int info = METIS_NodeND(&metis_nVtx, metis_rowptr.data(), metis_colidx.data(),
+                                NULL, NULL, metis_perm, metis_iperm);
+        if (METIS_OK != info) {
+          throw std::runtime_error(std::string("METIS_NodeND returned info = " + info));
+        }
+
+        // Copy result back
+        for (size_t i = 0; i < nVtx; i++)
+          TPL_Traits<lno_t, idx_t>::ASSIGN(perm[i], metis_perm[i]);
+
+        delete [] metis_iperm;
+        delete [] metis_perm;
+      } else {
+        for (size_t i = 0; i < nVtx; i++)
+          perm[i] = i;
+      }
+
+      solution->setHavePerm(true);
+      return ierr;
+#endif
+    }
+};
+
+}
+
+
+
+#endif

packages/zoltan2/core/src/algorithms/order/Zoltan2_OrderingAlgorithms.hpp

@@ -51,6 +51,7 @@
 #include "Zoltan2_AlgRCM.hpp"
 #include "Zoltan2_AlgSortedDegree.hpp"
 #include "Zoltan2_AlgAMD.hpp"
+#include "Zoltan2_AlgMetis.hpp"
 #include <Zoltan2_AlgScotch.hpp> // BDD
 
 #ifdef INCLUDE_ZOLTAN2_EXPERIMENTAL

packages/zoltan2/core/src/problems/Zoltan2_OrderingProblem.hpp

@@ -145,7 +145,7 @@ class OrderingProblem : public Problem<Adapter>
 
     RCP<Teuchos::StringValidator> order_method_Validator =
       Teuchos::rcp( new Teuchos::StringValidator(
-        Teuchos::tuple<std::string>( "rcm", "minimum_degree", "natural",
+        Teuchos::tuple<std::string>( "rcm", "metis", "minimum_degree", "natural",
           "random", "sorted_degree", "scotch", "nd" )));
     pl.set("order_method", "rcm", "order algorithm",
       order_method_Validator);
@@ -298,6 +298,10 @@ void OrderingProblem<Adapter>::solve(bool /* updateInputData */)
       this->comm_);
     ZOLTAN2_COMPUTE_ORDERING
   }
+  if (method.compare("metis") == 0) {
+    AlgMetis<base_adapter_t> alg(this->graphModel_, this->params_, this->comm_);
+    ZOLTAN2_COMPUTE_ORDERING
+  }
   else if (method.compare("minimum_degree") == 0) {
     std::string pkg = this->params_->template get<std::string>(
       "order_package", "amd");
@@ -356,6 +360,7 @@ void OrderingProblem<Adapter>::createOrderingProblem()
 
   if ((method == std::string("rcm")) || 
       (method == std::string("sorted_degree")) || 
+      (method == std::string("metis")) || 
       (method == std::string("minimum_degree"))) {
     modelType = GraphModelType;
   }

packages/zoltan2/test/core/order/CMakeLists.txt

@@ -91,6 +91,19 @@ IF (${PACKAGE_NAME}_ENABLE_Scotch)
   )
 ENDIF()
 
+IF (${PACKAGE_NAME}_ENABLE_METIS)
+  TRIBITS_ADD_EXECUTABLE_AND_TEST(
+    OrderingMetis
+    SOURCES orderingMetis.cpp
+    NUM_MPI_PROCS 4
+    COMM serial mpi
+    ARGS
+     "--inputFile=simple"
+    PASS_REGULAR_EXPRESSION "PASS"
+    FAIL_REGULAR_EXPRESSION "FAIL"
+  )
+ENDIF()
+
 ##########################################
 IF (${PACKAGE_NAME}_ENABLE_Experimental)