Merge Pull Request #7566 from roeverf/Trilinos/mlgdsw

Automatically Merged using Trilinos Pull Request AutoTester PR Title: ShyLU/FROSch: Additional Level for GDSW preconditioner PR Author: roeverf
trilinos · Aug 10, 2020 · 98c9efe · 98c9efe
2 parents 52b9c81 + 50bda62
commit 98c9efe
Show file tree

Hide file tree

Showing 37 changed files with 1,618 additions and 76 deletions.
diff --git a/packages/shylu/shylu_dd/frosch/TODO b/packages/shylu/shylu_dd/frosch/TODO
@@ -37,3 +37,8 @@
     - Reorganization of the DDInterface
     - Do we need the dimension for the DDInterface?
     - InterfaceEntity -> Entity?
+
+24.07.2020
+    - Move Repeated Map build for the coarse level to the specific partitionOfUnity
+    - Seperate parameterList for CoarseSolver
+    - Seperate build map functions
diff --git a/packages/shylu/shylu_dd/frosch/src/Adapters/Thyra_FROSchFactory_decl.hpp b/packages/shylu/shylu_dd/frosch/src/Adapters/Thyra_FROSchFactory_decl.hpp
@@ -137,6 +137,8 @@ namespace Thyra {
         using ConstXMultiVector             = const XMultiVector;
         using XMultiVectorPtr               = RCP<XMultiVector>;
         using ConstXMultiVectorPtr          = RCP<ConstXMultiVector>;
+        using XMultiVectorPtrVecPtr         = ArrayRCP<XMultiVectorPtr>;
+        using ConstXMultiVectorPtrVecPtr    = ArrayRCP<ConstXMultiVectorPtr>;
 
         using ParameterListPtr              = RCP<ParameterList>;
         using ConstParameterListPtr         = RCP<const ParameterList>;

diff --git a/packages/shylu/shylu_dd/frosch/src/Adapters/Thyra_FROSchFactory_def.hpp b/packages/shylu/shylu_dd/frosch/src/Adapters/Thyra_FROSchFactory_def.hpp
@@ -248,22 +248,34 @@ namespace Thyra {
                 SchwarzPreconditioner = TLP;
             } else if (!paramList_->get("FROSch Preconditioner Type","TwoLevelPreconditioner").compare("TwoLevelBlockPreconditioner")) {
                 ConstXMapPtrVecPtr repeatedMaps = null;
+                ConstXMultiVectorPtrVecPtr coordinatesList = null;
                 UNVecPtr dofsPerNodeVector;
                 DofOrderingVecPtr dofOrderings;
 
                 FROSCH_ASSERT(paramList_->isParameter("DofsPerNode Vector"),"Currently, TwoLevelBlockPreconditioner cannot be constructed without DofsPerNode Vector.");
                 FROSCH_ASSERT(paramList_->isParameter("DofOrdering Vector"),"Currently, TwoLevelBlockPreconditioner cannot be constructed without DofOrdering Vector.");
+                // Extract the repeated map vector
                 if (paramList_->isParameter("Repeated Map Vector")) {
                     XMapPtrVecPtr repeatedMapsTmp = ExtractVectorFromParameterList<XMapPtr>(*paramList_,"Repeated Map Vector");
+                    XMultiVectorPtrVecPtr nodeListVecTmp = ExtractVectorFromParameterList<XMultiVectorPtr>(*paramList_,"Coordinates List Vector");
                     if (!repeatedMapsTmp.is_null()) {
                         repeatedMaps.resize(repeatedMapsTmp.size());
                         for (unsigned i=0; i<repeatedMaps.size(); i++) {
                             repeatedMaps[i] = repeatedMapsTmp[i].getConst();
                         }
                     }
-                    FROSCH_ASSERT(!repeatedMaps.is_null(),"FROSch::FROSchFactory : ERROR: repeatedMaps.is_null()");
+                    // Extract the nodeList map vector
+                    if(!nodeListVecTmp.is_null()){
+                      coordinatesList.resize(nodeListVecTmp.size());
+                      for(unsigned i = 0; i<coordinatesList.size();i++){
+                        coordinatesList[i] = nodeListVecTmp[i].getConst();
+                      }
+                    }
 
+                    FROSCH_ASSERT(!repeatedMaps.is_null(),"FROSch::FROSchFactory : ERROR: repeatedMaps.is_null()");
+                    // Extract the DofsPerNode  vector
                     dofsPerNodeVector = ExtractVectorFromParameterList<UN>(*paramList_,"DofsPerNode Vector");
+                    // Extract the DofOrdering vector
                     dofOrderings = ExtractVectorFromParameterList<DofOrdering>(*paramList_,"DofOrdering Vector");
                 } else {
                     FROSCH_ASSERT(false,"Currently, TwoLevelBlockPreconditioner cannot be constructed without Repeated Maps.");
@@ -278,6 +290,7 @@ namespace Thyra {
                                  dofsPerNodeVector,
                                  dofOrderings,
                                  paramList_->get("Overlap",1),
+                                 coordinatesList,
                                  repeatedMaps);
 
                 SchwarzPreconditioner = TLBP;

diff --git a/packages/shylu/shylu_dd/frosch/src/CoarseSpaces/FROSch_CoarseSpace_decl.hpp b/packages/shylu/shylu_dd/frosch/src/CoarseSpaces/FROSch_CoarseSpace_decl.hpp
@@ -105,6 +105,7 @@ namespace FROSch {
 
         int assembleCoarseSpace();
 
+
         int buildGlobalBasisMatrix(ConstXMapPtr rowMap,
                                    ConstXMapPtr rangeMap,
                                    ConstXMapPtr repeatedMap,
@@ -148,7 +149,6 @@ namespace FROSch {
 
         ConstXMapPtr AssembledBasisMap_;
         ConstXMapPtr AssembledBasisMapUnique_;
-
         XMultiVectorPtr AssembledBasis_;
 
         UNVec LocalSubspacesSizes_ = UNVec(0);

diff --git a/packages/shylu/shylu_dd/frosch/src/CoarseSpaces/FROSch_CoarseSpace_def.hpp b/packages/shylu/shylu_dd/frosch/src/CoarseSpaces/FROSch_CoarseSpace_def.hpp
@@ -86,6 +86,8 @@ namespace FROSch {
     template <class SC,class LO,class GO,class NO>
     int CoarseSpace<SC,LO,GO,NO>::assembleCoarseSpace()
     {
+
+
         FROSCH_ASSERT(UnassembledBasesMaps_.size()>0,"FROSch::CoarseSpace : ERROR: UnassembledBasesMaps_.size()==0");
         FROSCH_ASSERT(UnassembledBasesMapsUnique_.size()>0,"FROSch::CoarseSpace : ERROR: UnassembledBasesMapsUnique_.size()==0");
         FROSCH_ASSERT(UnassembledSubspaceBases_.size()>0,"FROSch::CoarseSpace : ERROR: UnassembledSubspaceBases_.size()==0");

diff --git a/packages/shylu/shylu_dd/frosch/src/CoarseSpaces/FROSch_ConstantPartitionOfUnity_def.hpp b/packages/shylu/shylu_dd/frosch/src/CoarseSpaces/FROSch_ConstantPartitionOfUnity_def.hpp
@@ -39,6 +39,7 @@
 // ************************************************************************
 //@HEADER
 
+
 #ifndef _FROSCH_CONSTANTPARTITIONOFUNITY_DEF_HPP
 #define _FROSCH_CONSTANTPARTITIONOFUNITY_DEF_HPP
 
@@ -66,7 +67,7 @@ namespace FROSch {
     DDInterface_ (ddInterface)
     {
         FROSCH_TIMER_START_LEVELID(constantPartitionOfUnityTime,"ConstantPartitionOfUnity::ConstantPartitionOfUnity");
-
+      
         if (!this->ParameterList_->get("Type","Full").compare("Full")) {
             UseVolumes_ = true;
         } else if (!this->ParameterList_->get("Type","Full").compare("Volumes")) {
@@ -76,7 +77,8 @@ namespace FROSch {
         } else {
             FROSCH_ASSERT(false,"FROSch::ConstantPartitionOfUnity : ERROR: Specify a valid Type.");
         }
-
+        this->MpiComm_->barrier();this->MpiComm_->barrier();this->MpiComm_->barrier();
+        if(this->Verbose_)std::cout<<"CPart1\n";
         CommunicationStrategy communicationStrategy = CreateOneToOneMap;
         if (!this->ParameterList_->get("Interface Communication Strategy","CreateOneToOneMap").compare("CrsMatrix")) {
             communicationStrategy = CommCrsMatrix;
@@ -123,7 +125,9 @@ namespace FROSch {
     {
         FROSCH_TIMER_START_LEVELID(computePartitionOfUnityTime,"ConstantPartitionOfUnity::computePartitionOfUnity");
         // Interface
+
         UN dofsPerNode = DDInterface_->getInterior()->getEntity(0)->getDofsPerNode();
+
         UN numInteriorDofs = dofsPerNode*DDInterface_->getInterior()->getEntity(0)->getNumNodes();
 
         if (UseVolumes_) Volumes_->buildEntityMap(DDInterface_->getNodesMap());

diff --git a/packages/shylu/shylu_dd/frosch/src/CoarseSpaces/FROSch_GDSWInterfacePartitionOfUnity_def.hpp b/packages/shylu/shylu_dd/frosch/src/CoarseSpaces/FROSch_GDSWInterfacePartitionOfUnity_def.hpp
@@ -144,6 +144,7 @@ namespace FROSch {
     int GDSWInterfacePartitionOfUnity<SC,LO,GO,NO>::sortInterface(ConstXMatrixPtr matrix,
                                                                   ConstXMultiVectorPtr nodeList)
     {
+
         FROSCH_TIMER_START_LEVELID(sortInterfaceTime,"GDSWInterfacePartitionOfUnity::sortInterface");
         if (this->ParameterList_->get("Test Unconnected Interface",true)) {
             if (matrix.is_null()) {
@@ -162,7 +163,6 @@ namespace FROSch {
         // Interface
         UN dofsPerNode = this->DDInterface_->getInterface()->getEntity(0)->getDofsPerNode();
         UN numInterfaceDofs = dofsPerNode*this->DDInterface_->getInterface()->getEntity(0)->getNumNodes();
-
         this->DDInterface_->buildEntityMaps(UseVertices_,
                                             UseShortEdges_,
                                             UseStraightEdges_,
@@ -171,6 +171,7 @@ namespace FROSch {
                                             false,
                                             false);
 
+
         // Maps
         if (UseVertices_) {
             Vertices_ = this->DDInterface_->getVertices();

diff --git a/packages/shylu/shylu_dd/frosch/src/CoarseSpaces/FROSch_LocalPartitionOfUnityBasis_decl.hpp b/packages/shylu/shylu_dd/frosch/src/CoarseSpaces/FROSch_LocalPartitionOfUnityBasis_decl.hpp
@@ -45,6 +45,7 @@
 #include <Xpetra_Operator.hpp>
 #include <Xpetra_MapFactory_fwd.hpp>
 
+#include <Teuchos_ScalarTraits.hpp>
 #include <FROSch_CoarseSpace_def.hpp>
 
 #include "FROSch_Tools_def.hpp"
@@ -54,7 +55,7 @@ namespace FROSch {
 
     using namespace Teuchos;
     using namespace Xpetra;
-    
+
     template <class SC = double,
               class LO = int,
               class GO = DefaultGlobalOrdinal,

diff --git a/packages/shylu/shylu_dd/frosch/src/CoarseSpaces/FROSch_PartitionOfUnity_decl.hpp b/packages/shylu/shylu_dd/frosch/src/CoarseSpaces/FROSch_PartitionOfUnity_decl.hpp
@@ -63,6 +63,7 @@ namespace FROSch {
         using XMap                          = Map<LO,GO,NO>;
         using XMapPtr                       = RCP<XMap>;
         using ConstXMapPtr                  = RCP<const XMap>;
+        using XMapPtrVec                    = Array<XMapPtr>;
         using XMapPtrVecPtr                 = ArrayRCP<XMapPtr>;
         using ConstXMapPtrVecPtr            = ArrayRCP<ConstXMapPtr>;
 
@@ -88,11 +89,11 @@ namespace FROSch {
 
         using UN                            = unsigned;
         using ConstUN                       = const UN;
-        
+
         using LOVec                         = Array<LO>;
         using LOVecPtr                      = ArrayRCP<LO>;
         using LOVecPtr2D                    = ArrayRCP<LOVecPtr>;
-        
+
         using GOVec                         = Array<GO>;
         using GOVecView                     = ArrayView<GO>;
 
@@ -118,11 +119,11 @@ namespace FROSch {
         virtual int computePartitionOfUnity(ConstXMultiVectorPtr nodeList = null) = 0;
 
         int assembledPartitionOfUnityMaps();
-        
+
         XMultiVectorPtrVecPtr getLocalPartitionOfUnity() const;
 
         XMapPtrVecPtr getPartitionOfUnityMaps() const;
-        
+
         XMapPtr getAssembledPartitionOfUnityMap() const;
 
     protected:
@@ -135,11 +136,11 @@ namespace FROSch {
         XMultiVectorPtrVecPtr LocalPartitionOfUnity_;
 
         XMapPtrVecPtr PartitionOfUnityMaps_;
-        
+
         XMapPtr AssmbledPartitionOfUnityMap_;
 
         bool Verbose_ = false;
-        
+
         Verbosity Verbosity_ = All;
 
         const UN LevelID_;

diff --git a/packages/shylu/shylu_dd/frosch/src/CoarseSpaces/FROSch_PartitionOfUnity_def.hpp b/packages/shylu/shylu_dd/frosch/src/CoarseSpaces/FROSch_PartitionOfUnity_def.hpp
@@ -66,7 +66,7 @@ namespace FROSch {
     Verbosity_ (verbosity),
     LevelID_ (levelID)
     {
-        
+
     }
 
     template <class SC,class LO,class GO,class NO>
@@ -82,7 +82,7 @@ namespace FROSch {
             FROSCH_NOTIFICATION("FROSch::PartitionOfUnity",Verbosity_,"AssmbledPartitionOfUnityMap_ has already been assembled previously.");
         }
         LOVecPtr2D partMappings;
-        AssmbledPartitionOfUnityMap_ = AssembleMaps(PartitionOfUnityMaps_(),partMappings);
+        AssmbledPartitionOfUnityMap_ = AssembleMapsNonConst(PartitionOfUnityMaps_(),partMappings);
         return 0;
     }
 

diff --git a/packages/shylu/shylu_dd/frosch/src/InterfaceSets/FROSch_DDInterface_decl.hpp b/packages/shylu/shylu_dd/frosch/src/InterfaceSets/FROSch_DDInterface_decl.hpp
@@ -55,6 +55,7 @@
 #include <FROSch_ExtractSubmatrices_def.hpp>
 
 
+
 namespace FROSch {
 
     using namespace Teuchos;
@@ -195,11 +196,12 @@ namespace FROSch {
         EntitySetConstPtr & getInterior() const;
 
         EntitySetConstPtr & getRoots() const;
-        
+
         EntitySetConstPtr & getLeafs() const;
 
         EntitySetPtrConstVecPtr & getEntitySetVector() const;
 
+        GOVec getNumEnt()const;
         //! This function returns those entities which are to be used to build a connectivity graph on the subdomain
         //! level. They have to identified first using the function identifyConnectivityEntities().
         EntitySetConstPtr & getConnectivityEntities() const;
@@ -243,6 +245,7 @@ namespace FROSch {
         Verbosity Verbosity_ = All;
 
         ConstUN LevelID_ = 1;
+        GOVec NumEntity_;
     };
 
 }

diff --git a/packages/shylu/shylu_dd/frosch/src/InterfaceSets/FROSch_DDInterface_def.hpp b/packages/shylu/shylu_dd/frosch/src/InterfaceSets/FROSch_DDInterface_def.hpp
@@ -170,7 +170,6 @@ namespace FROSch {
         }
         XMapPtr map = MapFactory<LO,GO,NO>::Build(matrix->getRowMap()->lib(),-1,indicesGammaDofs(),0,MpiComm_);
         matrix = FROSch::ExtractLocalSubdomainMatrix(matrix.getConst(),map.getConst(),ScalarTraits<SC>::one());
-
         // Operate on hierarchy
         for (UN i=0; i<EntitySetVector_.size(); i++) {
             EntitySetVector_[i]->divideUnconnectedEntities(matrix,MpiComm_->getRank());
@@ -190,7 +189,6 @@ namespace FROSch {
         */
 
         removeEmptyEntities();
-
         // We need to set the unique ID; otherwise, we cannot sort entities
         for (UN i=0; i<EntitySetVector_.size(); i++) {
             EntitySetVector_[i]->setUniqueIDToFirstGlobalNodeID();
@@ -232,6 +230,8 @@ namespace FROSch {
         FROSCH_TIMER_START_LEVELID(sortVerticesEdgesFacesTime,"DDInterface::sortVerticesEdgesFaces");
         //if (Verbose_ && Verbosity_==All) cout << "FROSch::DDInterface : Sorting interface components" << endl;
 
+
+
         // Clear EntitySets if non-empty
         if (Vertices_->getNumEntities()>0) Vertices_.reset(new EntitySet<SC,LO,GO,NO>(VertexType));
         if (ShortEdges_->getNumEntities()>0) ShortEdges_.reset(new EntitySet<SC,LO,GO,NO>(EdgeType));
@@ -318,6 +318,7 @@ namespace FROSch {
         FROSCH_TIMER_START_LEVELID(buildEntityMapsTime,"DDInterface::buildEntityMaps");
         //if (Verbose_ && Verbosity_==All) cout << "FROSch::DDInterface : Building global interface component maps" << endl;
 
+
         if (buildVerticesMap) Vertices_->buildEntityMap(NodesMap_);
         if (buildShortEdgesMap) ShortEdges_->buildEntityMap(NodesMap_);
         if (buildStraightEdgesMap) StraightEdges_->buildEntityMap(NodesMap_);
@@ -474,7 +475,7 @@ namespace FROSch {
                     globalVec[i] = -1;
                 }
             }
-
+            NumEntity_ = globalVec;
             if (Verbose_) {
                 cout
                 << "\n" << setw(FROSCH_INDENT) << " "
@@ -560,6 +561,11 @@ namespace FROSch {
         return 0;
     }
 
+    template <class SC,class LO,class GO,class NO>
+    typename DDInterface<SC,LO,GO,NO>::GOVec DDInterface<SC,LO,GO,NO>::getNumEnt() const{
+      return NumEntity_;
+    }
+
     template <class SC,class LO,class GO,class NO>
     int DDInterface<SC,LO,GO,NO>::buildEntityHierarchy()
     {
@@ -838,6 +844,9 @@ namespace FROSch {
         FROSCH_TIMER_START_LEVELID(identifyLocalComponentsTime,"DDInterface::identifyLocalComponents");
         //if (Verbose_ && Verbosity_==All) cout << "FROSch::DDInterface : Classifying interface components based on equivalence classes" << endl;
 
+
+
+
         // Hier herausfinden, ob Ecke, Kante oder Fläche
         UNVecPtr componentsMultiplicity(componentsSubdomainsUnique.size());
         IntVecVecPtr components(componentsSubdomainsUnique.size());
@@ -851,7 +860,6 @@ namespace FROSch {
         for (UN i=0; i<maxMultiplicity+1; i++) {
             EntitySetVector_[i].reset(new EntitySet<SC,LO,GO,NO>(DefaultType));
         }
-
         typename IntVecVec::iterator classIterator;
         LOVecPtr localComponentIndices(NumMyNodes_);
         for (int i=0; i<NumMyNodes_; i++) {