Clean up equivalent polynomials implementation and cutfem method (#1242)

* add backward compatibility for meshing options in numerics

* enable P1-P2 quadrature instance

* fixed 3d mesh generation and extra sqrt on p_L2

* updates from centennial

* fix forgotten beginAssembly and pressure null space in parallel

* memBase from NumericalSolution.py is Profiling.memLast inside the constructor 

* rits-true option for linear solver convergence (exit on r or its)

* safeguard against degenerate quad cut case (cut hits two vertices)

* commented out hack to pin pressure dof for pure neumann problem

* fix memBase issue in refactored mesh gen; add tol to eqp zero test

* fix meshing options backward compatibility

* cleaning up, update openblas in stack

* roll back stack to master

Co-authored-by: Haydel Collins <jhcollins12@gmail.com>

proteus/LinearAlgebraTools.py

@@ -1440,11 +1440,13 @@ def apply(self,A,x,y):
         zero_array = numpy.zeros(len(self.known_dof_is.getIndices()))
 
         tmp2.setValues(self.known_dof_is.getIndices(),zero_array)
+        tmp2.assemblyBegin()
         tmp2.assemblyEnd()
 
         self.kspAp_rho.solve(tmp2, tmp1)
         y.axpy(1.,tmp1)
         y.setValues(self.known_dof_is.getIndices(),zero_array)
+        y.assemblyBegin()
         y.assemblyEnd()
 
         assert numpy.isnan(y.norm())==False, "Applying the schur complement \

proteus/LinearSolvers.py

@@ -421,11 +421,13 @@ def __init__(self,L,par_L,
         self.ksp.setFromOptions()
         # set null space class
         self.null_space = self._set_null_space_class()
-
-        if convergenceTest == 'r-true':
+        self.converged_on_maxit=False
+        if convergenceTest in ['r-true', 'rits-true']:
             self.r_work = self.petsc_L.getVecLeft()
             self.rnorm0 = None
             self.ksp.setConvergenceTest(self._converged_trueRes)
+            if convergenceTest == 'rits-true':
+                self.converged_on_maxit=True
         else:
             self.r_work = None
 
@@ -563,6 +565,8 @@ def _converged_trueRes(self,ksp,its,rnorm):
                 return p4pyPETSc.KSP.ConvergedReason.CONVERGED_RTOL
             if truenorm < ksp.atol:
                 return p4pyPETSc.KSP.ConvergedReason.CONVERGED_ATOL
+            if self.converged_on_maxit and its == ksp.max_it:
+                return p4pyPETSc.KSP.ConvergedReason.CONVERGED_ITS
         return False
 
     def _setPreconditioner(self,
@@ -4421,6 +4425,8 @@ def _defineNullSpaceVec(self,
         This needs to be tested more throughly for parallel
         implmentations.
         """
+        from proteus import Comm
+        comm = Comm.get()
         ksp = self.get_global_ksp()
         stabilized = False
         if ksp.par_L.pde.u[0].femSpace.dofMap.nDOF_all_processes==ksp.par_L.pde.u[1].femSpace.dofMap.nDOF_all_processes:
@@ -4431,14 +4437,18 @@ def _defineNullSpaceVec(self,
         null_space_vector = par_b.copy()
         null_space_vector.getArray().fill(0.)
         N_DOF_pressure = ksp.par_L.pde.u[0].femSpace.dofMap.nDOF_all_processes
-
+        N_DOF_pressure_subdomain_owned = ksp.par_L.pde.u[0].femSpace.dofMap.dof_offsets_subdomain_owned[comm.rank()+1] -ksp.par_L.pde.u[0].femSpace.dofMap.dof_offsets_subdomain_owned[comm.rank()]
         if stabilized:
-            tmp = null_space_vector.getArray()[::3]
-            tmp[:] = old_div(1.0, (sqrt(N_DOF_pressure)))
+            tmp = null_space_vector.getArray()[::ksp.par_L.pde.nSpace_global+1]
+            assert ksp.par_L.pde.isActiveDOF_p[:N_DOF_pressure_subdomain_owned].shape[0] == tmp.shape[0], str(tmp.shape) + " "+ str(ksp.par_L.pde.isActiveDOF_p[:N_DOF_pressure_subdomain_owned].shape)
+            N_ACTIVE_DOF_pressure = comm.globalSum(ksp.par_L.pde.isActiveDOF_p[:N_DOF_pressure_subdomain_owned].sum())
+            tmp[:] = np.where(ksp.par_L.pde.isActiveDOF_p[:N_DOF_pressure_subdomain_owned]==1.0, 1.0/sqrt(N_ACTIVE_DOF_pressure),0.0)
         else:
             n_DOF_pressure = ksp.par_L.pde.u[0].femSpace.dofMap.nDOF
             tmp = null_space_vector.getArray()[0:n_DOF_pressure]
             tmp[:] = old_div(1.0, (sqrt(N_DOF_pressure)))
+        null_space_vector.assemblyBegin()
+        null_space_vector.assemblyEnd()
         self.global_null_space = [null_space_vector]
 
 class ConstantNullSpace(SolverNullSpace):

proteus/MixedModelFactory.h

@@ -435,6 +435,16 @@ namespace proteus
 				abort();
 			      }
 			  }
+			else if (nQuadraturePoints_elementIn == 7)
+			  {
+			    if (nQuadraturePoints_elementBoundaryIn == 5)
+			      return static_cast<Model_Base*>(new ModelTemplate<CompKernelTemplate<2,3,3,3>,CompKernelTemplate_v<2,3,6,6>,2,7,3,3,3,6,6,5>());
+			    else
+			      {
+				NO_INSTANCE;
+				abort();
+			      }
+			  }
 			else if (nQuadraturePoints_elementIn == 12)
 			  {
 			    if (nQuadraturePoints_elementBoundaryIn == 6)

proteus/NumericalSolution.py

@@ -192,11 +192,11 @@ def __init__(self,so,pList,nList,sList,opts,simFlagsList=None,TwoPhaseFlow=False
                 p.domain.MeshOptions.quad = n.quad
                 p.domain.MeshOptions.triangleFlag=n.triangleFlag
                 p.domain.MeshOptions.triangleOptions=n.triangleOptions
-
+                p.domain.MeshOptions.genMesh=p.genMesh
         try:
             mlMesh = self.pList[0].domain.mesh
         except:
-            mlMesh = MeshTools.generateMesh(self.pList[0].domain,opts.generatePartitionedMeshFromFiles)
+            mlMesh = MeshTools.generateMesh(self.pList[0],self.nList[0],opts.generatePartitionedMeshFromFiles)
 
         theMesh = mlMesh.meshList[0].subdomainMesh
 

proteus/SpatialTools.py

@@ -595,7 +595,7 @@ def constructShape(self):
 
         hxi = old_div(radius,(math.sqrt(2.0)*float(nSectors)));
         heta = old_div(radius,(math.sqrt(2.0)*float(nSectors)));
-    #now loop over grains
+        #now loop over grains
         #top half  sphere nodes
         top_nodes = {}
         for ii in range(2*nSectors+1):

proteus/config/__init__.py

@@ -1,8 +1,8 @@
 from __future__ import absolute_import
 import os
 
-if 'PROTEUS_ARCH' in os.environ and os.environ['PROTEUS_ARCH'].startswith('garnet'):
-    from .garnet import *
+if 'PROTEUS_ARCH' in os.environ and os.environ['PROTEUS_ARCH'].startswith('centennial'):
+    from .centennial import *
 elif 'PROTEUS_ARCH' in os.environ and os.environ['PROTEUS_ARCH'].startswith('jim'):
     from .jim import *
 elif 'PROTEUS_ARCH' in os.environ and os.environ['PROTEUS_ARCH'].startswith('mustang'):

proteus/equivalent_polynomials.h

@@ -3,6 +3,7 @@
 #include <cmath>
 #include <cassert>
 #include <iostream>
+#include <iomanip>
 #include "equivalent_polynomials_coefficients.h"
 #include "equivalent_polynomials_coefficients_quad.h"
 #include "equivalent_polynomials_utils.h"
@@ -220,6 +221,10 @@ namespace equivalent_polynomials
 	    C_D[i] = 0.0;
 	    for (unsigned int j=0; j < nDOF; j++)
 	      {
+		assert(!isnan(Ainv[i*nDOF + j]));
+		assert(!isnan(b_H[j]));
+		assert(!isnan(b_ImH[j]));
+		assert(!isnan(b_D[j]));
 		C_H[i]   += Ainv[i*nDOF + j]*b_H[j];
 		C_ImH[i] += Ainv[i*nDOF + j]*b_ImH[j];
 		C_D[i]   += Ainv[i*nDOF + j]*b_D[j];
@@ -265,15 +270,16 @@ namespace equivalent_polynomials
     root_node=0;
     inside_out=false;
     quad_cut=false;
+    const double eps=1.0e-8;
     for (unsigned int i=0; i < nN; i++)
       {
-        if(phi_dof[i] > 0.0)
+        if(phi_dof[i] > eps)
           {
 	    if (pcount == 0)
 	      p_i = i;
             pcount  += 1;
           }
-        else if(phi_dof[i] < 0.0)
+        else if(phi_dof[i] < -eps)
           {
 	    if (ncount == 0)
 	      n_i = i;
@@ -445,6 +451,7 @@ namespace equivalent_polynomials
   template<int nSpace, int nP, int nQ, int nEBQ>
   inline void Simplex<nSpace,nP,nQ,nEBQ>::_calculate_cuts()
   {
+    const double eps=1.0e-8;
     for (unsigned int i=0; i < nN-1;i++)
       {
         if(phi[i+1]*phi[0] < 0.0)
@@ -459,7 +466,7 @@ namespace equivalent_polynomials
           }
         else
           {
-            assert(phi[i+1] == 0.0);
+            assert(phi[i+1] < eps);
             X_0[i] = 1.0;
             for (unsigned int I=0; I < 3; I++)
               {
@@ -472,10 +479,18 @@ namespace equivalent_polynomials
   template<int nSpace, int nP, int nQ, int nEBQ>
   inline void Simplex<nSpace,nP,nQ,nEBQ>::_calculate_cuts_quad()
   {
-    THETA_01 = 0.5 - 0.5*(phi[1] + phi[0])/(phi[1]-phi[0]);
-    THETA_02 = 0.5 - 0.5*(phi[2] + phi[0])/(phi[2]-phi[0]);
-    THETA_31 = 0.5 - 0.5*(phi[1] + phi[3])/(phi[1]-phi[3]);
-    THETA_32 = 0.5 - 0.5*(phi[2] + phi[3])/(phi[2]-phi[3]);
+    const double eps=1.0e-8,Imeps=1.0-eps;
+    THETA_01 = 0.5 - 0.5*(phi[1] + phi[0])/(phi[1] - phi[0]);
+    THETA_02 = 0.5 - 0.5*(phi[2] + phi[0])/(phi[2] - phi[0]);
+    THETA_31 = 0.5 - 0.5*(phi[1] + phi[3])/(phi[1] - phi[3]);
+    THETA_32 = 0.5 - 0.5*(phi[2] + phi[3])/(phi[2] - phi[3]);
+    if ( (THETA_01 < eps || THETA_01 > Imeps) || (THETA_02 < eps || THETA_02 > Imeps) || (THETA_31 < eps || THETA_31 > Imeps) || (THETA_32 < eps || THETA_32 > Imeps))
+      {
+	THETA_01 = fmin(Imeps,fmax(eps,0.5 - 0.5*(phi[1] + phi[0])/(phi[1] - phi[0])));
+	THETA_02 = fmin(Imeps,fmax(eps,0.5 - 0.5*(phi[2] + phi[0])/(phi[2] - phi[0])));
+	THETA_31 = fmin(Imeps,fmax(eps,0.5 - 0.5*(phi[1] + phi[3])/(phi[1] - phi[3])));
+	THETA_32 = fmin(Imeps,fmax(eps,0.5 - 0.5*(phi[2] + phi[3])/(phi[2] - phi[3])));
+      }
     for (unsigned int I=0; I < 3; I++)
       {
 	phys_nodes_cut_quad_01[I] = (1-THETA_01)*nodes[I] + THETA_01*nodes[1*3 + I];

proteus/equivalent_polynomials_utils.h

@@ -29,8 +29,11 @@ namespace equivalent_polynomials
       for(unsigned int I=0; I < nSpace; I++)
         level_set_normal[I] /=  norm;
     else
-      for(unsigned int I=0; I < nSpace; I++)
-        level_set_normal[I] =0.0;
+      {
+	for(unsigned int I=0; I < nSpace; I++)
+	  level_set_normal[I] =0.0;
+	level_set_normal[1] = 1.0;
+      }
   }
 
   template<>
@@ -56,8 +59,12 @@ namespace equivalent_polynomials
       for(unsigned int I=0; I < nSpace; I++)
         level_set_normal[I] /=  norm;
     else
-      for(unsigned int I=0; I < nSpace; I++)
-        level_set_normal[I] =0.0;
+      {
+	for(unsigned int I=0; I < nSpace; I++)
+	  level_set_normal[I] =0.0;
+	level_set_normal[2] =1.0;
+      }
+    assert(std::fabs(1.0-level_set_normal[0]*level_set_normal[0] - level_set_normal[1]*level_set_normal[1] - level_set_normal[2]*level_set_normal[2]) < 1.0e-8);
   }
 
   inline void _calculate_normal_quad(double* phys_nodes_cut_quad_01,
@@ -67,52 +74,31 @@ namespace equivalent_polynomials
 				     double* level_set_normal)
   {
     const unsigned int nSpace(3);
-    double level_set_tangent_a[3], level_set_tangent_b[3],level_set_normal_2[3];
+    double level_set_tangent_a[3], level_set_tangent_b[3], level_set_tangent_c[3], level_set_normal_2[3];
     for(unsigned int I=0; I < nSpace; I++)
       {
         level_set_tangent_a[I] = phys_nodes_cut_quad_02[I] - phys_nodes_cut_quad_01[I];
         level_set_tangent_b[I] = phys_nodes_cut_quad_32[I] - phys_nodes_cut_quad_01[I];
+        level_set_tangent_c[I] = phys_nodes_cut_quad_31[I] - phys_nodes_cut_quad_01[I];
       }
-    level_set_normal[0] =   level_set_tangent_a[1]*level_set_tangent_b[2]
-      - level_set_tangent_a[2]*level_set_tangent_b[1];
-    level_set_normal[1] = - level_set_tangent_a[0]*level_set_tangent_b[2]
-      + level_set_tangent_a[2]*level_set_tangent_b[0];
-    level_set_normal[2] =   level_set_tangent_a[0]*level_set_tangent_b[1]
-      - level_set_tangent_a[1]*level_set_tangent_b[0];
+    level_set_normal[0] =   level_set_tangent_a[1]*level_set_tangent_b[2] - level_set_tangent_a[2]*level_set_tangent_b[1];
+    level_set_normal[1] = - level_set_tangent_a[0]*level_set_tangent_b[2] + level_set_tangent_a[2]*level_set_tangent_b[0];
+    level_set_normal[2] =   level_set_tangent_a[0]*level_set_tangent_b[1] - level_set_tangent_a[1]*level_set_tangent_b[0];
     double norm = std::sqrt(level_set_normal[0]*level_set_normal[0] +
                             level_set_normal[1]*level_set_normal[1] +
                             level_set_normal[2]*level_set_normal[2]);
     if (norm > 0.0)
-      for(unsigned int I=0; I < nSpace; I++)
-        level_set_normal[I] /=  norm;
-    else
-      for(unsigned int I=0; I < nSpace; I++)
-        level_set_normal[I] =0.0;
-    for(unsigned int I=0; I < nSpace; I++)
       {
-        level_set_tangent_a[I] = phys_nodes_cut_quad_32[I] - phys_nodes_cut_quad_01[I];
-        level_set_tangent_b[I] = phys_nodes_cut_quad_31[I] - phys_nodes_cut_quad_01[I];
+	for(unsigned int I=0; I < nSpace; I++)
+	  level_set_normal[I] =  level_set_normal[I] / norm;
       }
-    level_set_normal_2[0] =   level_set_tangent_a[1]*level_set_tangent_b[2]
-      - level_set_tangent_a[2]*level_set_tangent_b[1];
-    level_set_normal_2[1] = - level_set_tangent_a[0]*level_set_tangent_b[2]
-      + level_set_tangent_a[2]*level_set_tangent_b[0];
-    level_set_normal_2[2] =   level_set_tangent_a[0]*level_set_tangent_b[1]
-      - level_set_tangent_a[1]*level_set_tangent_b[0];
-    norm = std::sqrt(level_set_normal_2[0]*level_set_normal_2[0] +
-		     level_set_normal_2[1]*level_set_normal_2[1] +
-		     level_set_normal_2[2]*level_set_normal_2[2]);
-    if (norm > 0.0)
+    else
       {
 	for(unsigned int I=0; I < nSpace; I++)
-	  level_set_normal_2[I] /=  norm;
-	double coplanarity_error=1.0;
-	for(unsigned int I=0; I < nSpace; I++)
-	  coplanarity_error -= level_set_normal[I]*level_set_normal_2[I];
-	assert(coplanarity_error < 1.0e-12);
-	for(unsigned int I=0; I < nSpace; I++)
-	  level_set_normal[I] = 0.5*(level_set_normal[I]+level_set_normal_2[I]);
+	  level_set_normal[I] =0.0;
+	level_set_normal[2] = 1.0;
       }
+    assert(std::fabs(1.0-level_set_normal[0]*level_set_normal[0] - level_set_normal[1]*level_set_normal[1] - level_set_normal[2]*level_set_normal[2]) < 1.0e-8);
   }
 
   template<int nP>

proteus/mprans/RANS2P.h

@@ -4745,10 +4745,6 @@ namespace proteus
 		    }
                 }
             }
-          p_L2 = sqrt(p_L2);
-          //        std::cout<<"Pressure Integral Shifted"<<p_dv_new<<std::endl
-          //         <<"Analytical Pressure Integral 2 "<<pa_dv_new<<std::endl
-          //         <<"Errors "<<p_L1<<'\t'<<p_L2<<'\t'<<p_LI<<std::endl;
         }
       assert(errors.shape(0)*errors.shape(1) == 15);
       MPI_Allreduce(MPI_IN_PLACE, errors.data(),(errors.shape(0)-1)*errors.shape(1),MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);

proteus/mprans/RANS2P.py

@@ -1885,6 +1885,13 @@ def getResidual(self, u, r):
                                 r[self.offset[cj] + self.stride[cj] * dofN] -= self.mesh.nodeVelocityArray[dofN, cj - 1]
                         #assert(abs(r[self.offset[cj] + self.stride[cj] * dofN]) < 1.0e-8)
 
+        #cek hack to fix singular system due pressure
+        # comm = Comm.get()
+        # if comm.rank() == 0:
+        #     for i in range(self.u[0].dof.shape[0]):
+        #         if self.isActiveR[i] != 1.0:
+        #             self.isActiveR[i] == 0.0
+        #             break
         try:
             #is sensitive to inactive DOF at velocity due to time derivative
             if self.coefficients.nParticles ==1:
@@ -2225,6 +2232,13 @@ def getJacobian(self, jacobian):
                         else:
                             self.nzval[i] = 0.0
                             # print "RBLES zeroing residual cj = %s dofN= %s global_dofN= %s " % (cj,dofN,global_dofN)
+        #cek hack to fix singular system due pressure
+        # comm = Comm.get()
+        # if comm.rank() == 0:
+        #     for i in range(self.u[0].dof.shape[0]):
+        #         if self.isActiveR[i] != 1.0:
+        #             self.isActiveR[i] == 0.0
+        #             break
         for global_dofN_a in np.argwhere(self.isActiveR==0.0):
             #assert(False)
             global_dofN = global_dofN_a[0]

proteus/mprans/RANS2P2D.h

@@ -2875,7 +2875,6 @@ namespace proteus
           mesh_volume_conservation_err_max=fmax(mesh_volume_conservation_err_max,fabs(mesh_volume_conservation_element));
           mesh_volume_conservation_err_max_weak=fmax(mesh_volume_conservation_err_max_weak,fabs(mesh_volume_conservation_element_weak));
         }//elements
-      //std::cout<<"p,u,v L2 error integrals (shoudl be non-negative) "<<p_L2<<'\t'<<u_L2<<'\t'<<v_L2<<'\t'<<"Flow Domain Volume = "<<domain_volume<<std::endl;
       for (std::set<int>::iterator it=cutfem_boundaries.begin(); it!=cutfem_boundaries.end(); )
         {
           if(elementIsActive[elementBoundaryElementsArray[(*it)*2+0]] && elementIsActive[elementBoundaryElementsArray[(*it)*2+1]])
@@ -3970,10 +3969,6 @@ namespace proteus
 		    }
                 }
             }
-          p_L2 = sqrt(p_L2);
-          //        std::cout<<"Pressure Integral Shifted"<<p_dv_new<<std::endl
-          //         <<"Analytical Pressure Integral 2 "<<pa_dv_new<<std::endl
-          //         <<"Errors "<<p_L1<<'\t'<<p_L2<<'\t'<<p_LI<<std::endl;
         }
       assert(errors.shape(0)*errors.shape(1) == 15);
       MPI_Allreduce(MPI_IN_PLACE, errors.data(),(errors.shape(0)-1)*errors.shape(1),MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);

proteus/tests/ci/poisson_3d_tetgen_c0p1_n.py

@@ -68,8 +68,8 @@
     #parallelPartitioningType = MeshParallelPartitioningTypes.element
     #have to have a numerical flux in parallel
     numericalFluxType = Advection_DiagonalUpwind_Diffusion_IIPG_exterior
-    #for true residual test
-    linearSolverConvergenceTest = 'r-true'
+    #for true residual test or maxits
+    linearSolverConvergenceTest = 'rits-true'
     #to allow multiple models to set different ksp options
     #linear_solver_options_prefix = 'poisson_'
     linearSmoother = None