Multi-body and moorings dynamics with Chrono for Proteus (#525)

* [ADD] Chrono wrappers for multibody dynamics

* MBD: chrono moorings coupling working

* MBD: Chrono collision detection added

* MBD: chrono wrappers refactoring; external forces for cable

* MBD: chrono cable external forces updated

* typo fix

* Chrono testing version for force oscillation

* [BUG] chrono: nans in parallel when using moorings

* [BUG] nans on master processor using hydra

* mbd: commented out debugging lines; corrected typo

* mbd: choice between beam and cable elements for moorings

* [FIX] chrono calculations on rank 1 in parallel

* ChElementCableANCF used in chrono; prescribed sine motion added

* mbd: added documentation (partial)

* MBD: cythonised further; py headers in different files

* MBD: added ISS coupling scheme

* msh2simplex: update node flags with face flags (3D)

* Isosurfaces output in .h5 files

* NumericalSolution: fix fileprefix when using gmsh for 3D domain

* MBD: kdtree parallel lookup fixed; WIP on coupling schemes

* MBD: added Euler beams for cables, updated external forces

* msh2simplex: removed flag assignment for triangle nodes in 2D

* MBD: added separated mesh motion function (translation/rotation)

* reintroduced MeshAdaptPUMI in setup.py

* Cekees/mbd/chrono (#570)

* re-enabled Nitsche adjoint for pinc

* updated optimized pressure calculation for projection scheme

* enabled optimized pressure solve for rans3p

* enabled optimized pressure solve for rans3p

* fixed mass flux for rotational form of pressure calc

* worked on isotropic interface size field

* fix 2D Darcy term and isotropic adaptive scheme

* update stack with fixed zoltan build

* updated hashstack (chrono)

* removed irrlicht dependency

* enable avx extensions by default for chrono

.hashstack_default

@@ -1 +1,2 @@
-d3d27601f2de3c4538f11972f68973b142bdbc92
+5eb94cea715c2d5e1f8d2e90fd0a53c96a7a7df5
+

Makefile

@@ -230,6 +230,7 @@ install: profile $(shell find proteus -type f) $(wildcard *.py) proteus
 	$(call howto)
 
 develop: proteus profile 
+	-ln -sf ${PROTEUS}/${PROTEUS_ARCH}/lib64/* ${PROTEUS}/${PROTEUS_ARCH}/lib
 	@echo "************************"
 	@echo "Installing development version"
 	@echo "************************"

proteus/BoundaryConditions.py

@@ -22,6 +22,8 @@ def __init__(self, shape=None, name=None, b_or=None, b_i=0, nd=None):
             assert nd is not None, 'Shape or nd must be passed to BC'
         if b_or is not None:
             self._b_or = b_or[b_i]  # array of orientation of all boundaries of shape
+        else:
+            self._b_or = None
 
     # @staticmethod
     # def newGlobalBC(name, default_value):

proteus/Isosurface.pyx

@@ -8,6 +8,7 @@ AuxiliaryVariables subclasses for extracting isosurfaces and contours
 """
 from collections import defaultdict, OrderedDict
 from itertools import product
+import os
 #from proteus.EGeometry import etriple, ecross, enorm, edot
 
 from mpi4py import MPI
@@ -18,6 +19,7 @@ from numpy.linalg import norm
 
 from . import Comm
 from .AuxiliaryVariables import AV_base
+from proteus import Profiling
 from .Profiling import logEvent as log
 
 from libc.math cimport sqrt
@@ -104,6 +106,7 @@ class Isosurface(AV_base):
         self.format = format
         self.comm = Comm.get()
         self.writeBoundary = writeBoundary
+        self.fileprefix = 'isosurface'
 
     def attachModel(self, model, ar):
         """ Attach this isosurface to the given simulation model.
@@ -128,7 +131,7 @@ class Isosurface(AV_base):
         self.u = fine_grid.u
         self.timeIntegration = fine_grid.timeIntegration
         self.nFrames = 0
-        self.last_output = None
+        self.next_output = 0
         self.writeSceneHeader()
         return self
 
@@ -138,6 +141,7 @@ class Isosurface(AV_base):
         """
         from collections import namedtuple
         self.fieldNames = [isosurface[0] for isosurface in self.isosurfaces]
+        print("ATTACHING TO HDF5 !!", self.fieldNames)
         self.elementNodesArray = h5.getNode("/elementsSpatial_Domain" +
                                             repr(step))[:]
         self.nodeArray = h5.getNode("/nodesSpatial_Domain" + repr(step))[:]
@@ -149,7 +153,7 @@ class Isosurface(AV_base):
                                                       self.isosurfaces[0][0] +
                                                       repr(step))[:])
         self.nFrames = step
-        self.last_output = None
+        self.next_output = 0
         if step == 0:
             self.writeSceneHeader(cam)
         return self
@@ -348,11 +352,30 @@ class Isosurface(AV_base):
         if self.format == 'pov':
             log("Writing pov frame " + repr(frame))
             self.writeIsosurfaceMesh_povray(field, value, frame)
+        elif self.format == 'h5':
+            self.writeIsosurfaceMesh_h5(field, value, frame)
         elif self.format is None:
             pass
         else:
             log("Isosurface file format not recognized")
 
+    def writeIsosurfaceMesh_h5(self, field, value, frame):
+        import h5py
+        nodes = self.nodes[(field, value)]
+        elements = self.elements[(field, value)]
+        normals = self.normals[(field, value)]
+        normal_indices = self.normal_indices[(field, value)]
+        filename = os.path.join(Profiling.logDir, self.fileprefix+str(self.comm.rank())+'.h5')
+        if self.nFrames == 0:
+            f = h5py.File(filename, "w")
+        else:
+            f = h5py.File(filename, "a")
+        dset = f.create_dataset('nodes'+str(self.nFrames), data=nodes)
+        dset = f.create_dataset('elems'+str(self.nFrames), data=elements)
+        dset = f.create_dataset('normals'+str(self.nFrames), data=normals)
+        dset = f.create_dataset('normal_indices'+str(self.nFrames), data=normal_indices)
+        f.close()
+
     def writeIsosurfaceMesh_povray(self, field, value, frame):
         """
         Write the triangular mesh to a povray file
@@ -517,8 +540,7 @@ vertex_vectors {"""
                 return
 
             # check that gauge is ready to be sampled again
-            if (self.last_output is not None and
-                    time < self.last_output + self.sampleRate):
+            if (time < self.next_output):
                 return
         assert self.elementNodesArray.shape[1] == 4, \
             "Elements have {0:d} vertices but algorithm is for tets".format(
@@ -528,8 +550,8 @@ vertex_vectors {"""
                 self.triangulateIsosurface(field, v)
                 self.writeIsosurfaceMesh(field, v, self.nFrames)
         self.nFrames += 1
-        if checkTime:
-            self.last_output = time
+        if checkTime and time != 0:
+            self.next_output += self.sampleRate
 
     def writeSceneHeader(self, cam = None):
         """

proteus/MeshAdaptPUMI/SizeField.cpp

@@ -17,13 +17,9 @@ static void SmoothField(apf::Field* f);
    thickness of refinement near the interface */
 static double isotropicFormula(double phi, double hmin, double hmax)
 {
-  static double const epsilon = 0.02;
-  phi = fabs(phi);
   double size;
-  if (fabs(phi) < epsilon)
+  if (fabs(phi) < 5.0*hmin)
     size = hmin;
-  else if (phi < 3 * epsilon)
-    size = (hmin + hmax) / 2;
   else
     size = hmax;
   return size;
@@ -43,8 +39,30 @@ int MeshAdaptPUMIDrvr::calculateSizeField()
     apf::setScalar(size_iso, v, 0, size);
   }
   m->end(it);
-  for(int i=0; i < 3; i++)
-    SmoothField(size_iso);
+  /*
+    If you just smooth then hmax will just diffuse into the hmin band
+    and you won't really get a band around phi=0 with uniform diameter
+    hmin. Instead, reset to hmin after each smooth within the band in
+    order to ensure the band uses hmin. Iterate on that process until
+    changes in the smoothed size are less than 50% of hmin.
+   */
+  double err_h_max=hmax;
+  int its=0;
+  while (err_h_max > 0.5*hmin && its < 200)
+    {
+      its++;
+      SmoothField(size_iso);
+      err_h_max=0.0;
+      it = m->begin(0);
+      while ((v = m->iterate(it))) {
+	double phi = apf::getScalar(phif, v, 0);
+	double size_current = apf::getScalar(size_iso, v, 0);
+	double size = fmin(size_current,isotropicFormula(phi, hmin, hmax));
+	err_h_max = fmax(err_h_max,fabs(size_current-size));
+	apf::setScalar(size_iso, v, 0, size);
+      }
+      m->end(it);
+    }    
   return 0;
 }
 

proteus/MeshAdaptPUMI/cMeshAdaptPUMI.cpp

@@ -351,7 +351,7 @@ int MeshAdaptPUMIDrvr::adaptPUMIMesh()
     size_iso = samSz::isoSize(m);
   }
   else if (size_field_config == "isotropic")
-    testIsotropicSizeField();
+    calculateSizeField();
   else {
     std::cerr << "unknown size field config " << size_field_config << '\n';
     abort();

proteus/MeshTools.py

@@ -6429,7 +6429,7 @@ def msh2simplex(fileprefix, nd):
     switch = None
     switch_count = -1
     logEvent('msh2simplex: getting nodes and elements')
-    for line in mshfile:
+    for i, line in enumerate(mshfile):
         if 'Nodes' in line:
             switch = 'nodes'
             switch_count = -1
@@ -6466,6 +6466,11 @@ def msh2simplex(fileprefix, nd):
                 elif el_type == 2: # triangle
                     triangle_nb += 1
                     triangles += [[triangle_nb, int(words[s]), int(words[s+1]), int(words[s+2]), flag]]
+                    # update nodes flags
+                    if nd == 3:
+                        for i in range(3):
+                            if nodes[int(words[s+i])-1][4] == 0:
+                                nodes[int(words[s+i])-1][4] = flag
                 elif el_type == 4: # tetrahedron 
                     tetrahedron_nb += 1
                     tetrahedra += [[tetrahedron_nb, int(words[s]), int(words[s+1]), int(words[s+2]), int(words[s+3]), flag]]

proteus/NumericalSolution.py

@@ -294,9 +294,9 @@ def __init__(self,so,pList,nList,sList,opts,simFlagsList=None):
                     fileprefix = p.domain.geofile
                 else:
                     fileprefix = p.domain.polyfile
-                if comm.rank() == 0 and (p.genMesh or not (os.path.exists(p.domain.polyfile+".ele") and
-                                                           os.path.exists(p.domain.polyfile+".node") and
-                                                           os.path.exists(p.domain.polyfile+".face"))):
+                if comm.rank() == 0 and (p.genMesh or not (os.path.exists(fileprefix+".ele") and
+                                                           os.path.exists(fileprefix+".node") and
+                                                           os.path.exists(fileprefix+".face"))):
                     if p.domain.use_gmsh is True:
                         logEvent("Running gmsh to generate 3D mesh for "+p.name,level=1)
                         gmsh_cmd = "time gmsh {0:s} -v 10 -3 -o {1:s} -format msh".format(fileprefix+'.geo', p.domain.geofile+'.msh')
@@ -307,7 +307,7 @@ def __init__(self,so,pList,nList,sList,opts,simFlagsList=None):
                         check_call("tetgen -Vfeen {0:s}.ele".format(fileprefix), shell=True)
                     else:
                         logEvent("Running tetgen to generate 3D mesh for "+p.name, level=1)
-                        tetcmd = "tetgen -{0} {1}.poly".format(n.triangleOptions, p.domain.polyfile)
+                        tetcmd = "tetgen -{0} {1}.poly".format(n.triangleOptions, fileprefix)
                         logEvent("Calling tetgen on rank 0 with command %s" % (tetcmd,))
                         check_call(tetcmd, shell=True)
                         logEvent("Done running tetgen")
@@ -1239,7 +1239,7 @@ def calculateSolution(self,runName):
        #     print "Min / Max residual %s / %s" %(lr.min(),lr.max())
 
         self.nSequenceSteps = 0
-        self.nSolveSteps = 0
+        self.nSolveSteps=self.nList[0].adaptMesh_nSteps-1
         for (self.tn_last,self.tn) in zip(self.tnList[:-1],self.tnList[1:]):
             logEvent("==============================================================",level=0)
             logEvent("Solving over interval [%12.5e,%12.5e]" % (self.tn_last,self.tn),level=0)
@@ -1381,7 +1381,10 @@ def calculateSolution(self,runName):
                     self.tCount+=1
                     for index,model in enumerate(self.modelList):
                         self.archiveSolution(model,index,self.systemStepController.t_system_last)
-
+                #can only handle PUMIDomain's for now
+                self.nSolveSteps += 1
+                if(self.PUMI_estimateError()):
+                    self.PUMI_adaptMesh()
             #end system step iterations
             if self.archiveFlag == ArchiveFlags.EVERY_USER_STEP:
                 self.tCount+=1
@@ -1398,7 +1401,6 @@ def calculateSolution(self,runName):
             self.nSolveSteps += 1
             if(self.PUMI_estimateError()):
               self.PUMI_adaptMesh()
-
         logEvent("Finished calculating solution",level=3)
 
         for index,model in enumerate(self.modelList):