From 938f5d12eb222646b92e79a3eac95d2d10dfe0c6 Mon Sep 17 00:00:00 2001 From: sseraj Date: Mon, 11 Sep 2023 17:36:56 -0400 Subject: [PATCH 01/16] starting to parallelize tri surf --- pygeo/parameterization/DVGeoMulti.py | 65 +++++++++++++++++++++++++--- 1 file changed, 59 insertions(+), 6 deletions(-) diff --git a/pygeo/parameterization/DVGeoMulti.py b/pygeo/parameterization/DVGeoMulti.py index eeec1d16..feff8856 100644 --- a/pygeo/parameterization/DVGeoMulti.py +++ b/pygeo/parameterization/DVGeoMulti.py @@ -114,14 +114,37 @@ def addComponent(self, comp, DVGeo, triMesh=None, scale=1.0, bbox=None, pointSet # scale the nodes nodes *= scale - # add these points to the corresponding dvgeo - DVGeo.addPointSet(nodes, "triMesh", **pointSetKwargs) + # We will split up the points by processor when adding them to the component DVGeo + + # Compute the processor sizes with integer division + sizes = np.zeros(self.comm.size, dtype="intc") + nPts = nodes.shape[0] + sizes[:] = nPts // self.comm.size + + # Add the leftovers + sizes[: nPts % self.comm.size] += 1 + + # Compute the processor displacements + disp = np.zeros(self.comm.size + 1, dtype="intc") + disp[1:] = np.cumsum(sizes) + + # Save the size and displacement in a dictionary + triSurfData = {} + triSurfData["sizes"] = sizes + triSurfData["disp"] = disp + + # Split up the points into the points for this processor + procNodes = nodes[disp[self.comm.rank] : disp[self.comm.rank + 1]] + + # Add these points to the component DVGeo + DVGeo.addPointSet(procNodes, "triMesh", **pointSetKwargs) else: # the user has not provided a triangulated surface mesh for this file nodes = None triConn = None triConnStack = None barsConn = None + triSurfData = None # we will need the bounding box information later on, so save this here xMin, xMax = DVGeo.FFD.getBounds() @@ -141,7 +164,9 @@ def addComponent(self, comp, DVGeo, triMesh=None, scale=1.0, bbox=None, pointSet xMax[2] = bbox["zmax"] # initialize the component object - self.comps[comp] = component(comp, DVGeo, nodes, triConn, triConnStack, barsConn, xMin, xMax) + self.comps[comp] = component( + self.comm, comp, DVGeo, nodes, triConn, triConnStack, barsConn, xMin, xMax, triSurfData + ) # add the name to the list self.compNames.append(comp) @@ -910,8 +935,9 @@ def _computeTotalJacobian(self, ptSetName): class component: - def __init__(self, name, DVGeo, nodes, triConn, triConnStack, barsConn, xMin, xMax): + def __init__(self, comm, name, DVGeo, nodes, triConn, triConnStack, barsConn, xMin, xMax, triSurfData): # save the info + self.comm = comm self.name = name self.DVGeo = DVGeo self.nodes = nodes @@ -920,6 +946,7 @@ def __init__(self, name, DVGeo, nodes, triConn, triConnStack, barsConn, xMin, xM self.barsConn = barsConn self.xMin = xMin self.xMax = xMax + self.triSurfData = triSurfData # also a dictionary for DV names self.dvDict = {} @@ -931,8 +958,34 @@ def __init__(self, name, DVGeo, nodes, triConn, triConnStack, barsConn, xMin, xM self.triMesh = True def updateTriMesh(self): - # update the triangulated surface mesh - self.nodes = self.DVGeo.update("triMesh") + # We need the full triangulated surface for this component + # Get the stored processor splitting information + sizes = self.triSurfData["sizes"] + disp = self.triSurfData["disp"] + nPts = disp[-1] + + # Update the triangulated surface mesh to get the points on this processor + procNodes = self.DVGeo.update("triMesh") + + # Create the send buffer + procNodes = procNodes.flatten() + sendbuf = [procNodes, sizes[self.comm.rank] * 3] + + # Set the appropriate type for the receiving buffer + if procNodes.dtype == float: + mpiType = MPI.DOUBLE + elif procNodes.dtype == complex: + mpiType = MPI.DOUBLE_COMPLEX + + # Create the receiving buffer + newPtsGlobal = np.zeros(nPts * 3, dtype=procNodes.dtype) + recvbuf = [newPtsGlobal, sizes * 3, disp[0:-1] * 3, mpiType] + + # Allgather the updated coordinates + self.comm.Allgatherv(sendbuf, recvbuf) + + # Reshape into a nPts, 3 array + self.nodes = newPtsGlobal.reshape((nPts, 3)) class PointSet: From 583f686da06ece7bff24e16c0d9c3973deac17fb Mon Sep 17 00:00:00 2001 From: sseraj Date: Mon, 11 Sep 2023 19:22:52 -0400 Subject: [PATCH 02/16] renamed triSurf to triMesh for consistency --- pygeo/parameterization/DVGeoMulti.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pygeo/parameterization/DVGeoMulti.py b/pygeo/parameterization/DVGeoMulti.py index feff8856..47b24206 100644 --- a/pygeo/parameterization/DVGeoMulti.py +++ b/pygeo/parameterization/DVGeoMulti.py @@ -129,9 +129,9 @@ def addComponent(self, comp, DVGeo, triMesh=None, scale=1.0, bbox=None, pointSet disp[1:] = np.cumsum(sizes) # Save the size and displacement in a dictionary - triSurfData = {} - triSurfData["sizes"] = sizes - triSurfData["disp"] = disp + triMeshData = {} + triMeshData["sizes"] = sizes + triMeshData["disp"] = disp # Split up the points into the points for this processor procNodes = nodes[disp[self.comm.rank] : disp[self.comm.rank + 1]] @@ -144,7 +144,7 @@ def addComponent(self, comp, DVGeo, triMesh=None, scale=1.0, bbox=None, pointSet triConn = None triConnStack = None barsConn = None - triSurfData = None + triMeshData = None # we will need the bounding box information later on, so save this here xMin, xMax = DVGeo.FFD.getBounds() @@ -165,7 +165,7 @@ def addComponent(self, comp, DVGeo, triMesh=None, scale=1.0, bbox=None, pointSet # initialize the component object self.comps[comp] = component( - self.comm, comp, DVGeo, nodes, triConn, triConnStack, barsConn, xMin, xMax, triSurfData + self.comm, comp, DVGeo, nodes, triConn, triConnStack, barsConn, xMin, xMax, triMeshData ) # add the name to the list @@ -935,7 +935,7 @@ def _computeTotalJacobian(self, ptSetName): class component: - def __init__(self, comm, name, DVGeo, nodes, triConn, triConnStack, barsConn, xMin, xMax, triSurfData): + def __init__(self, comm, name, DVGeo, nodes, triConn, triConnStack, barsConn, xMin, xMax, triMeshData): # save the info self.comm = comm self.name = name @@ -946,7 +946,7 @@ def __init__(self, comm, name, DVGeo, nodes, triConn, triConnStack, barsConn, xM self.barsConn = barsConn self.xMin = xMin self.xMax = xMax - self.triSurfData = triSurfData + self.triMeshData = triMeshData # also a dictionary for DV names self.dvDict = {} @@ -960,8 +960,8 @@ def __init__(self, comm, name, DVGeo, nodes, triConn, triConnStack, barsConn, xM def updateTriMesh(self): # We need the full triangulated surface for this component # Get the stored processor splitting information - sizes = self.triSurfData["sizes"] - disp = self.triSurfData["disp"] + sizes = self.triMeshData["sizes"] + disp = self.triMeshData["disp"] nPts = disp[-1] # Update the triangulated surface mesh to get the points on this processor From 0b728bd5c45d935ab0d7b685d514a2d405f7f30f Mon Sep 17 00:00:00 2001 From: sseraj Date: Mon, 11 Sep 2023 19:39:33 -0400 Subject: [PATCH 03/16] parallelized tri mesh derivatives --- pygeo/parameterization/DVGeoMulti.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/pygeo/parameterization/DVGeoMulti.py b/pygeo/parameterization/DVGeoMulti.py index 47b24206..5c602430 100644 --- a/pygeo/parameterization/DVGeoMulti.py +++ b/pygeo/parameterization/DVGeoMulti.py @@ -2455,8 +2455,12 @@ def _projectToComponent_b(self, dIdpt, comp, projDict, surface=None): # Now we are done with the ADT self.adtAPI.adtdeallocateadts(adtID) + # Extract the entries of dIdptComp that are for points on this processor + disp = comp.triMeshData["disp"] + dIdptComp = dIdptComp[:, disp[comp.comm.rank] : disp[comp.comm.rank + 1], :] + # Call the total sensitivity of the component's DVGeo - compSens = comp.DVGeo.totalSensitivity(dIdptComp, "triMesh") + compSens = comp.DVGeo.totalSensitivity(dIdptComp, "triMesh", comm=comp.comm) # the entries in dIdpt is replaced with AD seeds of initial points that were projected # we also return the total sensitivity contributions from components' triMeshes @@ -3187,13 +3191,20 @@ def _getIntersectionSeam_b(self, seamBar, comm): coorAb[ii] += cAb.T coorBb[ii] += cBb.T + # Extract the entries of coorAb and coorBb that are for points on this processor + disp = self.compA.triMeshData["disp"] + coorAb = coorAb[:, disp[self.compA.comm.rank] : disp[self.compA.comm.rank + 1], :] + + disp = self.compB.triMeshData["disp"] + coorBb = coorBb[:, disp[self.compB.comm.rank] : disp[self.compB.comm.rank + 1], :] + # get the total sensitivities from both components compSens_local = {} - compSensA = self.compA.DVGeo.totalSensitivity(coorAb, "triMesh") + compSensA = self.compA.DVGeo.totalSensitivity(coorAb, "triMesh", comm=self.compA.comm) for k, v in compSensA.items(): compSens_local[k] = v - compSensB = self.compB.DVGeo.totalSensitivity(coorBb, "triMesh") + compSensB = self.compB.DVGeo.totalSensitivity(coorBb, "triMesh", comm=self.compB.comm) for k, v in compSensB.items(): compSens_local[k] = v From 13dcdb5c43ab741132cfe8066ea1e00b9421d22d Mon Sep 17 00:00:00 2001 From: sseraj Date: Mon, 18 Sep 2023 17:01:50 -0400 Subject: [PATCH 04/16] fixed parallel KeyError for excludeSurfaces --- pygeo/parameterization/DVGeoMulti.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/pygeo/parameterization/DVGeoMulti.py b/pygeo/parameterization/DVGeoMulti.py index 5c602430..424d871d 100644 --- a/pygeo/parameterization/DVGeoMulti.py +++ b/pygeo/parameterization/DVGeoMulti.py @@ -1293,11 +1293,17 @@ def addPointSet(self, pts, ptSetName, compMap, comm): # Combine the excluded indices using a set to avoid duplicates excludeSet = set() for surface in self.excludeSurfaces: - if surface in self.compA.triConn: + compASurfaceInd = self.projData[ptSetName]["compA"]["surfaceInd"] + compBSurfaceInd = self.projData[ptSetName]["compB"]["surfaceInd"] + + if surface in compASurfaceInd: # Pop this surface from the saved data - surfaceInd = self.projData[ptSetName]["compA"]["surfaceInd"].pop(surface) - elif surface in self.compB.triConn: - surfaceInd = self.projData[ptSetName]["compB"]["surfaceInd"].pop(surface) + surfaceInd = compASurfaceInd.pop(surface) + elif surface in compBSurfaceInd: + surfaceInd = compBSurfaceInd.pop(surface) + else: + # This processor has no points on this excluded surface + surfaceInd = set() excludeSet.update(surfaceInd) From 3a7d5dab38e450d38f28bb87304ad6aae7771680 Mon Sep 17 00:00:00 2001 From: sseraj Date: Mon, 18 Sep 2023 18:56:44 -0400 Subject: [PATCH 05/16] working on parallelizing the DVGeoMulti test --- tests/reg_tests/test_DVGeometryMulti.py | 66 +++++++++++++++++-------- 1 file changed, 46 insertions(+), 20 deletions(-) diff --git a/tests/reg_tests/test_DVGeometryMulti.py b/tests/reg_tests/test_DVGeometryMulti.py index 4ff0b851..3a6673d7 100644 --- a/tests/reg_tests/test_DVGeometryMulti.py +++ b/tests/reg_tests/test_DVGeometryMulti.py @@ -41,6 +41,9 @@ def test_boxes(self, train=False): ffdFiles = [os.path.join(inputDir, f"{comp}.xyz") for comp in comps] triMeshFiles = [os.path.join(inputDir, f"{comp}.cgns") for comp in comps] + # Define the communicator + comm = MPI.COMM_WORLD + # Set up real component DVGeo objects DVGeoBox1_real = DVGeometry(ffdFiles[0]) DVGeoBox2_real = DVGeometry(ffdFiles[1]) @@ -52,13 +55,13 @@ def test_boxes(self, train=False): DVGeoBox3_complex = DVGeometry(ffdFiles[2], isComplex=True) # Set up real DVGeometryMulti object - DVGeo_real = DVGeometryMulti() + DVGeo_real = DVGeometryMulti(comm=comm) DVGeo_real.addComponent("box1", DVGeoBox1_real, triMeshFiles[0]) DVGeo_real.addComponent("box2", DVGeoBox2_real, triMeshFiles[1]) DVGeo_real.addComponent("box3", DVGeoBox3_real, None) # Set up complex DVGeometryMulti object - DVGeo_complex = DVGeometryMulti(isComplex=True) + DVGeo_complex = DVGeometryMulti(comm=comm, isComplex=True) DVGeo_complex.addComponent("box1", DVGeoBox1_complex, triMeshFiles[0]) DVGeo_complex.addComponent("box2", DVGeoBox2_complex, triMeshFiles[1]) DVGeo_complex.addComponent("box3", DVGeoBox3_complex, None) @@ -99,6 +102,9 @@ def test_boxes(self, train=False): "part_40": 1e-3, } + # Define a name for the point set + ptSetName = "test_set" + # Define a test point set pts = np.array( [ @@ -136,13 +142,24 @@ def test_boxes(self, train=False): [0.5, 0.25, 0.6], [0.25, 0.5, 0.6], [0.5, -0.25, 0.6], - [0.25, -0.5, 0.6], + # [0.25, -0.5, 0.6], ] ) - # Define a name and comm for the point set - ptSetName = "test_set" - comm = MPI.COMM_WORLD + # Compute the processor sizes with integer division + sizes = np.zeros(comm.size, dtype="intc") + nPtsGlobal = pts.shape[0] + sizes[:] = nPtsGlobal // comm.size + + # Add the leftovers + sizes[: nPtsGlobal % comm.size] += 1 + + # Compute the processor displacements + disp = np.zeros(comm.size + 1, dtype="intc") + disp[1:] = np.cumsum(sizes) + + # Split up the point set + localPts = pts[disp[comm.rank] : disp[comm.rank + 1]] # Set up the complex and real DVGeoMulti objects for DVGeo in [DVGeo_complex, DVGeo_real]: @@ -177,9 +194,9 @@ def twist(val, geo, nRefAxPts=nRefAxPts): # Set the correct dtype for the point set if DVGeo == DVGeo_complex: - pts_dtype = pts.astype(complex) + pts_dtype = localPts.astype(complex) else: - pts_dtype = pts + pts_dtype = localPts # Add the point set DVGeo.addPointSet(pts_dtype, ptSetName, comm=comm, applyIC=True) @@ -195,25 +212,25 @@ def twist(val, geo, nRefAxPts=nRefAxPts): # Regression test the updated points for the real DVGeo refFile = os.path.join(baseDir, "ref/test_DVGeometryMulti.ref") - with BaseRegTest(refFile, train=train) as handler: - handler.par_add_val("ptsUpdated", ptsUpdated, tol=1e-14) + # with BaseRegTest(refFile, train=train) as handler: + # handler.par_add_val("ptsUpdated", ptsUpdated, tol=1e-14) # Now we will test the derivatives # Build a dIdpt array - # We will have nNodes*3 many functions of interest - nNodes = pts.shape[0] - dIdpt = np.zeros((nNodes * 3, nNodes, 3)) + # We will have nPtsGlobal*3 many functions of interest + nPtsLocal = localPts.shape[0] + dIdpt = np.zeros((nPtsGlobal * 3, nPtsLocal, 3)) # Set the seeds to one such that we get individual derivatives for each coordinate of each point # The first function of interest gets the first coordinate of the first point # The second function gets the second coordinate of first point, and so on - for i in range(nNodes): + for i in range(disp[comm.rank], disp[comm.rank + 1]): for j in range(3): - dIdpt[i * 3 + j, i, j] = 1 + dIdpt[i * 3 + j, i - disp[comm.rank], j] = 1 # Get the derivatives from the real DVGeoMulti object - funcSens = DVGeo_real.totalSensitivity(dIdpt, ptSetName) + funcSens = DVGeo_real.totalSensitivity(dIdpt, ptSetName, comm=comm) # Compute FD and CS derivatives dvDict_real = DVGeo_real.getValues() @@ -226,8 +243,8 @@ def twist(val, geo, nRefAxPts=nRefAxPts): for x in dvDict_real: nx = len(dvDict_real[x]) - funcSensFD[x] = np.zeros((nx, nNodes * 3)) - funcSensCS[x] = np.zeros((nx, nNodes * 3)) + funcSensFD[x] = np.zeros((nx, nPtsGlobal * 3)) + funcSensCS[x] = np.zeros((nx, nPtsGlobal * 3)) for i in range(nx): xRef_real = dvDict_real[x][i].copy() xRef_complex = dvDict_complex[x][i].copy() @@ -241,7 +258,9 @@ def twist(val, geo, nRefAxPts=nRefAxPts): DVGeo_real.setDesignVars(dvDict_real) ptsNewMinus = DVGeo_real.update(ptSetName) - funcSensFD[x][i, :] = (ptsNewPlus.flatten() - ptsNewMinus.flatten()) / (2 * stepSize_FD) + funcSensFD[x][i, disp[comm.rank] * 3 : disp[comm.rank + 1] * 3] = ( + ptsNewPlus.flatten() - ptsNewMinus.flatten() + ) / (2 * stepSize_FD) # Set the real DV back to the original value dvDict_real[x][i] = xRef_real.copy() @@ -251,13 +270,20 @@ def twist(val, geo, nRefAxPts=nRefAxPts): DVGeo_complex.setDesignVars(dvDict_complex) ptsNew = DVGeo_complex.update(ptSetName) - funcSensCS[x][i, :] = np.imag(ptsNew.flatten()) / stepSize_CS + funcSensCS[x][i, disp[comm.rank] * 3 : disp[comm.rank + 1] * 3] = ( + np.imag(ptsNew.flatten()) / stepSize_CS + ) # Set the complex DV back to the original value dvDict_complex[x][i] = xRef_complex.copy() # Check that the analytic derivatives are consistent with FD and CS for x in dvDict_real: + funcSensFD[x] = comm.allreduce(funcSensFD[x]) + funcSensCS[x] = comm.allreduce(funcSensCS[x]) + + print(max(max(funcSens[x].T - funcSensCS[x]))) + np.testing.assert_allclose(funcSens[x].T, funcSensFD[x], rtol=1e-4, atol=1e-10) np.testing.assert_allclose(funcSens[x].T, funcSensCS[x], rtol=1e-4, atol=1e-10) From 780ccaa564053e6e5c283c932f76d6b9e9092b6d Mon Sep 17 00:00:00 2001 From: sseraj Date: Mon, 25 Sep 2023 16:16:39 -0400 Subject: [PATCH 06/16] removed unnecessary allreduces --- pygeo/parameterization/DVGeoMulti.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pygeo/parameterization/DVGeoMulti.py b/pygeo/parameterization/DVGeoMulti.py index 424d871d..3dd2033d 100644 --- a/pygeo/parameterization/DVGeoMulti.py +++ b/pygeo/parameterization/DVGeoMulti.py @@ -2466,7 +2466,7 @@ def _projectToComponent_b(self, dIdpt, comp, projDict, surface=None): dIdptComp = dIdptComp[:, disp[comp.comm.rank] : disp[comp.comm.rank + 1], :] # Call the total sensitivity of the component's DVGeo - compSens = comp.DVGeo.totalSensitivity(dIdptComp, "triMesh", comm=comp.comm) + compSens = comp.DVGeo.totalSensitivity(dIdptComp, "triMesh") # the entries in dIdpt is replaced with AD seeds of initial points that were projected # we also return the total sensitivity contributions from components' triMeshes @@ -3206,11 +3206,11 @@ def _getIntersectionSeam_b(self, seamBar, comm): # get the total sensitivities from both components compSens_local = {} - compSensA = self.compA.DVGeo.totalSensitivity(coorAb, "triMesh", comm=self.compA.comm) + compSensA = self.compA.DVGeo.totalSensitivity(coorAb, "triMesh") for k, v in compSensA.items(): compSens_local[k] = v - compSensB = self.compB.DVGeo.totalSensitivity(coorBb, "triMesh", comm=self.compB.comm) + compSensB = self.compB.DVGeo.totalSensitivity(coorBb, "triMesh") for k, v in compSensB.items(): compSens_local[k] = v From db669a3ddf3045b73044f09991cd535410b67f03 Mon Sep 17 00:00:00 2001 From: sseraj Date: Mon, 25 Sep 2023 16:45:01 -0400 Subject: [PATCH 07/16] fixed intersection derivatives --- pygeo/parameterization/DVGeoMulti.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pygeo/parameterization/DVGeoMulti.py b/pygeo/parameterization/DVGeoMulti.py index 3dd2033d..8caede8d 100644 --- a/pygeo/parameterization/DVGeoMulti.py +++ b/pygeo/parameterization/DVGeoMulti.py @@ -3197,6 +3197,10 @@ def _getIntersectionSeam_b(self, seamBar, comm): coorAb[ii] += cAb.T coorBb[ii] += cBb.T + # Allreduce the derivative seeds + coorAb = comm.allreduce(coorAb) + coorBb = comm.allreduce(coorBb) + # Extract the entries of coorAb and coorBb that are for points on this processor disp = self.compA.triMeshData["disp"] coorAb = coorAb[:, disp[self.compA.comm.rank] : disp[self.compA.comm.rank + 1], :] From 2bd170af183e8d30180464a5b9fb434234d3f84e Mon Sep 17 00:00:00 2001 From: sseraj Date: Mon, 25 Sep 2023 18:17:29 -0400 Subject: [PATCH 08/16] fixed projection derivatives --- pygeo/parameterization/DVGeoMulti.py | 89 ++++++++++++---------------- 1 file changed, 39 insertions(+), 50 deletions(-) diff --git a/pygeo/parameterization/DVGeoMulti.py b/pygeo/parameterization/DVGeoMulti.py index 8caede8d..fdbdfd66 100644 --- a/pygeo/parameterization/DVGeoMulti.py +++ b/pygeo/parameterization/DVGeoMulti.py @@ -1945,7 +1945,7 @@ def project_b(self, ptSetName, dIdpt, comm): indAComp = self.projData[ptSetName]["compA"]["indAComp"] if indAComp: dIdptA = dIdpt[:, indAComp] - dIdpt[:, indAComp], compSensA = self._projectToComponent_b( + dIdpt[:, indAComp], dIdptTriA = self._projectToComponent_b( dIdptA, self.compA, self.projData[ptSetName]["compA"] ) @@ -1961,37 +1961,39 @@ def project_b(self, ptSetName, dIdpt, comm): dIdptA = dIdpt[:, indA] # call the projection routine with the info # this returns the projected points and we use the same mapping to put them back in place - dIdpt[:, indA], compSensA_temp = self._projectToComponent_b( + dIdpt[:, indA], dIdptTriA_temp = self._projectToComponent_b( dIdptA, self.compA, self.projData[ptSetName][surface], surface=surface ) - # Accumulate triangulated mesh sensitivities - for k, v in compSensA_temp.items(): - try: - compSensA[k] += v - except KeyError: - compSensA[k] = v + # Accumulate triangulated mesh seeds + try: + dIdptTriA += dIdptTriA_temp + except NameError: + dIdptTriA = dIdptTriA_temp - for k, v in compSensA.items(): - compSens_local[k] = v - - # set the compSens entries to all zeros on these procs + # Set the triangulated mesh seeds to all zeros on these procs else: - # get the values from each DVGeo - xA = self.compA.DVGeo.getValues() + dIdptTriA = np.zeros(self.compA.nodes.shape) + + # Allreduce the triangulated mesh seeds + dIdptTriA = self.comm.allreduce(dIdptTriA) + + # Extract the entries of dIdptTri that are for points on this processor + disp = self.compA.triMeshData["disp"] + dIdptTriA = dIdptTriA[:, disp[self.compA.comm.rank] : disp[self.compA.comm.rank + 1], :] - # loop over each entry in xA and xB and create a dummy zero gradient array for all - for k, v in xA.items(): - # create the zero array: - zeroSens = np.zeros((N, v.shape[0])) - compSens_local[k] = zeroSens + # Call the total sensitivity of the component's DVGeo + compSensA = self.compA.DVGeo.totalSensitivity(dIdptTriA, "triMesh") + + for k, v in compSensA.items(): + compSens_local[k] = v # do the same for B if flagB: indBComp = self.projData[ptSetName]["compB"]["indBComp"] if indBComp: dIdptB = dIdpt[:, indBComp] - dIdpt[:, indBComp], compSensB = self._projectToComponent_b( + dIdpt[:, indBComp], dIdptTriB = self._projectToComponent_b( dIdptB, self.compB, self.projData[ptSetName]["compB"] ) @@ -2000,29 +2002,23 @@ def project_b(self, ptSetName, dIdpt, comm): surfaceInd = surfaceIndB[surface] indB = [self.projData[ptSetName]["compB"]["ind"][i] for i in surfaceInd] dIdptB = dIdpt[:, indB] - dIdpt[:, indB], compSensB_temp = self._projectToComponent_b( + dIdpt[:, indB], dIdptTriB_temp = self._projectToComponent_b( dIdptB, self.compB, self.projData[ptSetName][surface], surface=surface ) - for k, v in compSensB_temp.items(): - try: - compSensB[k] += v - except KeyError: - compSensB[k] = v - - for k, v in compSensB.items(): - compSens_local[k] = v - - # set the compSens entries to all zeros on these procs + try: + dIdptTriB += dIdptTriB_temp + except NameError: + dIdptTriB = dIdptTriB_temp else: - # get the values from each DVGeo - xB = self.compB.DVGeo.getValues() + dIdptTriB = np.zeros(self.compB.nodes.shape) - # loop over each entry in xA and xB and create a dummy zero gradient array for all - for k, v in xB.items(): - # create the zero array: - zeroSens = np.zeros((N, v.shape[0])) - compSens_local[k] = zeroSens + dIdptTriB = self.comm.allreduce(dIdptTriB) + disp = self.compB.triMeshData["disp"] + dIdptTriB = dIdptTriB[:, disp[self.compB.comm.rank] : disp[self.compB.comm.rank + 1], :] + compSensB = self.compB.DVGeo.totalSensitivity(dIdptTriB, "triMesh") + for k, v in compSensB.items(): + compSens_local[k] = v # finally sum the results across procs if we are provided with a comm if comm: @@ -2418,7 +2414,7 @@ def _projectToComponent_b(self, dIdpt, comp, projDict, surface=None): normProjb = np.zeros_like(normProjNotNorm) # also create the dIdtp for the triangulated surface nodes - dIdptComp = np.zeros((dIdpt.shape[0], comp.nodes.shape[0], 3)) + dIdptTri = np.zeros((dIdpt.shape[0], comp.nodes.shape[0], 3)) # now propagate the ad seeds back for each function for i in range(dIdpt.shape[0]): @@ -2456,21 +2452,14 @@ def _projectToComponent_b(self, dIdpt, comp, projDict, surface=None): # Put the reverse ad seed back into dIdpt dIdpt[i] = xyzb # Also save the triangulated surface node seeds - dIdptComp[i] = coorb + dIdptTri[i] = coorb # Now we are done with the ADT self.adtAPI.adtdeallocateadts(adtID) - # Extract the entries of dIdptComp that are for points on this processor - disp = comp.triMeshData["disp"] - dIdptComp = dIdptComp[:, disp[comp.comm.rank] : disp[comp.comm.rank + 1], :] - - # Call the total sensitivity of the component's DVGeo - compSens = comp.DVGeo.totalSensitivity(dIdptComp, "triMesh") - - # the entries in dIdpt is replaced with AD seeds of initial points that were projected - # we also return the total sensitivity contributions from components' triMeshes - return dIdpt, compSens + # The entries in dIdpt are replaced with AD seeds of initial points that were projected + # We also return the total sensitivity contributions from component's triangulated mesh + return dIdpt, dIdptTri def _getUpdatedCoords(self): # this code returns the updated coordinates From 7e614e1780f75fb2522995d91c1970699908794e Mon Sep 17 00:00:00 2001 From: sseraj Date: Tue, 26 Sep 2023 17:50:12 -0400 Subject: [PATCH 09/16] corrected comment on _projectToComponent_b returns --- pygeo/parameterization/DVGeoMulti.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygeo/parameterization/DVGeoMulti.py b/pygeo/parameterization/DVGeoMulti.py index 5c5a5546..13e15f6f 100644 --- a/pygeo/parameterization/DVGeoMulti.py +++ b/pygeo/parameterization/DVGeoMulti.py @@ -2487,7 +2487,7 @@ def _projectToComponent_b(self, dIdpt, comp, projDict, surface=None): self.adtAPI.adtdeallocateadts(adtID) # The entries in dIdpt are replaced with AD seeds of initial points that were projected - # We also return the total sensitivity contributions from component's triangulated mesh + # We also return the seeds for the component's triangulated mesh in dIdptTri return dIdpt, dIdptTri def _getUpdatedCoords(self): From 0c6eae1dfb61491e1c3932536376a1f74b2a1511 Mon Sep 17 00:00:00 2001 From: sseraj Date: Thu, 28 Sep 2023 14:56:46 -0400 Subject: [PATCH 10/16] slight renaming --- pygeo/parameterization/DVGeoMulti.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pygeo/parameterization/DVGeoMulti.py b/pygeo/parameterization/DVGeoMulti.py index 13e15f6f..6a757398 100644 --- a/pygeo/parameterization/DVGeoMulti.py +++ b/pygeo/parameterization/DVGeoMulti.py @@ -978,14 +978,14 @@ def updateTriMesh(self): mpiType = MPI.DOUBLE_COMPLEX # Create the receiving buffer - newPtsGlobal = np.zeros(nPts * 3, dtype=procNodes.dtype) - recvbuf = [newPtsGlobal, sizes * 3, disp[0:-1] * 3, mpiType] + globalNodes = np.zeros(nPts * 3, dtype=procNodes.dtype) + recvbuf = [globalNodes, sizes * 3, disp[0:-1] * 3, mpiType] # Allgather the updated coordinates self.comm.Allgatherv(sendbuf, recvbuf) # Reshape into a nPts, 3 array - self.nodes = newPtsGlobal.reshape((nPts, 3)) + self.nodes = globalNodes.reshape((nPts, 3)) class PointSet: From d74425530f1af817fbb756a84687eb91611f41b8 Mon Sep 17 00:00:00 2001 From: sseraj Date: Thu, 28 Sep 2023 14:58:25 -0400 Subject: [PATCH 11/16] finished parallelizing the test --- tests/reg_tests/ref/test_DVGeometryMulti.ref | 238 +++++++++---------- tests/reg_tests/test_DVGeometryMulti.py | 42 +++- 2 files changed, 153 insertions(+), 127 deletions(-) diff --git a/tests/reg_tests/ref/test_DVGeometryMulti.ref b/tests/reg_tests/ref/test_DVGeometryMulti.ref index d97797b7..42eff81e 100644 --- a/tests/reg_tests/ref/test_DVGeometryMulti.ref +++ b/tests/reg_tests/ref/test_DVGeometryMulti.ref @@ -1,123 +1,121 @@ { - "ptsUpdated": [ - { - "__ndarray__": [ - [ - 0.00015229324522606294, - -0.008724874175625251, - 0.0 - ], - [ - 0.49680087946893736, - 0.09994415914341713, - -0.5000000000000001 - ], - [ - 0.49999999999999994, - 1.734723475976807e-18, - 1.9999999999999993 - ], - [ - 0.3748051686152182, - 0.12479824619498066, - 1.9999999999999996 - ], - [ - 2.4999999999999996, - 0.4999999999999999, - 0.0 - ], - [ - 0.300010152883015, - -0.000581658278375018, - 0.5000000000000001 - ], - [ - 0.7558449426479021, - -0.2439473847057298, - 0.5999997165630315 - ], - [ - 0.2560526152942702, - -0.2558449426479023, - 0.5999997165630314 - ], - [ - 0.244155058456546, - 0.2439473858494195, - 0.5999999999999999 - ], - [ - 0.24972186196458135, - 0.09926785071052377, - 0.5000000000000001 - ], - [ - 0.5053277931846962, - -0.2999322798665185, - 0.49999999999999994 - ], - [ - 0.24708313975314067, - 0.1136576670970726, - 0.5099996220283098 - ], - [ - 0.4991275125824375, - 0.2999847706754774, - 0.5 - ], - [ - 0.36910314030476443, - 0.24692177477292812, - 0.6 - ], - [ - 0.24928270996440818, - 0.25026018528384725, - 0.5000000000000001 - ], - [ - 0.49927001886063926, - 0.250987258131816, - 0.5 - ], - [ - 0.5034043514208585, - 0.2501560711472582, - 0.399999734349053 - ], - [ - 0.7439510052288243, - 0.25569291087605306, - 0.6000000785150911 - ], - [ - 0.4940512221529827, - 0.2498961636964368, - 0.5999999999999999 - ], - [ - 0.24999999999999997, - 0.4999999999999999, - 0.6 - ], - [ - 0.5058672786729258, - -0.24989810383051725, - 0.5999997144892112 - ], - [ - 0.24999999999999997, - -0.49999999999999994, - 0.5999999999999999 - ] - ], - "dtype": "float64", - "shape": [ - 22, - 3 + "ptsUpdated": { + "__ndarray__": [ + [ + 0.000152293245226056, + -0.008724874175625282, + 0.0 + ], + [ + 0.49680087946893736, + 0.09994415914341709, + -0.4999999999999999 + ], + [ + 0.5, + 1.734723475976807e-18, + 1.9999999999999998 + ], + [ + 0.37480516861521823, + 0.1247982461949807, + 2.0 + ], + [ + 2.5, + 0.5000000000000001, + 0.0 + ], + [ + 0.300010152883015, + -0.000581658278375018, + 0.49999999999999994 + ], + [ + 0.7558449426479023, + -0.24394738470572977, + 0.5999997165630315 + ], + [ + 0.2560526152942702, + -0.2558449426479023, + 0.5999997165630316 + ], + [ + 0.244155058456546, + 0.24394738584941955, + 0.5999999999999999 + ], + [ + 0.2497218619645813, + 0.09926785071052376, + 0.4999999999999999 + ], + [ + 0.5053277931846963, + -0.29993227986651844, + 0.49999999999999994 + ], + [ + 0.24708313975314072, + 0.11365766709707267, + 0.5099996220283096 + ], + [ + 0.4991275125824374, + 0.2999847706754774, + 0.49999999999999994 + ], + [ + 0.36910314030476443, + 0.2469217747729282, + 0.6 + ], + [ + 0.24928270996440818, + 0.25026018528384725, + 0.4999999999999998 + ], + [ + 0.4992700188606393, + 0.25098725813181605, + 0.4999999999999999 + ], + [ + 0.5034043514208585, + 0.25015607114725813, + 0.399999734349053 + ], + [ + 0.7439510052288242, + 0.25569291087605306, + 0.6000000785150912 + ], + [ + 0.4940512221529827, + 0.24989616369643675, + 0.6000000000000001 + ], + [ + 0.24999999999999992, + 0.49999999999999994, + 0.5999999999999999 + ], + [ + 0.5058672786729258, + -0.24989810383051722, + 0.5999997144892111 + ], + [ + 0.24999999999999997, + -0.49999999999999983, + 0.5999999999999999 ] - } - ] + ], + "dtype": "float64", + "shape": [ + 22, + 3 + ] + } } \ No newline at end of file diff --git a/tests/reg_tests/test_DVGeometryMulti.py b/tests/reg_tests/test_DVGeometryMulti.py index 4cdcd549..9ad7d14c 100644 --- a/tests/reg_tests/test_DVGeometryMulti.py +++ b/tests/reg_tests/test_DVGeometryMulti.py @@ -7,6 +7,7 @@ from baseclasses.utils import Error from mpi4py import MPI import numpy as np +from parameterized import parameterized_class # First party modules from pygeo import DVGeometry @@ -26,11 +27,22 @@ baseDir = os.path.dirname(os.path.abspath(__file__)) inputDir = os.path.join(baseDir, "../../input_files") +# Run the boxes test in series and in parallel +test_params = [ + { + "name": "one_proc", + "N_PROCS": 1, + }, + { + "name": "two_procs", + "N_PROCS": 2, + }, +] + +@parameterized_class(test_params) @unittest.skipUnless(pysurfInstalled, "requires pySurf") class TestDVGeoMulti(unittest.TestCase): - N_PROCS = 1 - def train_boxes(self, train=True): self.test_boxes(train=train) @@ -142,7 +154,7 @@ def test_boxes(self, train=False): [0.5, 0.25, 0.6], [0.25, 0.5, 0.6], [0.5, -0.25, 0.6], - # [0.25, -0.5, 0.6], + [0.25, -0.5, 0.6], ] ) @@ -210,10 +222,24 @@ def twist(val, geo, nRefAxPts=nRefAxPts): # Update the point set ptsUpdated = DVGeo.update(ptSetName) + # Create the send buffer + procPoints = ptsUpdated.flatten() + sendbuf = [procPoints, sizes[comm.rank] * 3] + + # Create the receiving buffer + globalPoints = np.zeros(nPtsGlobal * 3) + recvbuf = [globalPoints, sizes * 3, disp[0:-1] * 3, MPI.DOUBLE] + + # Allgather the updated coordinates + comm.Allgatherv(sendbuf, recvbuf) + + # Reshape into a nPtsGlobal, 3 array + ptsUpdated = globalPoints.reshape((nPtsGlobal, 3)) + # Regression test the updated points for the real DVGeo refFile = os.path.join(baseDir, "ref/test_DVGeometryMulti.ref") - # with BaseRegTest(refFile, train=train) as handler: - # handler.par_add_val("ptsUpdated", ptsUpdated, tol=1e-14) + with BaseRegTest(refFile, train=train) as handler: + handler.root_add_val("ptsUpdated", ptsUpdated, tol=1e-14) # Now we will test the derivatives @@ -282,8 +308,6 @@ def twist(val, geo, nRefAxPts=nRefAxPts): funcSensFD[x] = comm.allreduce(funcSensFD[x]) funcSensCS[x] = comm.allreduce(funcSensCS[x]) - print(max(max(funcSens[x].T - funcSensCS[x]))) - np.testing.assert_allclose(funcSens[x].T, funcSensFD[x], rtol=1e-4, atol=1e-10) np.testing.assert_allclose(funcSens[x].T, funcSensCS[x], rtol=1e-4, atol=1e-10) @@ -291,6 +315,10 @@ def twist(val, geo, nRefAxPts=nRefAxPts): with self.assertRaises(Error): DVGeo.addPointSet(np.array([[-1.0, 0.0, 0.0]]), "test_error") + +class TestDVGeoMultiEdgeCases(unittest.TestCase): + N_PROCS = 1 + def test_trackSurfaces_shared_points(self): """ Tests that points shared between two tracked surfaces are handled properly From d05387331593bc034248072269382ecb7d38753d Mon Sep 17 00:00:00 2001 From: sseraj Date: Fri, 29 Sep 2023 10:37:44 -0400 Subject: [PATCH 12/16] cleaned up comm usage a bit --- pygeo/parameterization/DVGeoMulti.py | 38 +++++++++++++--------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/pygeo/parameterization/DVGeoMulti.py b/pygeo/parameterization/DVGeoMulti.py index 6a757398..a797f455 100644 --- a/pygeo/parameterization/DVGeoMulti.py +++ b/pygeo/parameterization/DVGeoMulti.py @@ -36,6 +36,7 @@ class DVGeometryMulti: ---------- comm : MPI.IntraComm, optional The communicator associated with this geometry object. + This is also used to parallelize the triangulated meshes. checkDVs : bool, optional Flag to check whether there are duplicate DV names in or across components. @@ -164,9 +165,7 @@ def addComponent(self, comp, DVGeo, triMesh=None, scale=1.0, bbox=None, pointSet xMax[2] = bbox["zmax"] # initialize the component object - self.comps[comp] = component( - self.comm, comp, DVGeo, nodes, triConn, triConnStack, barsConn, xMin, xMax, triMeshData - ) + self.comps[comp] = component(comp, DVGeo, nodes, triConn, triConnStack, barsConn, xMin, xMax, triMeshData) # add the name to the list self.compNames.append(comp) @@ -325,9 +324,7 @@ def addPointSet(self, points, ptName, compNames=None, comm=None, applyIC=False, To ease bookkeepping, an empty point set with ptName will be added to components not in this list. If a list is not provided, this point set is added to all components. comm : MPI.IntraComm, optional - Comm that is associated with the added point set. Does not - work now, just added to be consistent with the API of - other DVGeo types. + The communicator that is associated with the added point set. applyIC : bool, optional Flag to specify whether this point set will follow the updated intersection curve(s). This is typically only needed for the CFD surface mesh. @@ -935,9 +932,8 @@ def _computeTotalJacobian(self, ptSetName): class component: - def __init__(self, comm, name, DVGeo, nodes, triConn, triConnStack, barsConn, xMin, xMax, triMeshData): + def __init__(self, name, DVGeo, nodes, triConn, triConnStack, barsConn, xMin, xMax, triMeshData): # save the info - self.comm = comm self.name = name self.DVGeo = DVGeo self.nodes = nodes @@ -957,7 +953,7 @@ def __init__(self, comm, name, DVGeo, nodes, triConn, triConnStack, barsConn, xM else: self.triMesh = True - def updateTriMesh(self): + def updateTriMesh(self, comm): # We need the full triangulated surface for this component # Get the stored processor splitting information sizes = self.triMeshData["sizes"] @@ -969,7 +965,7 @@ def updateTriMesh(self): # Create the send buffer procNodes = procNodes.flatten() - sendbuf = [procNodes, sizes[self.comm.rank] * 3] + sendbuf = [procNodes, sizes[comm.rank] * 3] # Set the appropriate type for the receiving buffer if procNodes.dtype == float: @@ -982,7 +978,7 @@ def updateTriMesh(self): recvbuf = [globalNodes, sizes * 3, disp[0:-1] * 3, mpiType] # Allgather the updated coordinates - self.comm.Allgatherv(sendbuf, recvbuf) + comm.Allgatherv(sendbuf, recvbuf) # Reshape into a nPts, 3 array self.nodes = globalNodes.reshape((nPts, 3)) @@ -1212,7 +1208,7 @@ def setSurface(self, comm): """This set the new udpated surface on which we need to compute the new intersection curve""" # get the updated surface coordinates - self._getUpdatedCoords() + self._getUpdatedCoords(comm) self.seam = self._getIntersectionSeam(comm) @@ -2010,7 +2006,7 @@ def project_b(self, ptSetName, dIdpt, comm): # Extract the entries of dIdptTri that are for points on this processor disp = self.compA.triMeshData["disp"] - dIdptTriA = dIdptTriA[:, disp[self.compA.comm.rank] : disp[self.compA.comm.rank + 1], :] + dIdptTriA = dIdptTriA[:, disp[self.comm.rank] : disp[self.comm.rank + 1], :] # Call the total sensitivity of the component's DVGeo compSensA = self.compA.DVGeo.totalSensitivity(dIdptTriA, "triMesh") @@ -2044,7 +2040,7 @@ def project_b(self, ptSetName, dIdpt, comm): dIdptTriB = self.comm.allreduce(dIdptTriB) disp = self.compB.triMeshData["disp"] - dIdptTriB = dIdptTriB[:, disp[self.compB.comm.rank] : disp[self.compB.comm.rank + 1], :] + dIdptTriB = dIdptTriB[:, disp[self.comm.rank] : disp[self.comm.rank + 1], :] compSensB = self.compB.DVGeo.totalSensitivity(dIdptTriB, "triMesh") for k, v in compSensB.items(): compSens_local[k] = v @@ -2490,14 +2486,14 @@ def _projectToComponent_b(self, dIdpt, comp, projDict, surface=None): # We also return the seeds for the component's triangulated mesh in dIdptTri return dIdpt, dIdptTri - def _getUpdatedCoords(self): + def _getUpdatedCoords(self, comm): # this code returns the updated coordinates # first comp a - self.compA.updateTriMesh() + self.compA.updateTriMesh(comm) # then comp b - self.compB.updateTriMesh() + self.compB.updateTriMesh(comm) return @@ -3216,15 +3212,15 @@ def _getIntersectionSeam_b(self, seamBar, comm): coorBb[ii] += cBb.T # Allreduce the derivative seeds - coorAb = comm.allreduce(coorAb) - coorBb = comm.allreduce(coorBb) + coorAb = self.comm.allreduce(coorAb) + coorBb = self.comm.allreduce(coorBb) # Extract the entries of coorAb and coorBb that are for points on this processor disp = self.compA.triMeshData["disp"] - coorAb = coorAb[:, disp[self.compA.comm.rank] : disp[self.compA.comm.rank + 1], :] + coorAb = coorAb[:, disp[self.comm.rank] : disp[self.comm.rank + 1], :] disp = self.compB.triMeshData["disp"] - coorBb = coorBb[:, disp[self.compB.comm.rank] : disp[self.compB.comm.rank + 1], :] + coorBb = coorBb[:, disp[self.comm.rank] : disp[self.comm.rank + 1], :] # get the total sensitivities from both components compSens_local = {} From 723cec50d619a2c8994781f82b7457a0d56b495b Mon Sep 17 00:00:00 2001 From: sseraj Date: Mon, 2 Oct 2023 10:18:14 -0400 Subject: [PATCH 13/16] keep the old ref values --- tests/reg_tests/ref/test_DVGeometryMulti.ref | 84 ++++++++++---------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/tests/reg_tests/ref/test_DVGeometryMulti.ref b/tests/reg_tests/ref/test_DVGeometryMulti.ref index 42eff81e..1734753e 100644 --- a/tests/reg_tests/ref/test_DVGeometryMulti.ref +++ b/tests/reg_tests/ref/test_DVGeometryMulti.ref @@ -2,113 +2,113 @@ "ptsUpdated": { "__ndarray__": [ [ - 0.000152293245226056, - -0.008724874175625282, + 0.00015229324522606294, + -0.008724874175625251, 0.0 ], [ 0.49680087946893736, - 0.09994415914341709, - -0.4999999999999999 + 0.09994415914341713, + -0.5000000000000001 ], [ - 0.5, + 0.49999999999999994, 1.734723475976807e-18, - 1.9999999999999998 + 1.9999999999999993 ], [ - 0.37480516861521823, - 0.1247982461949807, - 2.0 + 0.3748051686152182, + 0.12479824619498066, + 1.9999999999999996 ], [ - 2.5, - 0.5000000000000001, + 2.4999999999999996, + 0.4999999999999999, 0.0 ], [ 0.300010152883015, -0.000581658278375018, - 0.49999999999999994 + 0.5000000000000001 ], [ - 0.7558449426479023, - -0.24394738470572977, + 0.7558449426479021, + -0.2439473847057298, 0.5999997165630315 ], [ 0.2560526152942702, -0.2558449426479023, - 0.5999997165630316 + 0.5999997165630314 ], [ 0.244155058456546, - 0.24394738584941955, + 0.2439473858494195, 0.5999999999999999 ], [ - 0.2497218619645813, - 0.09926785071052376, - 0.4999999999999999 + 0.24972186196458135, + 0.09926785071052377, + 0.5000000000000001 ], [ - 0.5053277931846963, - -0.29993227986651844, + 0.5053277931846962, + -0.2999322798665185, 0.49999999999999994 ], [ - 0.24708313975314072, - 0.11365766709707267, - 0.5099996220283096 + 0.24708313975314067, + 0.1136576670970726, + 0.5099996220283098 ], [ - 0.4991275125824374, + 0.4991275125824375, 0.2999847706754774, - 0.49999999999999994 + 0.5 ], [ 0.36910314030476443, - 0.2469217747729282, + 0.24692177477292812, 0.6 ], [ 0.24928270996440818, 0.25026018528384725, - 0.4999999999999998 + 0.5000000000000001 ], [ - 0.4992700188606393, - 0.25098725813181605, - 0.4999999999999999 + 0.49927001886063926, + 0.250987258131816, + 0.5 ], [ 0.5034043514208585, - 0.25015607114725813, + 0.2501560711472582, 0.399999734349053 ], [ - 0.7439510052288242, + 0.7439510052288243, 0.25569291087605306, - 0.6000000785150912 + 0.6000000785150911 ], [ 0.4940512221529827, - 0.24989616369643675, - 0.6000000000000001 + 0.2498961636964368, + 0.5999999999999999 ], [ - 0.24999999999999992, - 0.49999999999999994, - 0.5999999999999999 + 0.24999999999999997, + 0.4999999999999999, + 0.6 ], [ 0.5058672786729258, - -0.24989810383051722, - 0.5999997144892111 + -0.24989810383051725, + 0.5999997144892112 ], [ 0.24999999999999997, - -0.49999999999999983, + -0.49999999999999994, 0.5999999999999999 ] ], From 65eaffbb7b364f09cba91e082e61fd18a44f5cdb Mon Sep 17 00:00:00 2001 From: sseraj Date: Mon, 2 Oct 2023 13:40:53 -0400 Subject: [PATCH 14/16] use a consistent complex type --- pygeo/parameterization/DVGeoMulti.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pygeo/parameterization/DVGeoMulti.py b/pygeo/parameterization/DVGeoMulti.py index a797f455..7d9acfd5 100644 --- a/pygeo/parameterization/DVGeoMulti.py +++ b/pygeo/parameterization/DVGeoMulti.py @@ -1047,13 +1047,13 @@ def __init__( self.curveSearchAPI = curveSearchAPI.curvesearchapi self.intersectionAPI = intersectionAPI.intersectionapi self.utilitiesAPI = utilitiesAPI.utilitiesapi - self.mpi_type = MPI.DOUBLE + self.mpiType = MPI.DOUBLE elif dtype == complex: self.adtAPI = adtAPI_cs.adtapi self.curveSearchAPI = curveSearchAPI_cs.curvesearchapi self.intersectionAPI = intersectionAPI_cs.intersectionapi self.utilitiesAPI = utilitiesAPI_cs.utilitiesapi - self.mpi_type = MPI.C_DOUBLE_COMPLEX + self.mpiType = MPI.DOUBLE_COMPLEX # tolerance used for each curve when mapping nodes to curves self.curveEpsDict = {} @@ -1858,7 +1858,7 @@ def project(self, ptSetName, newPts): nptsg = self.nCurvePts[ptSetName][curveName] deltaGlobal = np.zeros(nptsg * 3, dtype=self.dtype) - recvbuf = [deltaGlobal, sizes * 3, disp * 3, self.mpi_type] + recvbuf = [deltaGlobal, sizes * 3, disp * 3, self.mpiType] # do an allgatherv comm.Allgatherv(sendbuf, recvbuf) @@ -2241,7 +2241,7 @@ def _commCurveProj(self, pts, indices, comm): # recvbuf ptsGlobal = np.zeros(3 * nptsg, dtype=self.dtype) - recvbuf = [ptsGlobal, sizes * 3, disp * 3, self.mpi_type] + recvbuf = [ptsGlobal, sizes * 3, disp * 3, self.mpiType] # do an allgatherv comm.Allgatherv(sendbuf, recvbuf) From bc3e55f29b0ba886a816696c2e4a72edf73e6d18 Mon Sep 17 00:00:00 2001 From: sseraj Date: Mon, 2 Oct 2023 13:43:29 -0400 Subject: [PATCH 15/16] use three procs for more code coverage --- tests/reg_tests/test_DVGeometryMulti.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/reg_tests/test_DVGeometryMulti.py b/tests/reg_tests/test_DVGeometryMulti.py index 9ad7d14c..f89b6836 100644 --- a/tests/reg_tests/test_DVGeometryMulti.py +++ b/tests/reg_tests/test_DVGeometryMulti.py @@ -34,8 +34,8 @@ "N_PROCS": 1, }, { - "name": "two_procs", - "N_PROCS": 2, + "name": "three_procs", + "N_PROCS": 3, }, ] From be63b715ce66edf9de4f6219853e4d7a8b4eca1b Mon Sep 17 00:00:00 2001 From: sseraj Date: Mon, 2 Oct 2023 14:16:37 -0400 Subject: [PATCH 16/16] fixed test skipping --- tests/reg_tests/test_DVGeometryMulti.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/reg_tests/test_DVGeometryMulti.py b/tests/reg_tests/test_DVGeometryMulti.py index f89b6836..60424738 100644 --- a/tests/reg_tests/test_DVGeometryMulti.py +++ b/tests/reg_tests/test_DVGeometryMulti.py @@ -316,6 +316,7 @@ def twist(val, geo, nRefAxPts=nRefAxPts): DVGeo.addPointSet(np.array([[-1.0, 0.0, 0.0]]), "test_error") +@unittest.skipUnless(pysurfInstalled, "requires pySurf") class TestDVGeoMultiEdgeCases(unittest.TestCase): N_PROCS = 1