Make detail mesh edge detection more robust (recastnavigation#657)

Instead of using a distance check which can fail at large magnitudes due
to low precision we can check whether the edges are actually on the
hull.

Recast/Source/RecastMeshDetail.cpp

@@ -634,6 +634,40 @@ inline float getJitterY(const int i)
 	return (((i * 0xd8163841) & 0xffff) / 65535.0f * 2.0f) - 1.0f;
 }
 
+static bool onHull(int a, int b, int nhull, int* hull)
+{
+	// All internal sampled points come after the hull so we can early out for those.
+	if (a >= nhull || b >= nhull)
+		return false;
+
+	for (int j = nhull - 1, i = 0; i < nhull; j = i++)
+	{
+		if (a == hull[j] && b == hull[i])
+			return true;
+	}
+
+	return false;
+}
+
+// Find edges that lie on hull and mark them as such.
+static void setTriFlags(rcIntArray& tris, int nhull, int* hull)
+{
+	// Matches DT_DETAIL_EDGE_BOUNDARY
+	const int DETAIL_EDGE_BOUNDARY = 0x1;
+
+	for (int i = 0; i < tris.size(); i += 4)
+	{
+		int a = tris[i + 0];
+		int b = tris[i + 1];
+		int c = tris[i + 2];
+		unsigned short flags = 0;
+		flags |= (onHull(a, b, nhull, hull) ? DETAIL_EDGE_BOUNDARY : 0) << 0;
+		flags |= (onHull(b, c, nhull, hull) ? DETAIL_EDGE_BOUNDARY : 0) << 2;
+		flags |= (onHull(c, a, nhull, hull) ? DETAIL_EDGE_BOUNDARY : 0) << 4;
+		tris[i + 3] = (int)flags;
+	}
+}
+
 static bool buildPolyDetail(rcContext* ctx, const float* in, const int nin,
 							const float sampleDist, const float sampleMaxError,
 							const int heightSearchRadius, const rcCompactHeightfield& chf,
@@ -771,6 +805,7 @@ static bool buildPolyDetail(rcContext* ctx, const float* in, const int nin,
 	if (minExtent < sampleDist*2)
 	{
 		triangulateHull(nverts, verts, nhull, hull, nin, tris);
+		setTriFlags(tris, nhull, hull);
 		return true;
 	}
 
@@ -875,6 +910,8 @@ static bool buildPolyDetail(rcContext* ctx, const float* in, const int nin,
 		tris.resize(MAX_TRIS*4);
 		ctx->log(RC_LOG_ERROR, "rcBuildPolyMeshDetail: Shrinking triangle count from %d to max %d.", ntris, MAX_TRIS);
 	}
+
+	setTriFlags(tris, nhull, hull);
 
 	return true;
 }
@@ -1137,31 +1174,6 @@ static void getHeightData(rcContext* ctx, const rcCompactHeightfield& chf,
 	}
 }
 
-static unsigned char getEdgeFlags(const float* va, const float* vb,
-								  const float* vpoly, const int npoly)
-{
-	// The flag returned by this function matches dtDetailTriEdgeFlags in Detour.
-	// Figure out if edge (va,vb) is part of the polygon boundary.
-	static const float thrSqr = rcSqr(0.001f);
-	for (int i = 0, j = npoly-1; i < npoly; j=i++)
-	{
-		if (distancePtSeg2d(va, &vpoly[j*3], &vpoly[i*3]) < thrSqr &&
-			distancePtSeg2d(vb, &vpoly[j*3], &vpoly[i*3]) < thrSqr)
-			return 1;
-	}
-	return 0;
-}
-
-static unsigned char getTriFlags(const float* va, const float* vb, const float* vc,
-								 const float* vpoly, const int npoly)
-{
-	unsigned char flags = 0;
-	flags |= getEdgeFlags(va,vb,vpoly,npoly) << 0;
-	flags |= getEdgeFlags(vb,vc,vpoly,npoly) << 2;
-	flags |= getEdgeFlags(vc,va,vpoly,npoly) << 4;
-	return flags;
-}
-
 /// @par
 ///
 /// See the #rcConfig documentation for more information on the configuration parameters.
@@ -1377,7 +1389,7 @@ bool rcBuildPolyMeshDetail(rcContext* ctx, const rcPolyMesh& mesh, const rcCompa
 			dmesh.tris[dmesh.ntris*4+0] = (unsigned char)t[0];
 			dmesh.tris[dmesh.ntris*4+1] = (unsigned char)t[1];
 			dmesh.tris[dmesh.ntris*4+2] = (unsigned char)t[2];
-			dmesh.tris[dmesh.ntris*4+3] = getTriFlags(&verts[t[0]*3], &verts[t[1]*3], &verts[t[2]*3], poly, npoly);
+			dmesh.tris[dmesh.ntris*4+3] = (unsigned char)t[3];
 			dmesh.ntris++;
 		}
 	}