Document detail triangle edge flags

DebugUtils/Source/DetourDebugDraw.cpp

@@ -101,7 +101,9 @@ static void drawPolyBoundaries(duDebugDraw* dd, const dtMeshTile* tile,
 				}
 				for (int m = 0, n = 2; m < 3; n=m++)
 				{
-					if (((t[3] >> (n*2)) & 0x3) == 0) continue;	// Skip inner detail edges.
+					if ((dtGetDetailTriEdgeFlags(t[3], n) & DT_DETAIL_EDGE_BOUNDARY) == 0)
+						continue;
+
 					if (distancePtLine2d(tv[n],v0,v1) < thr &&
 						distancePtLine2d(tv[m],v0,v1) < thr)
 					{

Detour/Include/DetourNavMesh.h

@@ -130,6 +130,11 @@ enum dtRaycastOptions
 	DT_RAYCAST_USE_COSTS = 0x01,		///< Raycast should calculate movement cost along the ray and fill RaycastHit::cost
 };
 
+enum dtDetailTriEdgeFlags
+{
+	DT_DETAIL_EDGE_BOUNDARY = 0x01,		///< Detail triangle edge is part of the poly boundary
+};
+
 
 /// Limit raycasting during any angle pahfinding
 /// The limit is given as a multiple of the character radius
@@ -287,7 +292,8 @@ struct dtMeshTile
 	/// The detail mesh's unique vertices. [(x, y, z) * dtMeshHeader::detailVertCount]
 	float* detailVerts;	
 
-	/// The detail mesh's triangles. [(vertA, vertB, vertC) * dtMeshHeader::detailTriCount]
+	/// The detail mesh's triangles. [(vertA, vertB, vertC, triFlags) * dtMeshHeader::detailTriCount].
+	/// See dtDetailTriEdgeFlags and dtGetDetailTriEdgeFlags.
 	unsigned char* detailTris;	
 
 	/// The tile bounding volume nodes. [Size: dtMeshHeader::bvNodeCount]
@@ -305,6 +311,15 @@ struct dtMeshTile
 	dtMeshTile& operator=(const dtMeshTile&);
 };
 
+/// Get flags for edge in detail triangle.
+/// @param	triFlags[in]		The flags for the triangle (last component of detail vertices above).
+/// @param	edgeIndex[in]		The index of the first vertex of the edge. For instance, if 0,
+///								returns flags for edge AB.
+inline int dtGetDetailTriEdgeFlags(unsigned char triFlags, int edgeIndex)
+{
+	return (triFlags >> (edgeIndex * 2)) & 0x3;
+}
+
 /// Configuration parameters used to define multi-tile navigation meshes.
 /// The values are used to allocate space during the initialization of a navigation mesh.
 /// @see dtNavMesh::init()

Recast/Source/RecastMeshDetail.cpp

@@ -1141,7 +1141,8 @@ static void getHeightData(rcContext* ctx, const rcCompactHeightfield& chf,
 static unsigned char getEdgeFlags(const float* va, const float* vb,
 								  const float* vpoly, const int npoly)
 {
-	// Return true if edge (va,vb) is part of the polygon.
+	// The flag returned by this function matches dtDetailTriEdgeFlags in Detour.
+	// Figure out if edge (va,vb) is part of the polygon boundary.
 	static const float thrSqr = rcSqr(0.001f);
 	for (int i = 0, j = npoly-1; i < npoly; j=i++)
 	{