Preliminary backface culling fix. Matrices which effectively change the polygon winding without changing the Z component of the polygon normal are tested for when appending items to the display list. If necessary, the polygon winding is changed on the fly to ensure visibility is correct. This code desperately needs to be optimized. Note: Not yet thoroughly tested.

Src/Graphics/Models.cpp

@@ -71,6 +71,63 @@
 #define VBO_VERTEX_SIZE							23	// total size (may include padding for alignment)
 
 
+/******************************************************************************
+ Math Routines
+******************************************************************************/
+
+// Macro to generate column-major (OpenGL) index from y,x subscripts
+#define CMINDEX(y,x)	(x*4+y)
+
+static void CrossProd(GLfloat out[3], GLfloat a[3], GLfloat b[3])
+{
+	out[0] = a[1]*b[2]-a[2]*b[1];
+	out[1] = a[2]*b[0]-a[0]*b[2];
+	out[2] = a[0]*b[1]-a[1]*b[0];
+}
+
+// 3x3 matrix used (upper-left of m[])
+static void MultMat3Vec3(GLfloat out[3], GLfloat m[4*4], GLfloat v[3])
+{
+	out[0] = m[CMINDEX(0,0)]*v[0]+m[CMINDEX(0,1)]*v[1]+m[CMINDEX(0,2)]*v[2];
+	out[1] = m[CMINDEX(1,0)]*v[0]+m[CMINDEX(1,1)]*v[1]+m[CMINDEX(1,2)]*v[2];
+	out[2] = m[CMINDEX(2,0)]*v[0]+m[CMINDEX(2,1)]*v[1]+m[CMINDEX(2,2)]*v[2];
+}
+
+static GLfloat Sign(GLfloat x)
+{
+	if (x > 0.0f)
+		return 1.0f;
+	else if (x < 0.0f)
+		return -1.0f;
+	return 0.0f;
+}
+
+// Inverts and transposes a 3x3 matrix (upper-left of the 4x4), returning a 
+// 4x4 matrix with the extra components undefined (do not use them!)
+static void InvertTransposeMat3(GLfloat out[4*4], GLfloat m[4*4])
+{
+	GLfloat	invDet;
+	GLfloat	a00 = m[CMINDEX(0,0)], a01 = m[CMINDEX(0,1)], a02 = m[CMINDEX(0,2)];
+	GLfloat	a10 = m[CMINDEX(1,0)], a11 = m[CMINDEX(1,1)], a12 = m[CMINDEX(1,2)];
+	GLfloat	a20 = m[CMINDEX(2,0)], a21 = m[CMINDEX(2,1)], a22 = m[CMINDEX(2,2)];
+	
+	invDet = 1.0f/(a00*(a22*a11-a21*a12)-a10*(a22*a01-a21*a02)+a20*(a12*a01-a11*a02));
+	out[CMINDEX(0,0)] = invDet*(a22*a11-a21*a12);		out[CMINDEX(1,0)] = invDet*(-(a22*a01-a21*a02));	out[CMINDEX(2,0)] = invDet*(a12*a01-a11*a02);
+	out[CMINDEX(0,1)] = invDet*(-(a22*a10-a20*a12));	out[CMINDEX(1,1)] = invDet*(a22*a00-a20*a02);		out[CMINDEX(2,1)] = invDet*(-(a12*a00-a10*a02));
+	out[CMINDEX(0,2)] = invDet*(a21*a10-a20*a11);		out[CMINDEX(1,2)] = invDet*(-(a21*a00-a20*a01));	out[CMINDEX(2,2)] = invDet*(a11*a00-a10*a01);
+}
+
+static void PrintMatrix(GLfloat m[4*4])
+{
+	for (int i = 0; i < 3; i++)
+	{
+		for (int j = 0; j < 3; j++)
+			printf("%g\t", m[CMINDEX(i,j)]);
+		printf("\n");
+	}
+}
+
+
 /******************************************************************************
  Display Lists
  
@@ -136,8 +193,22 @@ void CRender3D::DrawDisplayList(ModelCache *Cache, POLY_STATE state)
        	}
        	else
        	{
+       		GLint	frontFace;
+       		
+       		if (D->Data.Model.frontFace == -GL_CW)	// no backface culling (all normals have lost their Z component)
+       			glDisable(GL_CULL_FACE);
+       		else									// use appropriate winding convention
+       		{
+       			glGetIntegerv(GL_FRONT_FACE, &frontFace);
+       			if (frontFace != D->Data.Model.frontFace)
+	       			glFrontFace(D->Data.Model.frontFace);
+	       	}
+       			
 			glUniformMatrix4fv(modelViewMatrixLoc, 1, GL_FALSE, D->Data.Model.modelViewMatrix);
 			glDrawArrays(GL_TRIANGLES, D->Data.Model.index, D->Data.Model.numVerts);
+			
+			if (D->Data.Model.frontFace == -GL_CW)
+       			glEnable(GL_CULL_FACE);
 		}
 		
 		D = D->next;
@@ -188,6 +259,55 @@ bool CRender3D::AppendDisplayList(ModelCache *Cache, bool isViewport, const stru
 			
 			// Copy modelview matrix
 			glGetFloatv(GL_MODELVIEW_MATRIX, Cache->List[lm].Data.Model.modelViewMatrix);
+			
+			/*
+			 * Determining if winding was reversed (but not polygon normal):
+			 *
+			 * Real3D performs backface culling in view space based on the
+			 * polygon normal unlike OpenGL, which uses the computed normal
+			 * from the edges (in screen space) of the polygon. Consequently,
+			 * it is possible to create a matrix that mirrors an axis without
+			 * rotating the normal, which in turn flips the polygon winding and
+			 * makes it invisible in OpenGL but not on Real3D, because the 
+			 * normal is still facing the right way.
+			 *
+			 * To detect such a situation, we create a fictitious polygon with
+			 * edges X = [1 0 0] and Y = [0 1 0], with normal Z = [0 0 1]. We 
+			 * rotate the edges by the matrix then compute a normal P, which is
+			 * what OpenGL would use for culling. We transform the normal Z by
+			 * the normal matrix (normals are special and must be multiplied by
+			 * Transpose(Inverse(M)), not M). If the Z components of P and the
+			 * transformed Z vector have opposite signs, the OpenGL winding 
+			 * mode must be switched in order to draw correctly. The X axis may
+			 * have been flipped, for example, changing the winding mode while
+			 * leaving the polygon normal unaffected. OpenGL would erroneously
+			 * discard these polygons, so we flip the winding convention, 
+			 * ensuring they are drawn correctly.
+			 *
+			 * We have to adjust the Z vector (fictitious normal) by the sign
+			 * of the Z axis specified by the coordinate system matrix (#0).
+			 * This is described further in InsertPolygon(), where the vertices
+			 * are ordered in clockwise fashion.
+			 */
+			GLfloat	x[3] = { 1.0f, 0.0f, 0.0f };
+			GLfloat	y[3] = { 0.0f, 1.0f, 0.0f };
+			GLfloat z[3] = { 0.0f, 0.0f, -1.0f*matrixBasePtr[0x5] };
+			GLfloat	m[4*4];
+			GLfloat	xT[3], yT[3], zT[3], pT[3];
+
+			InvertTransposeMat3(m,Cache->List[lm].Data.Model.modelViewMatrix);
+			MultMat3Vec3(xT,Cache->List[lm].Data.Model.modelViewMatrix,x);
+			MultMat3Vec3(yT,Cache->List[lm].Data.Model.modelViewMatrix,y);
+			MultMat3Vec3(zT,m,z);
+			CrossProd(pT,xT,yT);
+			
+			float s = Sign(zT[2]*pT[2]);
+			if (s < 0.0f)
+				Cache->List[lm].Data.Model.frontFace = GL_CCW;
+			else if (s > 0.0f)
+				Cache->List[lm].Data.Model.frontFace = GL_CW;
+			else
+				Cache->List[lm].Data.Model.frontFace = -GL_CW;
 		}
 		else	// nothing to do, continue loop
 			continue;
@@ -231,13 +351,6 @@ void CRender3D::ClearDisplayList(ModelCache *Cache)
  Vertices are copied in batches sorted by state when the model is complete.
 ******************************************************************************/
 
-static void CrossProd(GLfloat out[3], GLfloat a[3], GLfloat b[3])
-{
-	out[0] = a[1]*b[2]-a[2]*b[1];
-	out[1] = a[2]*b[0]-a[0]*b[2];
-	out[2] = a[0]*b[1]-a[1]*b[0];
-}
-
 // Inserts a vertex into the local vertex buffer, incrementing both the local and VBO pointers. The normal is scaled by normFlip.
 void CRender3D::InsertVertex(ModelCache *Cache, const Vertex *V, const Poly *P, float normFlip)
 {
@@ -388,10 +501,12 @@ bool CRender3D::InsertPolygon(ModelCache *Cache, const Poly *P)
 	 * clockwise and can be kept, otherwise it must be reversed.
 	 *
 	 * NOTE: This assumes that the Model 3 base coordinate system's Z axis
-	 * (into the screen) -1, like OpenGL's. For some games (eg., Lost World), 
+	 * (into the screen) is -1, like OpenGL. For some games (eg., Lost World),
 	 * this is not the case. Assuming games consistently use the same type of
 	 * coordinate system matrix, it seems that inverting the whole dot product
-	 * when Z is positive helps. I don't understand why...
+	 * when Z is positive helps. I don't understand exactly why... but it has
+	 * to do with using the correct Z convention to identify a vector pointing
+	 * toward or away from the screen.
 	 */
 	v1[0] = P->Vert[0].x-P->Vert[1].x;
 	v1[1] = P->Vert[0].y-P->Vert[1].y;

Src/Graphics/Render3D.cpp

@@ -28,6 +28,9 @@
  * Optimization To-Do List
  * -----------------------
  *
+ * 0. Optimize backface culling. Is it possible to compute normal matrix only
+ *    when needed? Should also be more careful about OpenGL state info, such as
+ *    the winding mode.
  * 1. Do not store matrices in a uniform, use glLoadMatrix() in MODELVIEW mode.
  *    It will no longer be necessary to compute normal matrix!
  * 2. Move stuff into vertex shader (vision by 2048? Subtract of 0.5,0.5 for bilinear filtering?)
@@ -1222,6 +1225,7 @@ void CRender3D::RenderFrame(void)
 		DrawDisplayList(&VROMCache, POLY_STATE_ALPHA);
 		DrawDisplayList(&PolyCache, POLY_STATE_ALPHA);
 	}
+	glFrontFace(GL_CW);	// restore front face
 	
 	// Disable VBO client states
 	glDisableVertexAttribArray(fogIntensityLoc);

Src/Graphics/Render3D.h

@@ -102,6 +102,7 @@ struct DisplayList
 			GLfloat		modelViewMatrix[4*4];	// model-view matrix
 			unsigned	index;					// index in VBO
 			unsigned	numVerts;				// number of vertices
+			GLint		frontFace;				// GL_CW (default), GL_CCW, or -GL_CW to indicate no culling
 		} Model;
 	} Data;