Modern hardware does backface culling in window space by calculating the face normal for the polygon, then doing a dot product against the view vector. The real3d pro-1000 on the other hand passes a pre-calculated face normal for each polygon which is used for culling. We were using this face normal to rewind the polygons so that regular backface culling would work. This worked 99.9% of the time. However this was failing on some models in Virtua Striker. The reason was because the pre-calculated face normals being passed were actually completely different to the actual face normals for the poly (not just inverted like you would expect). This broke our code. The solution was to emulate face culling directly in the vertex shader using the pre-calculated face normals directly. Only minimally tested this but hopefully there are no obvious regressions.

Src/Graphics/New3D/Model.h

@@ -27,6 +27,7 @@ struct Vertex
 	float normal[3];
 	float texcoords[2];
 	UINT8 color[4];
+	float faceNormal[3];
 	float fixedShade;
 };
 
@@ -119,7 +120,6 @@ struct Model
 
 	//matrices
 	float modelMat[16];
-	float determinant;				// we check if the determinant of the matrix is negative, if it is, the matrix will swap the axis order
 
 	//model scale step 1.5+
 	float scale = 1.0f;

Src/Graphics/New3D/New3D.cpp

@@ -242,19 +242,19 @@ void CNew3D::RenderFrame(void)
 	m_modelMat.Release();			// would hope we wouldn't need this but no harm in checking
 	m_nodeAttribs.Reset();
 
-	RenderViewport(0x800000);					// build model structure
+	RenderViewport(0x800000);						// build model structure
 
-	DrawScrollFog();							// fog layer if applicable must be drawn here
+	DrawScrollFog();								// fog layer if applicable must be drawn here
 
 	glDepthFunc		(GL_LEQUAL);
 	glEnable		(GL_DEPTH_TEST);
 	glDepthMask		(GL_TRUE);
 	glActiveTexture	(GL_TEXTURE0);
-	glEnable		(GL_CULL_FACE);
+	glDisable		(GL_CULL_FACE);					// we'll emulate this in the shader					
 	glFrontFace		(GL_CW);
 
 	glStencilFunc	(GL_EQUAL, 0, 0xFF);			// basically stencil test passes if the value is zero
-	glStencilOp		(GL_KEEP, GL_INCR, GL_INCR);		// if the stencil test passes, we incriment the value
+	glStencilOp		(GL_KEEP, GL_INCR, GL_INCR);	// if the stencil test passes, we incriment the value
 	glStencilMask	(0xFF);
 	
 	m_vbo.Bind(true);
@@ -286,12 +286,14 @@ void CNew3D::RenderFrame(void)
 	glEnableVertexAttribArray(2);
 	glEnableVertexAttribArray(3);
 	glEnableVertexAttribArray(4);
+	glEnableVertexAttribArray(5);
 
 	// before draw, specify vertex and index arrays with their offsets, offsetof is maybe evil ..
 	glVertexAttribPointer(m_r3dShader.GetVertexAttribPos("inVertex"),		4, GL_FLOAT, GL_FALSE, sizeof(Vertex), 0);
 	glVertexAttribPointer(m_r3dShader.GetVertexAttribPos("inNormal"),		3, GL_FLOAT, GL_FALSE, sizeof(Vertex), (void*)offsetof(Vertex, normal));
 	glVertexAttribPointer(m_r3dShader.GetVertexAttribPos("inTexCoord"),		2, GL_FLOAT, GL_FALSE, sizeof(Vertex), (void*)offsetof(Vertex, texcoords));
 	glVertexAttribPointer(m_r3dShader.GetVertexAttribPos("inColour"),		4, GL_UNSIGNED_BYTE, GL_TRUE, sizeof(Vertex), (void*)offsetof(Vertex, color));
+	glVertexAttribPointer(m_r3dShader.GetVertexAttribPos("inFaceNormal"),	3, GL_FLOAT, GL_FALSE, sizeof(Vertex), (void*)offsetof(Vertex, faceNormal));
 	glVertexAttribPointer(m_r3dShader.GetVertexAttribPos("inFixedShade"),	1, GL_FLOAT, GL_FALSE, sizeof(Vertex), (void*)offsetof(Vertex, fixedShade));
 
 	m_r3dShader.SetShader(true);
@@ -321,12 +323,13 @@ void CNew3D::RenderFrame(void)
 	m_vbo.Bind(false);
 
 	glDisable(GL_STENCIL_TEST);
-	glDisable(GL_CULL_FACE);
+
 	glDisableVertexAttribArray(0);
 	glDisableVertexAttribArray(1);
 	glDisableVertexAttribArray(2);
 	glDisableVertexAttribArray(3);
 	glDisableVertexAttribArray(4);
+	glDisableVertexAttribArray(5);
 }
 
 void CNew3D::BeginFrame(void)
@@ -410,9 +413,6 @@ bool CNew3D::DrawModel(UINT32 modelAddr)
 		m->modelMat[i] = m_modelMat.currentMatrix[i];
 	}
 
-	//calculate determinant
-	m->determinant = Determinant3x3(m_modelMat);
-
 	// update texture offsets
 	m->textureOffsetX = m_nodeAttribs.currentTexOffsetX;
 	m->textureOffsetY = m_nodeAttribs.currentTexOffsetY;
@@ -897,28 +897,11 @@ void CNew3D::RenderViewport(UINT32 addr)
 
 void CNew3D::CopyVertexData(const R3DPoly& r3dPoly, std::vector<Poly>& polyArray)
 {
-	//====================
+	Poly p;
-	Poly		p;
-	V3::Vec3	normal;
-	float		dotProd;
-	bool		clockWise;
-	//====================
 
-	V3::createNormal(r3dPoly.v[0].pos, r3dPoly.v[1].pos, r3dPoly.v[2].pos, normal);
+	p.p1 = r3dPoly.v[0];
-
+	p.p2 = r3dPoly.v[1];
-	dotProd		= V3::dotProduct(normal, r3dPoly.faceNormal);
+	p.p3 = r3dPoly.v[2];
-	clockWise	= dotProd >= 0;
-
-	if (clockWise) {
-		p.p1 = r3dPoly.v[0];
-		p.p2 = r3dPoly.v[1];
-		p.p3 = r3dPoly.v[2];
-	}
-	else {
-		p.p1 = r3dPoly.v[2];
-		p.p2 = r3dPoly.v[1];
-		p.p3 = r3dPoly.v[0];
-	}
 	
 	// Copy face colour to vertices
 	for (int i = 0; i < 4; i++) {
@@ -927,25 +910,20 @@ void CNew3D::CopyVertexData(const R3DPoly& r3dPoly, std::vector<Poly>& polyArray
 		p.p3.color[i] = r3dPoly.faceColour[i];
 	}
 
+	// Copy face normal
+	for (int i = 0; i < 3; i++) {
+		p.p1.faceNormal[i] = r3dPoly.faceNormal[i];
+		p.p2.faceNormal[i] = r3dPoly.faceNormal[i];
+		p.p3.faceNormal[i] = r3dPoly.faceNormal[i];
+	}
+	
 	polyArray.emplace_back(p);
 
 	if (r3dPoly.number == 4) {
 
-		V3::createNormal(r3dPoly.v[0].pos, r3dPoly.v[2].pos, r3dPoly.v[3].pos, normal);
+		p.p1 = r3dPoly.v[0];
-
+		p.p2 = r3dPoly.v[2];
-		dotProd		= V3::dotProduct(normal, r3dPoly.faceNormal);
+		p.p3 = r3dPoly.v[3];
-		clockWise	= dotProd >= 0;
-
-		if (clockWise) {
-			p.p1 = r3dPoly.v[0];
-			p.p2 = r3dPoly.v[2];
-			p.p3 = r3dPoly.v[3];
-		}
-		else {
-			p.p1 = r3dPoly.v[0];
-			p.p2 = r3dPoly.v[3];
-			p.p3 = r3dPoly.v[2];
-		}
 		
 		// Copy face colour to vertices
 		for (int i = 0; i < 4; i++) {
@@ -954,6 +932,13 @@ void CNew3D::CopyVertexData(const R3DPoly& r3dPoly, std::vector<Poly>& polyArray
 			p.p3.color[i] = r3dPoly.faceColour[i];
 		}
 
+		// Copy face normal
+		for (int i = 0; i < 3; i++) {
+			p.p1.faceNormal[i] = r3dPoly.faceNormal[i];
+			p.p2.faceNormal[i] = r3dPoly.faceNormal[i];
+			p.p3.faceNormal[i] = r3dPoly.faceNormal[i];
+		}
+
 		polyArray.emplace_back(p);
 	}
 }

Src/Graphics/New3D/R3DShader.cpp

@@ -18,6 +18,7 @@ attribute vec4	inVertex;
 attribute vec3	inNormal;
 attribute vec2	inTexCoord;
 attribute vec4	inColour;
+attribute vec3	inFaceNormal;		// used to emulate r3d culling 
 attribute float	inFixedShade;
 
 // outputs to fragment shader
@@ -26,8 +27,18 @@ varying vec3	fsViewVertex;
 varying vec3	fsViewNormal;		// per vertex normal vector
 varying vec2	fsTexCoord;
 varying vec4	fsColor;
+varying float	fsDiscard;			// can't have varying bool (glsl spec)
 varying float	fsFixedShade;
 
+float CalcBackFace(in vec3 viewVertex)
+{
+	vec3 vt = viewVertex - vec3(0.0);
+	vec3 vn = (mat3(gl_ModelViewMatrix) * inFaceNormal);
+
+	// dot product of face normal with view direction
+	return dot(vt, vn);
+}
+
 void main(void)
 {
 	fsViewVertex	= vec3(gl_ModelViewMatrix * inVertex);
@@ -35,6 +46,7 @@ void main(void)
 	float z			= length(fsViewVertex);
 	fsFogFactor		= fogIntensity * clamp(fogStart + z * fogDensity, 0.0, 1.0);
 
+	fsDiscard		= CalcBackFace(fsViewVertex);
 	fsColor    		= inColour;
 	fsTexCoord		= inTexCoord;
 	fsFixedShade	= inFixedShade;
@@ -82,6 +94,7 @@ varying vec3	fsViewVertex;
 varying vec3	fsViewNormal;		// per vertex normal vector
 varying vec4	fsColor;
 varying vec2	fsTexCoord;
+varying float	fsDiscard;
 varying float	fsFixedShade;
 
 vec4 GetTextureValue()
@@ -129,6 +142,10 @@ void main()
 	vec4 finalData;
 	vec4 fogData;
 
+	if(fsDiscard>0) {
+		discard;		//emulate back face culling here
+	}
+
 	fogData = vec4(fogColour.rgb * fogAmbient, fsFogFactor);
 	tex1Data = vec4(1.0, 1.0, 1.0, 1.0);
 
@@ -261,7 +278,6 @@ void R3DShader::Start()
 	m_textured2			= false;
 	m_textureAlpha		= false;		// use alpha in texture
 	m_alphaTest			= false;		// discard fragment based on alpha (ogl does this with fixed function)
-	m_doubleSided		= false;
 	m_lightEnabled		= false;
 	m_specularEnabled	= false;
 	m_layered			= false;
@@ -275,8 +291,6 @@ void R3DShader::Start()
 	m_baseTexSize[0] = 0;
 	m_baseTexSize[1] = 0;
 
-	m_matDet = MatDet::notset;
-
 	m_dirtyMesh		= true;			// dirty means all the above are dirty, ie first run
 	m_dirtyModel	= true;
 }
@@ -443,22 +457,6 @@ void R3DShader::SetMeshUniforms(const Mesh* m)
 		}
 	}
 
-	if (m_matDet!=MatDet::zero) {
-
-		if (m_dirtyMesh || m->doubleSided != m_doubleSided) {
-
-			m_doubleSided = m->doubleSided;
-
-			if (m_doubleSided) {
-				glDisable(GL_CULL_FACE);
-			}
-			else {
-				glEnable(GL_CULL_FACE);
-			}
-		}
-	}
-
-
 	m_dirtyMesh = false;
 }
 
@@ -484,42 +482,12 @@ void R3DShader::SetViewportUniforms(const Viewport *vp)
 
 void R3DShader::SetModelStates(const Model* model)
 {
-	//==========
-	MatDet test;
-	//==========
-
-	test = MatDet::notset;		// happens for bad matrices with NaN
-
-	if (model->determinant < 0)			{ test = MatDet::negative; }
-	else if (model->determinant > 0)	{ test = MatDet::positive; }
-	else if (model->determinant == 0)	{ test = MatDet::zero; }
-
-	if (m_dirtyModel || m_matDet!=test) {
-
-		switch (test) {
-		case MatDet::negative:
-			glCullFace(GL_FRONT);
-			glEnable(GL_CULL_FACE);
-			m_doubleSided = false;
-			break;
-		case MatDet::positive:
-			glCullFace(GL_BACK);
-			glEnable(GL_CULL_FACE);
-			m_doubleSided = false;
-			break;
-		default:
-			glDisable(GL_CULL_FACE);
-			m_doubleSided = true;		// basically drawing on both sides now
-		}
-	}
-
 	if (m_dirtyModel || model->scale != m_modelScale) {
 		glUniform1f(m_locModelScale, model->scale);
 		m_modelScale = model->scale;
 	}
 
-	m_matDet		= test;
+	m_dirtyModel = false;
-	m_dirtyModel	= false;
 }
 
 } // New3D

Src/Graphics/New3D/R3DShader.h

@@ -47,7 +47,6 @@ private:
 	bool	m_textureAlpha;		// use alpha in texture
 	bool	m_alphaTest;		// discard fragment based on alpha (ogl does this with fixed function)
 	float	m_fogIntensity;
-	bool	m_doubleSided;
 	bool	m_lightEnabled;
 	float	m_shininess;
 	float	m_specularValue;
@@ -60,8 +59,6 @@ private:
 	bool	m_textureInverted;
 	
 	// cached model values
-	enum class MatDet { notset, negative, positive, zero };
-	MatDet	m_matDet;
 	float	m_modelScale;
 
 	// are our cache values dirty