Stencil layered polys, fixes shadow z fighting in some of the games. Other games maybe using stipple alpha (yuck), not sure yet.

Src/Graphics/New3D/Model.h

@@ -49,6 +49,7 @@ struct Mesh
 	bool textureAlpha	= false;		// use alpha in texture
 	bool alphaTest		= false;		// discard fragment based on alpha (ogl does this with fixed function)
 	bool clockWise		= true;			// we need to check if the matrix will change the winding
+	bool layered		= false;		// stencil poly
 
 	// lighting
 	bool lighting		= false;

Src/Graphics/New3D/New3D.cpp

@@ -152,6 +152,7 @@ void CNew3D::RenderScene(int priority, bool alpha)
 
 	glDisable(GL_BLEND);
 	glDepthMask(GL_TRUE);
+	glDisable(GL_STENCIL_TEST);
 }
 
 void CNew3D::RenderFrame(void)
@@ -168,6 +169,10 @@ void CNew3D::RenderFrame(void)
 	glEnable		(GL_CULL_FACE);
 	glFrontFace		(GL_CW);
 
+	glStencilFunc	(GL_EQUAL, 0, 0xFF);			// basically stencil test passes if the value is zero
+	glStencilOp		(GL_KEEP, GL_INCR, GL_INCR);	// if the stencil test passes, we incriment the value
+	glStencilMask	(0xFF);
+
 	RenderViewport(0x800000);		// build model structure
 	
 	m_vbo.Bind(true);
@@ -208,7 +213,7 @@ void CNew3D::RenderFrame(void)
 	m_r3dShader.SetShader(true);
 
 	for (int pri = 0; pri <= 3; pri++) {
-		glClear		(GL_DEPTH_BUFFER_BIT);
+		glClear		(GL_DEPTH_BUFFER_BIT|GL_STENCIL_BUFFER_BIT);
 		RenderScene	(pri, false);
 		RenderScene	(pri, true);
 	}
@@ -883,6 +888,7 @@ void CNew3D::CacheModel(Model *m, const UINT32 *data)
 				currentMesh->textureAlpha	= ph.TextureAlpha();
 				currentMesh->polyAlpha		= ph.PolyAlpha();
 				currentMesh->lighting		= ph.LightEnabled() && !ph.FixedShading();
+				currentMesh->layered		= ph.Layered();
 				
 				if (currentMesh->lighting) {
 					if (ph.SpecularEnabled()) {

Src/Graphics/New3D/PolyHeader.cpp

@@ -278,6 +278,11 @@ int PolyHeader::Y()
 // header 6
 //
 
+bool PolyHeader::Layered()
+{
+	return (header[6] & 0x8) > 0;
+}
+
 float PolyHeader::Shininess()
 {
 	return ((header[6] >> 5) & 3) / 3.0f;

Src/Graphics/New3D/PolyHeader.h

@@ -126,6 +126,7 @@ public:
 	int		Y();
 
 	//header 6
+	bool	Layered();
 	float	Shininess();
 	int		TexFormat();
 	bool	TexEnabled();

Src/Graphics/New3D/R3DShader.cpp

@@ -153,6 +153,7 @@ void R3DShader::Start()
 	m_alphaTest		= false;		// discard fragment based on alpha (ogl does this with fixed function)
 	m_doubleSided	= false;
 	m_lightEnabled	= false;
+	m_layered		= false;
 
 	m_shininess = 0;
 	m_specularCoefficient = 0;
@@ -270,6 +271,16 @@ void R3DShader::SetMeshUniforms(const Mesh* m)
 		m_specularCoefficient = m->specularCoefficient;
 	}
 
+	if (m_dirtyMesh || m->layered != m_layered) {
+		m_layered = m->layered;
+		if (m_layered) {
+			glEnable(GL_STENCIL_TEST);
+		}
+		else {
+			glDisable(GL_STENCIL_TEST);
+		}
+	}
+
 	if (m_matDet!=MatDet::zero) {
 
 		if (m_dirtyMesh || m->doubleSided != m_doubleSided) {

Src/Graphics/New3D/R3DShader.h

@@ -43,6 +43,7 @@ private:
 	bool	m_lightEnabled;
 	float	m_shininess;
 	float	m_specularCoefficient;
+	bool	m_layered;
 
 	// cached model values
 	enum class MatDet { notset, negative, positive, zero };

Src/OSD/SDL/Main.cpp

@@ -222,6 +222,7 @@ static bool CreateGLScreen(const char *caption, unsigned *xOffsetPtr, unsigned *
   SDL_GL_SetAttribute(SDL_GL_GREEN_SIZE,8);
   SDL_GL_SetAttribute(SDL_GL_BLUE_SIZE,8);
   SDL_GL_SetAttribute(SDL_GL_DEPTH_SIZE,24);
+  SDL_GL_SetAttribute(SDL_GL_STENCIL_SIZE,8);
   SDL_GL_SetAttribute(SDL_GL_DOUBLEBUFFER,1);
 
   // Set vsync