From c40d6ac17b2936726e79c874473b7c8029d8425c Mon Sep 17 00:00:00 2001
From: Ian Curtis <i.curtis@gmail.com>
Date: Fri, 10 Nov 2023 19:13:37 +0000
Subject: [PATCH] Rewrite the stencil buffer usage slightly, so both the LOS
 and layered polys work. LOS uses the top bit of the stencil buffer. Fixes
 some minor issues with draw order in sega rally.

---
 Src/Graphics/New3D/New3D.cpp     |  5 +++--
 Src/Graphics/New3D/R3DShader.cpp | 22 +++++++++++++++++-----
 2 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/Src/Graphics/New3D/New3D.cpp b/Src/Graphics/New3D/New3D.cpp
index 55cba60..45ae512 100644
--- a/Src/Graphics/New3D/New3D.cpp
+++ b/Src/Graphics/New3D/New3D.cpp
@@ -330,8 +330,6 @@ void CNew3D::SetRenderStates()
 	glDisable		(GL_CULL_FACE);					// we'll emulate this in the shader		
 
 	glEnable		(GL_STENCIL_TEST);
-	glStencilOp		(GL_KEEP, GL_KEEP, GL_REPLACE);
-	glStencilMask	(0xFF);
 
 	glBlendFunc		(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
 	glDisable		(GL_BLEND);
@@ -1876,6 +1874,9 @@ bool CNew3D::ProcessLos(int priority)
 				GLubyte stencilVal;
 				glReadPixels(losX, losY, 1, 1, GL_STENCIL_INDEX, GL_UNSIGNED_BYTE, &stencilVal);
 
+				// apply our mask to stencil, because layered poly attributes use the lower bits
+				stencilVal &= 0x80;
+
 				// if the stencil val is zero that means we've hit sky or whatever, if it hits a 1 we've hit geometry
 				// the real3d returns 1 in the top bit of the float if the line of sight test passes (ie doesn't hit geometry)
 
diff --git a/Src/Graphics/New3D/R3DShader.cpp b/Src/Graphics/New3D/R3DShader.cpp
index ead8ef5..fb7a6bd 100644
--- a/Src/Graphics/New3D/R3DShader.cpp
+++ b/Src/Graphics/New3D/R3DShader.cpp
@@ -298,14 +298,26 @@ void R3DShader::SetMeshUniforms(const Mesh* m)
 		glUniform2iv(m_locTexWrapMode, 1, m_texWrapMode);
 	}
 
+	if (m_dirtyMesh || m->noLosReturn != m_noLosReturn) {
+		m_noLosReturn = m->noLosReturn;
+		glStencilFunc(GL_ALWAYS, m_noLosReturn << 7, 0b10000000);
+		glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
+		glStencilMask(0b10000000);
+	}
+
 	if (m_dirtyMesh || m->layered != m_layered) {
 		m_layered = m->layered;
 		// i think it should just disable z write, but the polys I think must be written first
-	}
-
-	if (m_dirtyMesh || m->noLosReturn != m_noLosReturn) {
-		m_noLosReturn = m->noLosReturn;
-		glStencilFunc(GL_ALWAYS, m_noLosReturn, -1);		// we'll write either a 0 or 1 to the stencil buffer
+		if (m_layered) {
+			glStencilFunc(GL_EQUAL, 0, 0b01111111);			// basically stencil test passes if the value is zero
+			glStencilOp(GL_KEEP, GL_INCR, GL_INCR);			// if the stencil test passes, we increment the value
+			glStencilMask(0b01111111);
+		}
+		else {
+			glStencilFunc(GL_ALWAYS, m_noLosReturn << 7, 0b10000000);
+			glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
+			glStencilMask(0b10000000);
+		}
 	}
 
 	m_dirtyMesh = false;