Fix the line of sight function.

Polygons have a line of sight value that's either a 0 or 1. We write this value into the stencil buffer, then read the app can read it back to determine the polygon attribute is visible or not. The returned value is 1/depth value in world coordinates. The first bit of the float is actually a control bit, 1 indicates no geometry hit, 0 indicates geometry hit.

Src/Graphics/New3D/Model.h

@@ -147,6 +147,7 @@ struct Mesh
 	bool highPriority	= false;		// rendered over the top
 	bool transLSelect	= false;		// actually the transparency layer, false = layer 0, true = layer 1
 	bool translatorMap	= false;		// colours are multiplied by 16
+	bool noLosReturn	= false;		// line of sight test
 
 	// lighting
 	bool fixedShading	= false;

Src/Graphics/New3D/New3D.cpp

@@ -324,8 +324,8 @@ void CNew3D::SetRenderStates()
 	glActiveTexture	(GL_TEXTURE0);
 	glDisable		(GL_CULL_FACE);					// we'll emulate this in the shader		
 
-	glStencilFunc	(GL_EQUAL, 0, 0xFF);			// basically stencil test passes if the value is zero
+	glEnable		(GL_STENCIL_TEST);
-	glStencilOp		(GL_KEEP, GL_INCR, GL_INCR);	// if the stencil test passes, we increment the value
+	glStencilOp		(GL_KEEP, GL_KEEP, GL_REPLACE);
 	glStencilMask	(0xFF);
 
 	glBlendFunc		(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
@@ -1160,6 +1160,7 @@ void CNew3D::SetMeshValues(SortingMesh *currentMesh, PolyHeader &ph)
 	currentMesh->specularValue	= ph.SpecularValue();
 	currentMesh->fogIntensity	= ph.LightModifier();
 	currentMesh->translatorMap	= ph.TranslatorMap();
+	currentMesh->noLosReturn	= ph.NoLosReturn();
 
 	if (currentMesh->textured) {
 
@@ -1827,17 +1828,32 @@ bool CNew3D::ProcessLos(int priority)
 				float depth;
 				glReadPixels(losX, losY, 1, 1, GL_DEPTH_COMPONENT, GL_FLOAT, &depth);
 
-				if (depth < 0.99f || depth == 1.0f) {		// kinda guess work but when depth = 1, haven't drawn anything, when 0.99~ drawing sky somewhere far
-					return false;
-				}
-
 				depth = 2.0f * depth - 1.0f;
 
 				float zNear = m_nfPairs[priority].zNear;
 				float zFar	= m_nfPairs[priority].zFar;
 				float zVal	= 2.0f * zNear * zFar / (zFar + zNear - depth * (zFar - zNear));
 
+				// real3d test program indicates that return values are 1/zVal
+				zVal = 1.0f / zVal;
+
+				GLubyte stencilVal;
+				glReadPixels(losX, losY, 1, 1, GL_STENCIL_INDEX, GL_UNSIGNED_BYTE, &stencilVal);
+
+				// if the stencil val is zero that means we've hit sky or whatever, if it hits a 1 we've hit geometry
+				// the real3d returns 1 in the top bit of the float if the line of sight test passes (ie doesn't hit geometry)
+
+				auto zValP = reinterpret_cast<unsigned char*>(&zVal);	// this is legal in c++, casting to int technically isn't
+
+				if (stencilVal == 0) {
+					zValP[0] |= 1;		// set first bit to 1
+				}
+				else {
+					zValP[0] &= 0xFE;	// set first bit to zero
+				}
+
 				m_losBack->value[priority] = zVal;
+
 				return true;
 			}
 		}

Src/Graphics/New3D/PolyHeader.cpp

@@ -408,6 +408,7 @@ UINT64 PolyHeader::Hash()
 	hash |= (UINT64)FixedShading() << 34;										// bits 34 fixed shading
 	hash |= (UINT64)(header[0] >> 26) << 35;									// bits 35-40 specular coefficient (opacity)
 	hash |= (UINT64)(header[6] & 0x3FFFF) << 41;								// bits 41-58 Translucency pattern select / disable lighting / Polygon light modifier / Texture enable / Texture format / Shininess / High priority / Layered polygon / Translucency mode
+	hash |= (UINT64)NoLosReturn() << 59;										// bits 59 no line of sight return
 
 	return hash;
 }

Src/Graphics/New3D/R3DShader.cpp

@@ -28,6 +28,7 @@ void R3DShader::Start()
 	m_lightEnabled		= false;
 	m_specularEnabled	= false;
 	m_layered			= false;
+	m_noLosReturn		= false;
 	m_textureInverted	= false;
 	m_fixedShading		= false;
 	m_translatorMap		= false;
@@ -299,12 +300,12 @@ void R3DShader::SetMeshUniforms(const Mesh* m)
 
 	if (m_dirtyMesh || m->layered != m_layered) {
 		m_layered = m->layered;
-		if (m_layered) {
+		// i think it should just disable z write, but the polys I think must be written first
-			glEnable(GL_STENCIL_TEST);
+	}
-		}
+
-		else {
+	if (m_dirtyMesh || m->noLosReturn != m_noLosReturn) {
-			glDisable(GL_STENCIL_TEST);
+		m_noLosReturn = m->noLosReturn;
-		}
+		glStencilFunc(GL_ALWAYS, m_noLosReturn, -1);		// we'll write either a 0 or 1 to the stencil buffer
 	}
 
 	m_dirtyMesh = false;

Src/Graphics/New3D/R3DShader.h

@@ -70,6 +70,7 @@ private:
 	bool	m_translatorMap;
 
 	bool	m_layered;
+	bool	m_noLosReturn;
 	float	m_microTexScale;
 	int		m_microTexID;
 	int		m_baseTexInfo[4];

Src/Model3/Real3D.cpp

@@ -809,12 +809,6 @@ uint32_t CReal3D::ReadRegister(unsigned reg)
 
 	int index = (reg - 20) / 4;
 	float val = Render3D->GetLosValue(index);
-
-	if (val != 0.f) {
-		//val = 1.0f / val;		// test program indicate z values are 1 over
-		return 0xffffffff;		// infinity
-	}
-
 	return *(uint32_t*)(&val);
   }