Composite the alpha layers at the end of rendering. To do this we need to mask the alpha pixels with the opaque pixels from the next priority layer. Fixes some overlapping shadows in vf3tb that have different priority layers. I assume that was a game bug, but it works on the real h/w.

Src/Graphics/New3D/Model.h

@@ -98,7 +98,7 @@ struct FVertex : Vertex			// full vertex including face attributes
 	}
 };
 
-enum class Layer { colour, trans1, trans2, all, none };
+enum class Layer { colour, trans1, trans2, trans12 /*both 1&2*/, all, none };
 
 struct Mesh
 {

Src/Graphics/New3D/New3D.cpp

@@ -226,8 +226,6 @@ bool CNew3D::RenderScene(int priority, bool renderOverlay, Layer layer)
 		}
 	}
 
-	glDisable(GL_BLEND);
-
 	return hasOverlay;
 }
 
@@ -269,6 +267,7 @@ void CNew3D::SetRenderStates()
 	glDepthMask		(GL_TRUE);
 	glActiveTexture	(GL_TEXTURE0);
 	glDisable		(GL_CULL_FACE);					// we'll emulate this in the shader		
+	glDisable		(GL_BLEND);
 
 	glStencilFunc	(GL_EQUAL, 0, 0xFF);			// basically stencil test passes if the value is zero
 	glStencilOp		(GL_KEEP, GL_INCR, GL_INCR);	// if the stencil test passes, we incriment the value
@@ -308,7 +307,6 @@ void CNew3D::RenderFrame(void)
 	m_nodeAttribs.Reset();
 
 	RenderViewport(0x800000);						// build model structure
-
 	DrawScrollFog();								// fog layer if applicable must be drawn here
 	
 	m_vbo.Bind(true);
@@ -335,6 +333,9 @@ void CNew3D::RenderFrame(void)
 		}
 	}
 
+	m_r3dFrameBuffers.SetFBO(Layer::trans12);
+	glClear(GL_COLOR_BUFFER_BIT);					// wipe both trans layers
+
 	for (int pri = 0; pri <= 3; pri++) {
 
 		//==============
@@ -347,7 +348,8 @@ void CNew3D::RenderFrame(void)
 
 			bool renderOverlay = (i == 1);
 
-			m_r3dFrameBuffers.Clear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT);
+			m_r3dFrameBuffers.SetFBO(Layer::colour);
+			glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT);
 			
 			SetRenderStates();
 
@@ -359,25 +361,31 @@ void CNew3D::RenderFrame(void)
 				ProcessLos(pri);
 			}
 
+			DisableRenderStates();
+
+			m_r3dFrameBuffers.DrawOverTransLayers();			// mask trans layer with opaque pixels
+			m_r3dFrameBuffers.CompositeBaseLayer();				// copy opaque pixels to back buffer
+
+			SetRenderStates();
+			
 			glDepthFunc(GL_LESS);								// alpha polys seem to use gl_less (ocean hunter)
 
-			m_r3dShader.DiscardAlpha(false);					// render only translucent pixels
-			m_r3dFrameBuffers.StoreDepth();						// save depth buffer for 1st trans pass
-			m_r3dFrameBuffers.SetFBO(Layer::trans1);
-			RenderScene(pri, renderOverlay, Layer::trans1);
+			m_r3dShader.DiscardAlpha		(false);			// render only translucent pixels
+			m_r3dFrameBuffers.StoreDepth	();					// save depth buffer for 1st trans pass
+			m_r3dFrameBuffers.SetFBO		(Layer::trans1);
+			RenderScene						(pri, renderOverlay, Layer::trans1);
 
-			m_r3dFrameBuffers.RestoreDepth();					// restore depth buffer, trans layers don't seem to depth test against each other
-			m_r3dFrameBuffers.SetFBO(Layer::trans2);
-			RenderScene(pri, renderOverlay, Layer::trans2);
+			m_r3dFrameBuffers.RestoreDepth	();					// restore depth buffer, trans layers don't seem to depth test against each other
+			m_r3dFrameBuffers.SetFBO		(Layer::trans2);
+			RenderScene						(pri, renderOverlay, Layer::trans2);
 
 			DisableRenderStates();
-			m_r3dFrameBuffers.Draw();							// draw current layer to back buffer
 
 			if (!hasOverlay) break;								// no high priority polys						
 		}
 	}
 
-	m_r3dFrameBuffers.DisableFBO();		// draw to back buffer normally
+	m_r3dFrameBuffers.CompositeAlphaLayer();
 }
 
 void CNew3D::BeginFrame(void)

Src/Graphics/New3D/R3DFrameBuffers.cpp

@@ -22,6 +22,7 @@ R3DFrameBuffers::R3DFrameBuffers()
 
 	AllocShaderTrans();
 	AllocShaderBase();
+	AllocShaderWipe();
 
 	FBVertex vertices[4];
 	vertices[0].Set(-1,-1, 0, 0);
@@ -36,6 +37,8 @@ R3DFrameBuffers::~R3DFrameBuffers()
 {
 	DestroyFBO();
 	m_shaderTrans.UnloadShaders();
+	m_shaderBase.UnloadShaders();
+	m_shaderWipe.UnloadShaders();
 	m_vbo.Destroy();
 }
 
@@ -158,30 +161,40 @@ void R3DFrameBuffers::SetFBO(Layer layer)
 		return;
 	}
 
-	glBindFramebuffer(GL_FRAMEBUFFER, m_frameBufferID);
-	GLenum buffers[] = { GL_COLOR_ATTACHMENT0 + (GLenum)layer };
-	glDrawBuffers(countof(buffers), buffers);
-	m_lastLayer = layer;
-}
-
-void R3DFrameBuffers::DisableFBO()
-{
-	glBindFramebuffer(GL_FRAMEBUFFER, 0);
-	glDrawBuffer(GL_BACK);
-	m_lastLayer = Layer::none;
-}
-
-void R3DFrameBuffers::Clear(GLbitfield mask)
-{
-	if (m_lastLayer != Layer::all) {
-		glBindFramebuffer(GL_FRAMEBUFFER, m_frameBufferID);		// bind frame buffer
+	switch (layer)
+	{
+	case Layer::colour:
+	case Layer::trans1:
+	case Layer::trans2:
+	{
+		glBindFramebuffer(GL_FRAMEBUFFER, m_frameBufferID);
+		GLenum buffers[] = { GL_COLOR_ATTACHMENT0 + (GLenum)layer };
+		glDrawBuffers(countof(buffers), buffers);
+		break;
+	}
+	case Layer::trans12:
+	{
+		glBindFramebuffer(GL_FRAMEBUFFER, m_frameBufferID);
+		GLenum buffers[] = { GL_COLOR_ATTACHMENT1, GL_COLOR_ATTACHMENT2 };
+		glDrawBuffers(countof(buffers), buffers);
+		break;
+	}
+	case Layer::all:
+	{
+		glBindFramebuffer(GL_FRAMEBUFFER, m_frameBufferID);
 		GLenum buffers[] = { GL_COLOR_ATTACHMENT0, GL_COLOR_ATTACHMENT1, GL_COLOR_ATTACHMENT2 };
 		glDrawBuffers(countof(buffers), buffers);
+		break;
+	}
+	case Layer::none:
+	{
+		glBindFramebuffer(GL_FRAMEBUFFER, 0);
+		glDrawBuffer(GL_BACK);
+		break;
+	}
 	}
 
-	glViewport(0, 0, m_width, m_height);
-	glClear(mask);
-	m_lastLayer = Layer::all;
+	m_lastLayer = layer;
 }
 
 void R3DFrameBuffers::AllocShaderBase()
@@ -276,12 +289,59 @@ void R3DFrameBuffers::AllocShaderTrans()
 	m_shaderTrans.attribLoc[1] = m_shaderTrans.GetAttributeLocation("inTexCoord");
 }
 
+void R3DFrameBuffers::AllocShaderWipe()
+{
+	const char *vertexShader = R"glsl(
+
+	// inputs
+	attribute vec3 inVertex; 
+	attribute vec2 inTexCoord;
+
+	// outputs
+	varying vec2 fsTexCoord;
+
+	void main(void)
+	{
+		fsTexCoord = inTexCoord;
+		gl_Position = vec4(inVertex,1.0);
+	}
+
+	)glsl";
+
+	const char *fragmentShader = R"glsl(
+
+	uniform sampler2D texColor;				// base colour layer
+	varying vec2 fsTexCoord;
+
+	void main()
+	{
+		vec4 colBase = texture2D( texColor, fsTexCoord);
+
+		if(colBase == 0.0) {
+			discard;						// no colour pixels have been written
+		}
+
+		gl_FragData[0] = vec4(0.0);			// wipe these parts of the alpha buffer
+		gl_FragData[1] = vec4(0.0);			// since they have been overwritten by the next priority layer
+	}
+
+	)glsl";
+
+	m_shaderWipe.LoadShaders(vertexShader, fragmentShader);
+
+	m_shaderWipe.uniformLoc[0] = m_shaderTrans.GetUniformLocation("texColor");
+
+	m_shaderWipe.attribLoc[0] = m_shaderTrans.GetAttributeLocation("inVertex");
+	m_shaderWipe.attribLoc[1] = m_shaderTrans.GetAttributeLocation("inTexCoord");
+}
+
 void R3DFrameBuffers::Draw()
 {
-	DisableFBO	();									// make sure to draw on the back buffer
+	SetFBO		(Layer::none);						// make sure to draw on the back buffer
 	glViewport	(0, 0, m_width, m_height);			// cover the entire screen
 	glDisable	(GL_DEPTH_TEST);					// disable depth testing / writing
 	glDisable	(GL_CULL_FACE);
+	glDisable	(GL_BLEND);
 
 	for (int i = 0; i < countof(m_texIDs); i++) {	// bind our textures to correct texture units
 		glActiveTexture(GL_TEXTURE0 + i);
@@ -302,6 +362,84 @@ void R3DFrameBuffers::Draw()
 	m_vbo.Bind		(false);
 }
 
+void R3DFrameBuffers::CompositeBaseLayer()
+{
+	SetFBO(Layer::none);							// make sure to draw on the back buffer
+	glViewport(0, 0, m_width, m_height);			// cover the entire screen
+	glDisable(GL_DEPTH_TEST);						// disable depth testing / writing
+	glDisable(GL_CULL_FACE);
+	glDisable(GL_BLEND);
+
+	for (int i = 0; i < countof(m_texIDs); i++) {	// bind our textures to correct texture units
+		glActiveTexture(GL_TEXTURE0 + i);
+		glBindTexture(GL_TEXTURE_2D, m_texIDs[i]);
+	}
+
+	glActiveTexture(GL_TEXTURE0);
+	m_vbo.Bind(true);
+
+	DrawBaseLayer();
+
+	m_vbo.Bind(false);
+}
+
+void R3DFrameBuffers::CompositeAlphaLayer()
+{
+	SetFBO(Layer::none);							// make sure to draw on the back buffer
+	glViewport(0, 0, m_width, m_height);			// cover the entire screen
+	glDisable(GL_DEPTH_TEST);						// disable depth testing / writing
+	glDisable(GL_CULL_FACE);
+
+	for (int i = 0; i < countof(m_texIDs); i++) {	// bind our textures to correct texture units
+		glActiveTexture(GL_TEXTURE0 + i);
+		glBindTexture(GL_TEXTURE_2D, m_texIDs[i]);
+	}
+
+	glActiveTexture(GL_TEXTURE0);
+	m_vbo.Bind(true);
+
+	glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
+	glEnable(GL_BLEND);
+
+	DrawAlphaLayer();
+
+	glDisable(GL_BLEND);
+	m_vbo.Bind(false);
+}
+
+void R3DFrameBuffers::DrawOverTransLayers()
+{
+	SetFBO(Layer::trans12);							// need to write to both layers
+
+	glViewport	(0, 0, m_width, m_height);			// cover the entire screen
+	glDisable	(GL_DEPTH_TEST);					// disable depth testing / writing
+	glDisable	(GL_CULL_FACE);
+	glDisable	(GL_BLEND);
+
+	glActiveTexture	(GL_TEXTURE0);
+	glBindTexture	(GL_TEXTURE_2D, m_texIDs[0]);
+
+	m_vbo.Bind(true);
+
+	m_shaderWipe.EnableShader();
+	glUniform1i(m_shaderWipe.uniformLoc[0], 0);
+
+		glEnableVertexAttribArray(0);
+		glEnableVertexAttribArray(1);
+
+		glVertexAttribPointer(m_shaderWipe.attribLoc[0], 3, GL_FLOAT, GL_FALSE, sizeof(FBVertex), (void*)offsetof(FBVertex, verts));
+		glVertexAttribPointer(m_shaderWipe.attribLoc[1], 2, GL_FLOAT, GL_FALSE, sizeof(FBVertex), (void*)offsetof(FBVertex, texCoords));
+
+		glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
+
+		glDisableVertexAttribArray(0);
+		glDisableVertexAttribArray(1);
+
+	m_shaderWipe.DisableShader();
+
+	m_vbo.Bind(false);
+}
+
 void R3DFrameBuffers::DrawBaseLayer()
 {
 	m_shaderBase.EnableShader();

Src/Graphics/New3D/R3DFrameBuffers.h

@@ -14,15 +14,16 @@ public:
 	R3DFrameBuffers();
 	~R3DFrameBuffers();
 
-	void	Draw();		// draw and composite the transparent layers
+	void	Draw();					// draw and composite the transparent layers
+	void	CompositeBaseLayer();
+	void	CompositeAlphaLayer();
+	void	DrawOverTransLayers();	// opaque pixels in next priority layer need to wipe trans pixels
 	
 	bool	CreateFBO(int width, int height);
 	void	DestroyFBO();
 
 	void	BindTexture(Layer layer);
 	void	SetFBO(Layer layer);
-	void	DisableFBO();
-	void	Clear(GLbitfield mask);
 	void	StoreDepth();
 	void	RestoreDepth();
 
@@ -47,6 +48,7 @@ private:
 	GLuint	CreateTexture(int width, int height);
 	void	AllocShaderTrans();
 	void	AllocShaderBase();
+	void	AllocShaderWipe();
 
 	void	DrawBaseLayer();
 	void	DrawAlphaLayer();
@@ -63,6 +65,7 @@ private:
 	// shaders
 	GLSLShader m_shaderBase;
 	GLSLShader m_shaderTrans;
+	GLSLShader m_shaderWipe;
 
 	// vertices for fbo
 	VBO m_vbo;

Src/Graphics/New3D/R3DShader.cpp

@@ -127,9 +127,14 @@ bool R3DShader::LoadShader(const char* vertexShader, const char* fragmentShader)
 	return true;
 }
 
-GLint R3DShader::GetVertexAttribPos(const char* attrib)
+GLint R3DShader::GetVertexAttribPos(const std::string& attrib)
 {
-	return glGetAttribLocation(m_shaderProgram, attrib);	// probably should cache this but only called 1x per frame anyway
+	if (m_vertexLocCache.count(attrib)==0) {
+		auto pos = glGetAttribLocation(m_shaderProgram, attrib.c_str());
+		m_vertexLocCache[attrib] = pos;
+	}
+
+	return m_vertexLocCache[attrib];
 }
 
 void R3DShader::SetShader(bool enable)

Src/Graphics/New3D/R3DShader.h

@@ -4,6 +4,8 @@
 #include "Pkgs/glew.h"
 #include "Util/NewConfig.h"
 #include "Model.h"
+#include <map>
+#include <string>
 
 namespace New3D {
 
@@ -18,7 +20,7 @@ public:
 	void	SetViewportUniforms	(const Viewport *vp);
 	void	Start				();
 	void	SetShader			(bool enable = true);
-	GLint	GetVertexAttribPos	(const char* attrib);
+	GLint	GetVertexAttribPos	(const std::string& attrib);
 	void	DiscardAlpha		(bool discard);				// use to remove alpha from texture alpha only polys for 1st pass
 
 private:
@@ -103,6 +105,10 @@ private:
 	// global uniforms
 	GLint m_locHardwareStep;
 	GLint m_locDiscardAlpha;
+
+	// vertex attribute position cache
+	std::map<std::string, GLint> m_vertexLocCache;
+
 };