Legacy engine: Added stencil buffering for layered models and shadows. Not as accurate as new engine because stencil buffering is applied per-model rather than per-polygon, so the entire model must consist of stencil-tested polygons. In practice, though, this seems to work fine.

Src/Graphics/Legacy3D/Legacy3D.cpp

@@ -975,6 +975,12 @@ void CLegacy3D::RenderFrame(void)
   glDepthFunc(GL_LESS);
   glEnable(GL_DEPTH_TEST);
 
+  // Stencil buffering
+  glStencilFunc(GL_EQUAL, 0, 0xFF);       // stencil test passes if stencil buffer value is 0
+  glStencilOp(GL_KEEP, GL_INCR, GL_INCR); // if the stencil test passes, increment value in stencil buffer
+  glStencilMask(0xFF);
+  glDisable(GL_STENCIL_TEST);             // enabled only for select models
+
   // Bind Real3D shader program and texture maps
   glUseProgram(shaderProgram);
   for (unsigned mapNum = 0; mapNum < numTexMaps; mapNum++)
@@ -1009,7 +1015,7 @@ void CLegacy3D::RenderFrame(void)
   ClearModelCache(&PolyCache);
   for (int pri = 0; pri <= 3; pri++)
   {
-    glClear(GL_DEPTH_BUFFER_BIT);
+    glClear(GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT);
     //ClearModelCache(&PolyCache);
     ClearDisplayList(&PolyCache);
     ClearDisplayList(&VROMCache);
@@ -1019,7 +1025,8 @@ void CLegacy3D::RenderFrame(void)
     DrawDisplayList(&VROMCache, POLY_STATE_ALPHA);
     DrawDisplayList(&PolyCache, POLY_STATE_ALPHA);
   }
-  glFrontFace(GL_CW); // restore front face
+  glFrontFace(GL_CW);         // restore front face
+  glDisable(GL_STENCIL_TEST); // make sure this is turned off
   
   // Disable VBO client states
   if (fogIntensityLoc != -1)  glDisableVertexAttribArray(fogIntensityLoc);

Src/Graphics/Legacy3D/Legacy3D.h

@@ -36,9 +36,6 @@ namespace Legacy3D {
 
 /******************************************************************************
  Internal Definitions and Data Structures
- 
- NOTE: These should probably be moved inside the Legacy3D namespace at some
- point.
 ******************************************************************************/
 
 // Model caches sort models by alpha (translucency) state
@@ -71,18 +68,26 @@ struct Poly
  * Reference to model polygons stored in a VBO. Each reference has two sets of
  * vertices: normal and alpha. Copies of the model with different texture
  * offsets applied are searchable via the linked list of texture offset states.
+ *
+ * Technically, a model may contain a mix of layered and non-layered polygons
+ * but we can't support that level of granularity in the current engine. The 
+ * useStencil flag is set only when all polygons in a model are layered, and
+ * also may be set when we detect that a polygon is likely to be used as a
+ * shadow. On the actual hardware, these are not stenciled but are most likely
+ * implemented with stipple masks. We cheat here to make shadows look nicer.
  */
 
 struct VBORef
 {
-	unsigned		index[2];		// index of model polygons in VBO
+	unsigned index[2];		// index of model polygons in VBO
-	unsigned		numVerts[2];	// number of vertices
+	unsigned numVerts[2]; // number of vertices
-	unsigned		lutIdx;			// LUT index associated with this model (for fast LUT clearing)
+	unsigned lutIdx;      // LUT index associated with this model (for fast LUT clearing)
 	
 	struct VBORef	*nextTexOffset;	// linked list of models with different texture offset states
-	UINT16			texOffset;		// texture offset data for this model
+	UINT16 texOffset;		          // texture offset data for this model
+	bool useStencil;              // whether to draw with stencil mask ("layered" polygons)
 	
-	CTextureRefs	texRefs;		// unique texture references contained in this model
+	CTextureRefs texRefs; // unique texture references contained in this model
 	
 	/*
 	 * Clear():
@@ -96,6 +101,7 @@ struct VBORef
 		lutIdx = 0;
 		texOffset = 0;
 		nextTexOffset = NULL;
+		useStencil = false;
 		for (int i = 0; i < 2; i++)
 		{
 			index[i] = 0;
@@ -107,34 +113,36 @@ struct VBORef
 // Display list items: model instances and viewport settings
 struct DisplayList
 {
-	bool		isViewport;				// if true, this is a viewport node
+	// Viewport instance data
+	struct ViewportInstance
+	{
+		GLfloat	projectionMatrix[4*4];	// projection matrix
+		GLfloat	lightingParams[6];		  // lighting parameters (see RenderViewport() and vertex shader)
+		GLfloat	spotEllipse[4];			    // spotlight ellipse (see RenderViewport())
+		GLfloat	spotRange[2];			      // Z range
+		GLfloat	spotColor[3];			      // color
+		GLfloat fogParams[5];			      // fog parameters (...)
+		GLint		x, y;					          // viewport coordinates (scaled and in OpenGL format)
+		GLint		width, height;			    // viewport dimensions (scaled for display surface size)
+	};
 	
+	// Model instance data
+	struct ModelInstance
+	{
+		GLfloat		modelViewMatrix[4*4]; // model-view matrix
+		unsigned	index;					      // index in VBO
+		unsigned	numVerts;				      // number of vertices
+		GLint		  frontFace;				    // GL_CW (default), GL_CCW, or -GL_CW to indicate no culling
+		bool      useStencil;           // draw with stencil testing
+	};
+
+  bool isViewport;    // if true, this is a viewport node
 	union
 	{
-		// Viewport data
+	  ModelInstance     Model;
-		struct
+	  ViewportInstance  Viewport;
-		{
-			GLfloat		projectionMatrix[4*4];	// projection matrix
-			GLfloat		lightingParams[6];		// lighting parameters (see RenderViewport() and vertex shader)
-			GLfloat		spotEllipse[4];			// spotlight ellipse (see RenderViewport())
-			GLfloat		spotRange[2];			// Z range
-			GLfloat		spotColor[3];			// color
-			GLfloat		fogParams[5];			// fog parameters (...)
-			GLint		x, y;					// viewport coordinates (scaled and in OpenGL format)
-			GLint		width, height;			// viewport dimensions (scaled for display surface size)
-		} Viewport;
-		
-		// Model data
-		struct
-		{
-			GLfloat		modelViewMatrix[4*4];	// model-view matrix
-			unsigned	index;					// index in VBO
-			unsigned	numVerts;				// number of vertices
-			GLint		frontFace;				// GL_CW (default), GL_CCW, or -GL_CW to indicate no culling
-		} Model;
 	} Data;
-			
+	DisplayList	*next;  // next display list item with the same state (alpha or non-alpha)
-	DisplayList	*next;					// next display list item with the same state (alpha or non-alpha)
 };
 
 /*
@@ -368,7 +376,7 @@ private:
 	bool 			InsertPolygon(ModelCache *cache, const Poly *p);
 	void 			InsertVertex(ModelCache *cache, const Vertex *v, const Poly *p, float normFlip);
 	struct VBORef	*BeginModel(ModelCache *cache);
-	void			EndModel(ModelCache *cache, struct VBORef *Model, int lutIdx, UINT16 texOffset);
+	void			EndModel(ModelCache *cache, struct VBORef *Model, int lutIdx, UINT16 texOffset, bool useStencil);
 	struct VBORef	*CacheModel(ModelCache *cache, int lutIdx, UINT16 texOffset, const UINT32 *data);
 	struct VBORef	*LookUpModel(ModelCache *cache, int lutIdx, UINT16 texOffset);
 	void 			ClearModelCache(ModelCache *cache);

Src/Graphics/Legacy3D/Models.cpp

@@ -230,6 +230,8 @@ void CLegacy3D::DrawDisplayList(ModelCache *Cache, POLY_STATE state)
   {
     glDisable(GL_BLEND);
   }
+  bool stencilEnabled = false;
+  glDisable(GL_STENCIL_TEST);
   
   // Draw if there are items in the list
   const DisplayList *D = Cache->ListHead[state];
@@ -255,24 +257,34 @@ void CLegacy3D::DrawDisplayList(ModelCache *Cache, POLY_STATE state)
     }
     else
     {
-      if (D->Data.Model.frontFace == -GL_CW)  // no backface culling (all normals have lost their Z component)
+      const DisplayList::ModelInstance &Model = D->Data.Model;
-        glDisable(GL_CULL_FACE);
+      if (stencilEnabled != Model.useStencil)
-      else                  // use appropriate winding convention
       {
+        if (Model.useStencil)
+          glEnable(GL_STENCIL_TEST);
+        else
+          glDisable(GL_STENCIL_TEST);
+        stencilEnabled = Model.useStencil;
+      }
+      if (Model.frontFace == -GL_CW)
+      {
+        // No backface culling (all normals have lost their Z component)
+        glDisable(GL_CULL_FACE);
+      }
+      else
+      {
+        // Use appropriate winding convention
         GLint frontFace;
         glGetIntegerv(GL_FRONT_FACE, &frontFace);
-        if (frontFace != D->Data.Model.frontFace)
+        if (frontFace != Model.frontFace)
-          glFrontFace(D->Data.Model.frontFace);
+          glFrontFace(Model.frontFace);
       }
-            
       if (modelViewMatrixLoc != -1)
-        glUniformMatrix4fv(modelViewMatrixLoc, 1, GL_FALSE, D->Data.Model.modelViewMatrix);
+        glUniformMatrix4fv(modelViewMatrixLoc, 1, GL_FALSE, Model.modelViewMatrix);
-      glDrawArrays(GL_TRIANGLES, D->Data.Model.index, D->Data.Model.numVerts);
+      glDrawArrays(GL_TRIANGLES, Model.index, Model.numVerts);
-      
+      if (Model.frontFace == -GL_CW)
-      if (D->Data.Model.frontFace == -GL_CW)
+        glEnable(GL_CULL_FACE);
-            glEnable(GL_CULL_FACE);
     }
-    
     D = D->next;
   }
 }
@@ -318,6 +330,9 @@ bool CLegacy3D::AppendDisplayList(ModelCache *Cache, bool isViewport, const stru
       Cache->List[lm].Data.Model.index = Model->index[i];
       Cache->List[lm].Data.Model.numVerts = Model->numVerts[i];
 
+      // Misc. parameters
+      Cache->List[lm].Data.Model.useStencil = Model->useStencil;
+      
       // Copy modelview matrix
       glGetFloatv(GL_MODELVIEW_MATRIX, Cache->List[lm].Data.Model.modelViewMatrix);
       
@@ -472,15 +487,14 @@ void CLegacy3D::InsertVertex(ModelCache *Cache, const Vertex *V, const Poly *P,
   GLfloat b = 1.0;
   if ((P->header[1]&2) == 0)
   {
-    //size_t sensorColorIdx = ((P->header[4]>>20)&0x7FF) - 0;  // works for Scud
+    //size_t sensorColorIdx = ((P->header[4]>>20)&0x7FF);
-    size_t colorIdx = ((P->header[4]>>8)&0x7FF) - 0;   // works for Daytona2 lights and Scud
+    size_t colorIdx = ((P->header[4]>>8)&0x7FF);
     b = (GLfloat) (polyRAM[m_colorTableAddr+colorIdx]&0xFF) * (1.0f/255.0f);
     g = (GLfloat) ((polyRAM[m_colorTableAddr+colorIdx]>>8)&0xFF) * (1.0f/255.0f);
     r = (GLfloat) ((polyRAM[m_colorTableAddr+colorIdx]>>16)&0xFF) * (1.0f/255.0f);
   }
   else
   {
-    // Colors are 8-bit (almost certainly true, see Star Wars)
     r = (GLfloat) (P->header[4]>>24) * (1.0f/255.0f);
     g = (GLfloat) ((P->header[4]>>16)&0xFF) * (1.0f/255.0f);
     b = (GLfloat) ((P->header[4]>>8)&0xFF) * (1.0f/255.0f);
@@ -892,7 +906,7 @@ struct VBORef *CLegacy3D::BeginModel(ModelCache *Cache)
 }
 
 // Uploads all vertices from the local vertex buffer to the VBO, sets up the VBO reference, updates the LUT
-void CLegacy3D::EndModel(ModelCache *Cache, struct VBORef *Model, int lutIdx, UINT16 texOffset)
+void CLegacy3D::EndModel(ModelCache *Cache, struct VBORef *Model, int lutIdx, UINT16 texOffset, bool useStencil)
 {
   int m = Cache->numModels++;
 
@@ -916,6 +930,9 @@ void CLegacy3D::EndModel(ModelCache *Cache, struct VBORef *Model, int lutIdx, UI
   // Texture offset of this model state
   Model->texOffset = texOffset;
   
+  // Should we use stencil?
+  Model->useStencil = useStencil;
+  
   // Update the LUT and link up to any existing model that already exists here
   if (Cache->lut[lutIdx] >= 0)  // another texture offset state already cached
     Model->nextTexOffset = &(Cache->Models[Cache->lut[lutIdx]]);
@@ -936,10 +953,6 @@ void CLegacy3D::EndModel(ModelCache *Cache, struct VBORef *Model, int lutIdx, UI
 
 struct VBORef *CLegacy3D::CacheModel(ModelCache *Cache, int lutIdx, UINT16 texOffset, const UINT32 *data)
 {
-  // Sega Rally 2 bad models
-  //if (lutIdx == 0x27a1  || lutIdx == 0x21e0)
-  //  return FAIL;
-    
   if (data == NULL)
     return NULL;
     
@@ -951,6 +964,7 @@ struct VBORef *CLegacy3D::CacheModel(ModelCache *Cache, int lutIdx, UINT16 texOf
   // Cache all polygons
   Vertex    Prev[4];  // previous vertices
   int       numPolys = 0;
+  bool      useStencil = true;
   bool      done = false;
   while (!done)
   {    
@@ -982,9 +996,10 @@ struct VBORef *CLegacy3D::CacheModel(ModelCache *Cache, int lutIdx, UINT16 texOf
     GLfloat uvScale   = (P.header[1]&0x40)?1.0f:(1.0f/8.0f);
     
     // Determine whether this is an alpha polygon (TODO: when testing textures, test if texturing enabled? Might not matter)
-    if (((P.header[6]&0x00800000)==0) ||  // translucent polygon
+    int isTranslucent = (P.header[6] & 0x00800000) == 0;
-      (texFormat==7) ||         // RGBA4 texture
+    if (isTranslucent ||
-      (texFormat==4))           // A4L4 texture
+      (texFormat==7) ||   // RGBA4 texture
+      (texFormat==4))     // A4L4 texture
       P.state = POLY_STATE_ALPHA;
     else
       P.state = POLY_STATE_NORMAL;
@@ -1003,6 +1018,18 @@ struct VBORef *CLegacy3D::CacheModel(ModelCache *Cache, int lutIdx, UINT16 texOf
         P.state = POLY_STATE_NORMAL;
     } 
       
+    // Layered polygons are implemented with a stencil buffer. Here, we also
+    // include a hack to detect likely shadow polygons. When not implemented as
+    // layered polygons, games use translucent polygons (which on the actual
+    // hardware are implemented with stipple) without texturing or lighting.
+    // Usually they are also black with the annoying exception of Spikeout.
+    // TODO: If this hack is too permissive and breaks anything, we should make
+    // it a config option.
+    int isLayered = P.header[6] & 0x8;
+    int isLightDisabled = P.header[6] & 0x00010000;
+    bool isProbablyShadow = isLightDisabled && isTranslucent && !texEnable;
+    useStencil &= (isLayered || isProbablyShadow);
+    
     // Decode the texture
     if (texEnable)
     {
@@ -1039,15 +1066,6 @@ struct VBORef *CLegacy3D::CacheModel(ModelCache *Cache, int lutIdx, UINT16 texOf
       UINT32 iz = data[2];
       UINT32 it = data[3];
       
-      /*
-      // Check for bad vertices (Sega Rally 2)
-      if (((ix>>28)==7) || ((iy>>28)==7) || ((iz>>28)==7))
-      {
-        //printf("%X ix=%08X, iy=%08X, iz=%08X\n", lutIdx, ix, iy, iz);
-        goto StopDecoding;
-      }
-      */
-      
       // Decode vertices
       P.Vert[j].x = (GLfloat) (((INT32)ix)>>8) * vertexFactor;
       P.Vert[j].y = (GLfloat) (((INT32)iy)>>8) * vertexFactor;
@@ -1087,7 +1105,7 @@ struct VBORef *CLegacy3D::CacheModel(ModelCache *Cache, int lutIdx, UINT16 texOf
   }
   
   // Finish model and enter it into the LUT
-  EndModel(Cache,Model,lutIdx,texOffset);
+  EndModel(Cache, Model, lutIdx, texOffset, useStencil);
   return Model;
 }