From 5302f8381807c3736a9d439bd6727404ba8dbd59 Mon Sep 17 00:00:00 2001
From: Connor McLaughlin <stenzek@gmail.com>
Date: Sat, 4 Apr 2020 00:10:45 +1000
Subject: [PATCH] GPU: Clamp coordinates to 11 bits after applying drawing
 offset

Fixes flickering in some scenes (e.g. Galbadia Missile Base).
---
 src/core/gpu.h      |  9 +++++++--
 src/core/gpu_hw.h   |  4 ++--
 src/core/gpu_sw.cpp | 20 ++++++++++----------
 3 files changed, 19 insertions(+), 14 deletions(-)

diff --git a/src/core/gpu.h b/src/core/gpu.h
index 8f6a22416..52e9a5a4b 100644
--- a/src/core/gpu.h
+++ b/src/core/gpu.h
@@ -251,10 +251,15 @@ protected:
   {
     u32 bits;
 
-    BitField<u32, s32, 0, 12> x;
-    BitField<u32, s32, 16, 12> y;
+    BitField<u32, s32, 0, 11> x;
+    BitField<u32, s32, 16, 11> y;
   };
 
+  // Vertices have to be clamped to 11 bits before rendering. Normally this would happen as part of the scanline,
+  // but in the hardware renderers we'll do it at the vertex. FF8 is a good test case here, once you go too far left
+  // in the first scene of Galbadia Missile Base, the screen will flicker.
+  ALWAYS_INLINE static s32 VertexPositionToVRAMCoordinate(s32 x) { return SignExtendN<11, s32>(x); }
+
   union VRAMPixel
   {
     u16 bits;
diff --git a/src/core/gpu_hw.h b/src/core/gpu_hw.h
index 3a6b69c88..649229386 100644
--- a/src/core/gpu_hw.h
+++ b/src/core/gpu_hw.h
@@ -61,8 +61,8 @@ protected:
 
     ALWAYS_INLINE void Set(s32 x_, s32 y_, u32 color_, u32 texpage_, u16 u_, u16 v_)
     {
-      x = x_;
-      y = y_;
+      x = VertexPositionToVRAMCoordinate(x_);
+      y = VertexPositionToVRAMCoordinate(y_);
       color = color_;
       texpage = texpage_;
       u = u_;
diff --git a/src/core/gpu_sw.cpp b/src/core/gpu_sw.cpp
index d5c29f0ad..5b574a012 100644
--- a/src/core/gpu_sw.cpp
+++ b/src/core/gpu_sw.cpp
@@ -303,12 +303,12 @@ void GPU_SW::DrawTriangle(const SWVertex* v0, const SWVertex* v1, const SWVertex
   if (IsClockwiseWinding(v0, v1, v2))
     std::swap(v1, v2);
 
-  const s32 px0 = v0->x + m_drawing_offset.x;
-  const s32 py0 = v0->y + m_drawing_offset.y;
-  const s32 px1 = v1->x + m_drawing_offset.x;
-  const s32 py1 = v1->y + m_drawing_offset.y;
-  const s32 px2 = v2->x + m_drawing_offset.x;
-  const s32 py2 = v2->y + m_drawing_offset.y;
+  const s32 px0 = VertexPositionToVRAMCoordinate(v0->x + m_drawing_offset.x);
+  const s32 py0 = VertexPositionToVRAMCoordinate(v0->y + m_drawing_offset.y);
+  const s32 px1 = VertexPositionToVRAMCoordinate(v1->x + m_drawing_offset.x);
+  const s32 py1 = VertexPositionToVRAMCoordinate(v1->y + m_drawing_offset.y);
+  const s32 px2 = VertexPositionToVRAMCoordinate(v2->x + m_drawing_offset.x);
+  const s32 py2 = VertexPositionToVRAMCoordinate(v2->y + m_drawing_offset.y);
 
   // Barycentric coordinates at minX/minY corner
   const s32 ws = orient2d(px0, py0, px1, py1, px2, py2);
@@ -425,7 +425,7 @@ void GPU_SW::DrawRectangle(s32 origin_x, s32 origin_y, u32 width, u32 height, u8
 
   for (u32 offset_y = 0; offset_y < height; offset_y++)
   {
-    const s32 y = origin_y + static_cast<s32>(offset_y);
+    const s32 y = VertexPositionToVRAMCoordinate(origin_y + static_cast<s32>(offset_y));
     if (y < static_cast<s32>(m_drawing_area.top) || y > static_cast<s32>(m_drawing_area.bottom))
       continue;
 
@@ -433,7 +433,7 @@ void GPU_SW::DrawRectangle(s32 origin_x, s32 origin_y, u32 width, u32 height, u8
 
     for (u32 offset_x = 0; offset_x < width; offset_x++)
     {
-      const s32 x = origin_x + static_cast<s32>(offset_x);
+      const s32 x = VertexPositionToVRAMCoordinate(origin_x + static_cast<s32>(offset_x));
       if (x < static_cast<s32>(m_drawing_area.left) || x > static_cast<s32>(m_drawing_area.right))
         continue;
 
@@ -649,8 +649,8 @@ void GPU_SW::DrawLine(const SWVertex* p0, const SWVertex* p1)
 
   for (s32 i = 0; i <= k; i++)
   {
-    const s32 x = m_drawing_offset.x + FixedToIntCoord(current_x);
-    const s32 y = m_drawing_offset.y + FixedToIntCoord(current_y);
+    const s32 x = VertexPositionToVRAMCoordinate(m_drawing_offset.x + FixedToIntCoord(current_x));
+    const s32 y = VertexPositionToVRAMCoordinate(m_drawing_offset.y + FixedToIntCoord(current_y));
 
     const u8 r = shading_enable ? FixedColorToInt(current_r) : p0->color_r;
     const u8 g = shading_enable ? FixedColorToInt(current_g) : p0->color_g;