GPU/HW: Fix crash with oversized writes and sw readback

src/core/gpu_hw.cpp

@@ -1092,30 +1092,6 @@ void GPU_HW::ResetBatchVertexDepth()
   m_current_depth = 1;
 }
 
-void GPU_HW::FillBackendCommandParameters(GPUBackendCommand* cmd) const
-{
-  cmd->params.bits = 0;
-  cmd->params.check_mask_before_draw = m_GPUSTAT.check_mask_before_draw;
-  cmd->params.set_mask_while_drawing = m_GPUSTAT.set_mask_while_drawing;
-  cmd->params.active_line_lsb = m_crtc_state.active_line_lsb;
-  cmd->params.interlaced_rendering = m_GPUSTAT.SkipDrawingToActiveField();
-}
-
-void GPU_HW::FillDrawCommand(GPUBackendDrawCommand* cmd, GPURenderCommand rc) const
-{
-  FillBackendCommandParameters(cmd);
-  cmd->rc.bits = rc.bits;
-  cmd->draw_mode.bits = m_draw_mode.mode_reg.bits;
-  cmd->palette.bits = m_draw_mode.palette_reg;
-  cmd->window = m_draw_mode.texture_window;
-}
-
-void GPU_HW::HandleVRAMReadWithSoftwareRenderer(u32 x, u32 y, u32 width, u32 height)
-{
-  DebugAssert(m_sw_renderer);
-  m_sw_renderer->Sync(false);
-}
-
 void GPU_HW::UpdateSoftwareRenderer(bool copy_vram_from_hw)
 {
   const bool current_enabled = (m_sw_renderer != nullptr);
@@ -1154,13 +1130,48 @@ void GPU_HW::UpdateSoftwareRenderer(bool copy_vram_from_hw)
   m_vram_ptr = m_sw_renderer->GetVRAM();
 }
 
-void GPU_HW::FillVRAM(u32 x, u32 y, u32 width, u32 height, u32 color)
+void GPU_HW::FillBackendCommandParameters(GPUBackendCommand* cmd) const
 {
-  IncludeVRAMDirtyRectangle(
+  cmd->params.bits = 0;
-    Common::Rectangle<u32>::FromExtents(x, y, width, height).Clamped(0, 0, VRAM_WIDTH, VRAM_HEIGHT));
+  cmd->params.check_mask_before_draw = m_GPUSTAT.check_mask_before_draw;
+  cmd->params.set_mask_while_drawing = m_GPUSTAT.set_mask_while_drawing;
+  cmd->params.active_line_lsb = m_crtc_state.active_line_lsb;
+  cmd->params.interlaced_rendering = m_GPUSTAT.SkipDrawingToActiveField();
+}
 
-  if (m_sw_renderer)
+void GPU_HW::FillDrawCommand(GPUBackendDrawCommand* cmd, GPURenderCommand rc) const
-  {
+{
+  FillBackendCommandParameters(cmd);
+  cmd->rc.bits = rc.bits;
+  cmd->draw_mode.bits = m_draw_mode.mode_reg.bits;
+  cmd->palette.bits = m_draw_mode.palette_reg;
+  cmd->window = m_draw_mode.texture_window;
+}
+
+void GPU_HW::ReadSoftwareRendererVRAM(u32 x, u32 y, u32 width, u32 height)
+{
+  DebugAssert(m_sw_renderer);
+  m_sw_renderer->Sync(false);
+}
+
+void GPU_HW::UpdateSoftwareRendererVRAM(u32 x, u32 y, u32 width, u32 height, const void* data, bool set_mask,
+                                        bool check_mask)
+{
+  const u32 num_words = width * height;
+  GPUBackendUpdateVRAMCommand* cmd = m_sw_renderer->NewUpdateVRAMCommand(num_words);
+  FillBackendCommandParameters(cmd);
+  cmd->params.set_mask_while_drawing = set_mask;
+  cmd->params.check_mask_before_draw = check_mask;
+  cmd->x = static_cast<u16>(x);
+  cmd->y = static_cast<u16>(y);
+  cmd->width = static_cast<u16>(width);
+  cmd->height = static_cast<u16>(height);
+  std::memcpy(cmd->data, data, sizeof(u16) * num_words);
+  m_sw_renderer->PushCommand(cmd);
+}
+
+void GPU_HW::FillSoftwareRendererVRAM(u32 x, u32 y, u32 width, u32 height, u32 color)
+{
   GPUBackendFillVRAMCommand* cmd = m_sw_renderer->NewFillVRAMCommand();
   FillBackendCommandParameters(cmd);
   cmd->x = static_cast<u16>(x);
@@ -1169,7 +1180,25 @@ void GPU_HW::FillVRAM(u32 x, u32 y, u32 width, u32 height, u32 color)
   cmd->height = static_cast<u16>(height);
   cmd->color = color;
   m_sw_renderer->PushCommand(cmd);
-  }
+}
+
+void GPU_HW::CopySoftwareRendererVRAM(u32 src_x, u32 src_y, u32 dst_x, u32 dst_y, u32 width, u32 height)
+{
+  GPUBackendCopyVRAMCommand* cmd = m_sw_renderer->NewCopyVRAMCommand();
+  FillBackendCommandParameters(cmd);
+  cmd->src_x = static_cast<u16>(src_x);
+  cmd->src_y = static_cast<u16>(src_y);
+  cmd->dst_x = static_cast<u16>(dst_x);
+  cmd->dst_y = static_cast<u16>(dst_y);
+  cmd->width = static_cast<u16>(width);
+  cmd->height = static_cast<u16>(height);
+  m_sw_renderer->PushCommand(cmd);
+}
+
+void GPU_HW::FillVRAM(u32 x, u32 y, u32 width, u32 height, u32 color)
+{
+  IncludeVRAMDirtyRectangle(
+    Common::Rectangle<u32>::FromExtents(x, y, width, height).Clamped(0, 0, VRAM_WIDTH, VRAM_HEIGHT));
 }
 
 void GPU_HW::UpdateVRAM(u32 x, u32 y, u32 width, u32 height, const void* data, bool set_mask, bool check_mask)
@@ -1182,21 +1211,6 @@ void GPU_HW::UpdateVRAM(u32 x, u32 y, u32 width, u32 height, const void* data, b
     // set new vertex counter since we want this to take into consideration previous masked pixels
     m_current_depth++;
   }
-
-  if (m_sw_renderer)
-  {
-    const u32 num_words = width * height;
-    GPUBackendUpdateVRAMCommand* cmd = m_sw_renderer->NewUpdateVRAMCommand(num_words);
-    FillBackendCommandParameters(cmd);
-    cmd->params.set_mask_while_drawing = set_mask;
-    cmd->params.check_mask_before_draw = check_mask;
-    cmd->x = static_cast<u16>(x);
-    cmd->y = static_cast<u16>(y);
-    cmd->width = static_cast<u16>(width);
-    cmd->height = static_cast<u16>(height);
-    std::memcpy(cmd->data, data, sizeof(u16) * num_words);
-    m_sw_renderer->PushCommand(cmd);
-  }
 }
 
 void GPU_HW::CopyVRAM(u32 src_x, u32 src_y, u32 dst_x, u32 dst_y, u32 width, u32 height)
@@ -1209,19 +1223,6 @@ void GPU_HW::CopyVRAM(u32 src_x, u32 src_y, u32 dst_x, u32 dst_y, u32 width, u32
     // set new vertex counter since we want this to take into consideration previous masked pixels
     m_current_depth++;
   }
-
-  if (m_sw_renderer)
-  {
-    GPUBackendCopyVRAMCommand* cmd = m_sw_renderer->NewCopyVRAMCommand();
-    FillBackendCommandParameters(cmd);
-    cmd->src_x = static_cast<u16>(src_x);
-    cmd->src_y = static_cast<u16>(src_y);
-    cmd->dst_x = static_cast<u16>(dst_x);
-    cmd->dst_y = static_cast<u16>(dst_y);
-    cmd->width = static_cast<u16>(width);
-    cmd->height = static_cast<u16>(height);
-    m_sw_renderer->PushCommand(cmd);
-  }
 }
 
 void GPU_HW::DispatchRenderCommand()

src/core/gpu_hw.h

@@ -262,8 +262,12 @@ protected:
 
   void FillBackendCommandParameters(GPUBackendCommand* cmd) const;
   void FillDrawCommand(GPUBackendDrawCommand* cmd, GPURenderCommand rc) const;
-  void HandleVRAMReadWithSoftwareRenderer(u32 x, u32 y, u32 width, u32 height);
   void UpdateSoftwareRenderer(bool copy_vram_from_hw);
+  void ReadSoftwareRendererVRAM(u32 x, u32 y, u32 width, u32 height);
+  void UpdateSoftwareRendererVRAM(u32 x, u32 y, u32 width, u32 height, const void* data, bool set_mask,
+                                  bool check_mask);
+  void FillSoftwareRendererVRAM(u32 x, u32 y, u32 width, u32 height, u32 color);
+  void CopySoftwareRendererVRAM(u32 src_x, u32 src_y, u32 dst_x, u32 dst_y, u32 width, u32 height);
 
   void FillVRAM(u32 x, u32 y, u32 width, u32 height, u32 color) override;
   void UpdateVRAM(u32 x, u32 y, u32 width, u32 height, const void* data, bool set_mask, bool check_mask) override;

src/core/gpu_hw_d3d11.cpp

@@ -949,7 +949,7 @@ void GPU_HW_D3D11::ReadVRAM(u32 x, u32 y, u32 width, u32 height)
 {
   if (IsUsingSoftwareRendererForReadbacks())
   {
-    HandleVRAMReadWithSoftwareRenderer(x, y, width, height);
+    ReadSoftwareRendererVRAM(x, y, width, height);
     return;
   }
 
@@ -988,6 +988,9 @@ void GPU_HW_D3D11::ReadVRAM(u32 x, u32 y, u32 width, u32 height)
 
 void GPU_HW_D3D11::FillVRAM(u32 x, u32 y, u32 width, u32 height, u32 color)
 {
+  if (IsUsingSoftwareRendererForReadbacks())
+    FillSoftwareRendererVRAM(x, y, width, height, color);
+
   if ((x + width) > VRAM_WIDTH || (y + height) > VRAM_HEIGHT)
   {
     // CPU round trip if oversized for now.
@@ -1015,6 +1018,9 @@ void GPU_HW_D3D11::FillVRAM(u32 x, u32 y, u32 width, u32 height, u32 color)
 
 void GPU_HW_D3D11::UpdateVRAM(u32 x, u32 y, u32 width, u32 height, const void* data, bool set_mask, bool check_mask)
 {
+  if (IsUsingSoftwareRendererForReadbacks())
+    UpdateSoftwareRendererVRAM(x, y, width, height, data, set_mask, check_mask);
+
   const Common::Rectangle<u32> bounds = GetVRAMTransferBounds(x, y, width, height);
   GPU_HW::UpdateVRAM(bounds.left, bounds.top, bounds.GetWidth(), bounds.GetHeight(), data, set_mask, check_mask);
 
@@ -1050,11 +1056,11 @@ void GPU_HW_D3D11::UpdateVRAM(u32 x, u32 y, u32 width, u32 height, const void* d
 
 void GPU_HW_D3D11::CopyVRAM(u32 src_x, u32 src_y, u32 dst_x, u32 dst_y, u32 width, u32 height)
 {
+  if (IsUsingSoftwareRendererForReadbacks())
+    CopySoftwareRendererVRAM(src_x, src_y, dst_x, dst_y, width, height);
+
   if (UseVRAMCopyShader(src_x, src_y, dst_x, dst_y, width, height) || IsUsingMultisampling())
   {
-    if (IsUsingSoftwareRendererForReadbacks())
-      GPU_HW::CopyVRAM(src_x, src_y, dst_x, dst_y, width, height);
-
     const Common::Rectangle<u32> src_bounds = GetVRAMTransferBounds(src_x, src_y, width, height);
     const Common::Rectangle<u32> dst_bounds = GetVRAMTransferBounds(dst_x, dst_y, width, height);
     if (m_vram_dirty_rect.Intersects(src_bounds))

src/core/gpu_hw_opengl.cpp

@@ -984,7 +984,7 @@ void GPU_HW_OpenGL::ReadVRAM(u32 x, u32 y, u32 width, u32 height)
 {
   if (IsUsingSoftwareRendererForReadbacks())
   {
-    HandleVRAMReadWithSoftwareRenderer(x, y, width, height);
+    ReadSoftwareRendererVRAM(x, y, width, height);
     return;
   }
 
@@ -1019,6 +1019,9 @@ void GPU_HW_OpenGL::ReadVRAM(u32 x, u32 y, u32 width, u32 height)
 
 void GPU_HW_OpenGL::FillVRAM(u32 x, u32 y, u32 width, u32 height, u32 color)
 {
+  if (IsUsingSoftwareRendererForReadbacks())
+    FillSoftwareRendererVRAM(x, y, width, height, color);
+
   if ((x + width) > VRAM_WIDTH || (y + height) > VRAM_HEIGHT)
   {
     // CPU round trip if oversized for now.
@@ -1066,6 +1069,9 @@ void GPU_HW_OpenGL::FillVRAM(u32 x, u32 y, u32 width, u32 height, u32 color)
 
 void GPU_HW_OpenGL::UpdateVRAM(u32 x, u32 y, u32 width, u32 height, const void* data, bool set_mask, bool check_mask)
 {
+  if (IsUsingSoftwareRendererForReadbacks())
+    UpdateSoftwareRendererVRAM(x, y, width, height, data, set_mask, check_mask);
+
   const Common::Rectangle<u32> bounds = GetVRAMTransferBounds(x, y, width, height);
   GPU_HW::UpdateVRAM(bounds.left, bounds.top, bounds.GetWidth(), bounds.GetHeight(), data, set_mask, check_mask);
 
@@ -1182,15 +1188,15 @@ void GPU_HW_OpenGL::UpdateVRAM(u32 x, u32 y, u32 width, u32 height, const void*
 
 void GPU_HW_OpenGL::CopyVRAM(u32 src_x, u32 src_y, u32 dst_x, u32 dst_y, u32 width, u32 height)
 {
+  if (IsUsingSoftwareRendererForReadbacks())
+    CopySoftwareRendererVRAM(src_x, src_y, dst_x, dst_y, width, height);
+
   const Common::Rectangle<u32> dst_bounds = GetVRAMTransferBounds(dst_x, dst_y, width, height);
   const Common::Rectangle<u32> src_bounds = GetVRAMTransferBounds(src_x, src_y, width, height);
   const bool src_dirty = m_vram_dirty_rect.Intersects(src_bounds);
 
   if (UseVRAMCopyShader(src_x, src_y, dst_x, dst_y, width, height))
   {
-    if (IsUsingSoftwareRendererForReadbacks())
-      GPU_HW::CopyVRAM(src_x, src_y, dst_x, dst_y, width, height);
-
     if (src_dirty)
       UpdateVRAMReadTexture();
     IncludeVRAMDirtyRectangle(dst_bounds);

src/core/gpu_hw_vulkan.cpp

@@ -1440,7 +1440,7 @@ void GPU_HW_Vulkan::ReadVRAM(u32 x, u32 y, u32 width, u32 height)
 {
   if (IsUsingSoftwareRendererForReadbacks())
   {
-    HandleVRAMReadWithSoftwareRenderer(x, y, width, height);
+    ReadSoftwareRendererVRAM(x, y, width, height);
     return;
   }
 
@@ -1489,6 +1489,9 @@ void GPU_HW_Vulkan::ReadVRAM(u32 x, u32 y, u32 width, u32 height)
 
 void GPU_HW_Vulkan::FillVRAM(u32 x, u32 y, u32 width, u32 height, u32 color)
 {
+  if (IsUsingSoftwareRendererForReadbacks())
+    FillSoftwareRendererVRAM(x, y, width, height, color);
+
   if ((x + width) > VRAM_WIDTH || (y + height) > VRAM_HEIGHT)
   {
     // CPU round trip if oversized for now.
@@ -1522,6 +1525,9 @@ void GPU_HW_Vulkan::FillVRAM(u32 x, u32 y, u32 width, u32 height, u32 color)
 
 void GPU_HW_Vulkan::UpdateVRAM(u32 x, u32 y, u32 width, u32 height, const void* data, bool set_mask, bool check_mask)
 {
+  if (IsUsingSoftwareRendererForReadbacks())
+    UpdateSoftwareRendererVRAM(x, y, width, height, data, set_mask, check_mask);
+
   const Common::Rectangle<u32> bounds = GetVRAMTransferBounds(x, y, width, height);
   GPU_HW::UpdateVRAM(bounds.left, bounds.top, bounds.GetWidth(), bounds.GetHeight(), data, set_mask, check_mask);
 
@@ -1576,11 +1582,11 @@ void GPU_HW_Vulkan::UpdateVRAM(u32 x, u32 y, u32 width, u32 height, const void*
 
 void GPU_HW_Vulkan::CopyVRAM(u32 src_x, u32 src_y, u32 dst_x, u32 dst_y, u32 width, u32 height)
 {
+  if (IsUsingSoftwareRendererForReadbacks())
+    CopySoftwareRendererVRAM(src_x, src_y, dst_x, dst_y, width, height);
+
   if (UseVRAMCopyShader(src_x, src_y, dst_x, dst_y, width, height) || IsUsingMultisampling())
   {
-    if (IsUsingSoftwareRendererForReadbacks())
-      GPU_HW::CopyVRAM(src_x, src_y, dst_x, dst_y, width, height);
-
     const Common::Rectangle<u32> src_bounds = GetVRAMTransferBounds(src_x, src_y, width, height);
     const Common::Rectangle<u32> dst_bounds = GetVRAMTransferBounds(dst_x, dst_y, width, height);
     if (m_vram_dirty_rect.Intersects(src_bounds))