From bf60f9dd613bf56816283aa33560e3fd3146af68 Mon Sep 17 00:00:00 2001
From: Connor McLaughlin <stenzek@gmail.com>
Date: Sun, 21 Jun 2020 01:33:08 +1000
Subject: [PATCH] GPU/Vulkan: Use geometry shader for line rendering at >1xIR

---
 src/common/vulkan/context.cpp | 10 ++++---
 src/common/vulkan/util.cpp    |  9 +++++++
 src/common/vulkan/util.h      |  3 ++-
 src/core/gpu_hw_vulkan.cpp    | 50 ++++++++++++++++++++++++-----------
 4 files changed, 51 insertions(+), 21 deletions(-)

diff --git a/src/common/vulkan/context.cpp b/src/common/vulkan/context.cpp
index bd861626d..cdebaf2bc 100644
--- a/src/common/vulkan/context.cpp
+++ b/src/common/vulkan/context.cpp
@@ -197,7 +197,6 @@ bool Context::SelectInstanceExtensions(ExtensionList* extension_list, bool enabl
     return false;
 #endif
 
-
   // VK_EXT_debug_report
   if (enable_debug_report && !SupportsExtension(VK_EXT_DEBUG_REPORT_EXTENSION_NAME, false))
     Log_WarningPrintf("Vulkan: Debug report requested, but extension is not available.");
@@ -422,12 +421,15 @@ bool Context::SelectDeviceExtensions(ExtensionList* extension_list, bool enable_
 
 bool Context::SelectDeviceFeatures()
 {
-  VkPhysicalDeviceProperties properties;
-  vkGetPhysicalDeviceProperties(m_physical_device, &properties);
-
   VkPhysicalDeviceFeatures available_features;
   vkGetPhysicalDeviceFeatures(m_physical_device, &available_features);
 
+  if (!available_features.fillModeNonSolid)
+  {
+    Log_ErrorPrintf("fillModeNonSolid feature is required for line drawing.");
+    return false;
+  }
+
   // Enable the features we use.
   m_device_features.dualSrcBlend = available_features.dualSrcBlend;
   m_device_features.geometryShader = available_features.geometryShader;
diff --git a/src/common/vulkan/util.cpp b/src/common/vulkan/util.cpp
index f4ef8b76c..efbe47e67 100644
--- a/src/common/vulkan/util.cpp
+++ b/src/common/vulkan/util.cpp
@@ -178,6 +178,15 @@ void SafeDestroyFramebuffer(VkFramebuffer& fb)
   }
 }
 
+void SafeDestroyShaderModule(VkShaderModule& sm)
+{
+  if (sm != VK_NULL_HANDLE)
+  {
+    vkDestroyShaderModule(g_vulkan_context->GetDevice(), sm, nullptr);
+    sm = VK_NULL_HANDLE;
+  }
+}
+
 void SafeDestroyPipeline(VkPipeline& p)
 {
   if (p != VK_NULL_HANDLE)
diff --git a/src/common/vulkan/util.h b/src/common/vulkan/util.h
index 9a3aed2df..636214c45 100644
--- a/src/common/vulkan/util.h
+++ b/src/common/vulkan/util.h
@@ -35,6 +35,7 @@ VkBlendFactor GetAlphaBlendFactor(VkBlendFactor factor);
 
 // Safe destroy helpers
 void SafeDestroyFramebuffer(VkFramebuffer& fb);
+void SafeDestroyShaderModule(VkShaderModule& sm);
 void SafeDestroyPipeline(VkPipeline& p);
 void SafeDestroyPipelineLayout(VkPipelineLayout& pl);
 void SafeDestroyDescriptorSetLayout(VkDescriptorSetLayout& dsl);
@@ -43,7 +44,7 @@ void SafeDestroySampler(VkSampler& samp);
 void SafeFreeGlobalDescriptorSet(VkDescriptorSet& ds);
 
 void SetViewport(VkCommandBuffer command_buffer, int x, int y, int width, int height, float min_depth = 0.0f,
-                           float max_depth = 1.0f);
+                 float max_depth = 1.0f);
 void SetScissor(VkCommandBuffer command_buffer, int x, int y, int width, int height);
 
 // Combines viewport and scissor updates
diff --git a/src/core/gpu_hw_vulkan.cpp b/src/core/gpu_hw_vulkan.cpp
index 50ec3883d..12180e4d3 100644
--- a/src/core/gpu_hw_vulkan.cpp
+++ b/src/core/gpu_hw_vulkan.cpp
@@ -544,20 +544,13 @@ bool GPU_HW_Vulkan::CompilePipelines()
   // fragment shaders - [render_mode][texture_mode][dithering][interlacing]
   DimensionalArray<VkShaderModule, 2> batch_vertex_shaders{};
   DimensionalArray<VkShaderModule, 2, 2, 9, 4> batch_fragment_shaders{};
-  Common::ScopeGuard batch_shader_guard([&batch_vertex_shaders, &batch_fragment_shaders]() {
-    batch_vertex_shaders.enumerate([](VkShaderModule& s) {
-      if (s != VK_NULL_HANDLE)
-      {
-        vkDestroyShaderModule(g_vulkan_context->GetDevice(), s, nullptr);
-      }
+  VkShaderModule batch_line_geometry_shader = VK_NULL_HANDLE;
+  Common::ScopeGuard batch_shader_guard(
+    [&batch_vertex_shaders, &batch_fragment_shaders, &batch_line_geometry_shader]() {
+      batch_vertex_shaders.enumerate(Vulkan::Util::SafeDestroyShaderModule);
+      batch_fragment_shaders.enumerate(Vulkan::Util::SafeDestroyShaderModule);
+      Vulkan::Util::SafeDestroyShaderModule(batch_line_geometry_shader);
     });
-    batch_fragment_shaders.enumerate([](VkShaderModule& s) {
-      if (s != VK_NULL_HANDLE)
-      {
-        vkDestroyShaderModule(g_vulkan_context->GetDevice(), s, nullptr);
-      }
-    });
-  });
 
   for (u8 textured = 0; textured < 2; textured++)
   {
@@ -591,6 +584,21 @@ bool GPU_HW_Vulkan::CompilePipelines()
     }
   }
 
+  if (m_resolution_scale > 1)
+  {
+    if (g_vulkan_context->GetDeviceFeatures().geometryShader)
+    {
+      const std::string gs = shadergen.GenerateBatchLineExpandGeometryShader();
+      batch_line_geometry_shader = g_vulkan_shader_cache->GetGeometryShader(gs);
+      if (batch_line_geometry_shader == VK_NULL_HANDLE)
+        return false;
+    }
+    else
+    {
+      Log_WarningPrintf("Upscaling requested but geometry shaders are unsupported, line rendering will be incorrect.");
+    }
+  }
+
   Vulkan::GraphicsPipelineBuilder gpbuilder;
 
   // [primitive][depth_test][render_mode][texture_mode][transparency_mode][dithering][interlacing]
@@ -608,7 +616,6 @@ bool GPU_HW_Vulkan::CompilePipelines()
             {
               for (u8 interlacing = 0; interlacing < 2; interlacing++)
               {
-                // TODO: GS
                 const bool textured = (static_cast<TextureMode>(texture_mode) != TextureMode::Disabled);
 
                 gpbuilder.SetPipelineLayout(m_batch_pipeline_layout);
@@ -626,8 +633,19 @@ bool GPU_HW_Vulkan::CompilePipelines()
                 gpbuilder.SetPrimitiveTopology(primitive_mapping[primitive]);
                 gpbuilder.SetVertexShader(batch_vertex_shaders[BoolToUInt8(textured)]);
                 gpbuilder.SetFragmentShader(batch_fragment_shaders[render_mode][texture_mode][dithering][interlacing]);
-                gpbuilder.SetRasterizationState(polygon_mode_mapping[primitive], VK_CULL_MODE_NONE,
-                                                VK_FRONT_FACE_CLOCKWISE);
+                if (static_cast<BatchPrimitive>(primitive) == BatchPrimitive::Lines &&
+                    batch_line_geometry_shader != VK_NULL_HANDLE)
+                {
+                  gpbuilder.SetGeometryShader(batch_line_geometry_shader);
+                  gpbuilder.SetRasterizationState(polygon_mode_mapping[static_cast<u8>(BatchPrimitive::Triangles)],
+                                                  VK_CULL_MODE_NONE, VK_FRONT_FACE_CLOCKWISE);
+                }
+                else
+                {
+                  gpbuilder.SetRasterizationState(polygon_mode_mapping[primitive], VK_CULL_MODE_NONE,
+                                                  VK_FRONT_FACE_CLOCKWISE);
+                }
+
                 gpbuilder.SetDepthState(true, true,
                                         (depth_test != 0) ? VK_COMPARE_OP_GREATER_OR_EQUAL : VK_COMPARE_OP_ALWAYS);