GPUDevice: Support ingesting SPIR-V

Will be transpiled to HLSL -> DXBC for DirectX backends.

scripts/deps/build-dependencies-linux.sh

@@ -67,7 +67,7 @@ ZSTD=1.5.6
 CPUINFO=7524ad504fdcfcf75a18a133da6abd75c5d48053
 DISCORD_RPC=144f3a3f1209994d8d9e8a87964a989cb9911c1e
 LUNASVG=9af1ac7b90658a279b372add52d6f77a4ebb482c
-SHADERC=f60bb80e255144e71776e2ad570d89b78ea2ab4f
+SHADERC=3a655d0f8d3c946efd690edea31e138d4efef417
 SOUNDTOUCH=463ade388f3a51da078dc9ed062bf28e4ba29da7
 SPIRV_CROSS=vulkan-sdk-1.3.290.0
 
@@ -91,7 +91,7 @@ fd6f417fe9e3a071cf1424a5152d926a34c4a3c5070745470be6cf12a404ed79  $LIBBACKTRACE.
 e1351218d270db49c3dddcba04fb2153b09731ea3fa6830e423f5952f44585be  cpuinfo-$CPUINFO.tar.gz
 3eea5ccce6670c126282f1ba4d32c19d486db49a1a5cbfb8d6f48774784d310c  discord-rpc-$DISCORD_RPC.tar.gz
 3998b024b0d442614a9ee270e76e018bb37a17b8c6941212171731123cbbcac7  lunasvg-$LUNASVG.tar.gz
-4c1780b6c65c27c4dcb109f08ab632241c98b77fe2e22be726c151ff514482bf  shaderc-$SHADERC.tar.gz
+93aa93c087aadd2d1e97db58399d5cc8aaca750fd5d9004520c7426a4eb1fa82  shaderc-$SHADERC.tar.gz
 fe45c2af99f6102d2704277d392c1c83b55180a70bfd17fb888cc84a54b70573  soundtouch-$SOUNDTOUCH.tar.gz
 EOF
 

scripts/deps/build-dependencies-mac.sh

@@ -48,7 +48,7 @@ QT=6.7.2
 CPUINFO=7524ad504fdcfcf75a18a133da6abd75c5d48053
 DISCORD_RPC=144f3a3f1209994d8d9e8a87964a989cb9911c1e
 LUNASVG=9af1ac7b90658a279b372add52d6f77a4ebb482c
-SHADERC=f60bb80e255144e71776e2ad570d89b78ea2ab4f
+SHADERC=3a655d0f8d3c946efd690edea31e138d4efef417
 SOUNDTOUCH=463ade388f3a51da078dc9ed062bf28e4ba29da7
 SPIRV_CROSS=vulkan-sdk-1.3.290.0
 
@@ -88,7 +88,7 @@ fb0d1286a35be3583fee34aeb5843c94719e07193bdf1d4d8b0dc14009caef01  qtsvg-everywhe
 e1351218d270db49c3dddcba04fb2153b09731ea3fa6830e423f5952f44585be  cpuinfo-$CPUINFO.tar.gz
 3eea5ccce6670c126282f1ba4d32c19d486db49a1a5cbfb8d6f48774784d310c  discord-rpc-$DISCORD_RPC.tar.gz
 3998b024b0d442614a9ee270e76e018bb37a17b8c6941212171731123cbbcac7  lunasvg-$LUNASVG.tar.gz
-4c1780b6c65c27c4dcb109f08ab632241c98b77fe2e22be726c151ff514482bf  shaderc-$SHADERC.tar.gz
+93aa93c087aadd2d1e97db58399d5cc8aaca750fd5d9004520c7426a4eb1fa82  shaderc-$SHADERC.tar.gz
 fe45c2af99f6102d2704277d392c1c83b55180a70bfd17fb888cc84a54b70573  soundtouch-$SOUNDTOUCH.tar.gz
 EOF
 

scripts/deps/build-dependencies-windows-arm64.bat

@@ -57,7 +57,7 @@ set ZSTD=1.5.6
 set CPUINFO=7524ad504fdcfcf75a18a133da6abd75c5d48053
 set DISCORD_RPC=144f3a3f1209994d8d9e8a87964a989cb9911c1e
 set LUNASVG=9af1ac7b90658a279b372add52d6f77a4ebb482c
-set SHADERC=f60bb80e255144e71776e2ad570d89b78ea2ab4f
+set SHADERC=3a655d0f8d3c946efd690edea31e138d4efef417
 set SOUNDTOUCH=463ade388f3a51da078dc9ed062bf28e4ba29da7
 set SPIRV_CROSS=vulkan-sdk-1.3.290.0
 
@@ -79,7 +79,7 @@ call :downloadfile "zstd-fd5f8106a58601a963ee816e6a57aa7c61fafc53.patch" https:/
 call :downloadfile "cpuinfo-%CPUINFO%.zip" "https://github.com/pytorch/cpuinfo/archive/%CPUINFO%.zip" 13146ae7983d767a678dd01b0d6af591e77cec82babd41264b9164ab808d7d41 || goto error
 call :downloadfile "discord-rpc-%DISCORD_RPC%.zip" "https://github.com/stenzek/discord-rpc/archive/%DISCORD_RPC%.zip" 61e185e75d37b360c314125bcdf4697192d15e2d5209db3306ed6cd736d508b3 || goto error
 call :downloadfile "lunasvg-%LUNASVG%.zip" "https://github.com/stenzek/lunasvg/archive/%LUNASVG%.zip" 1425ec2bda0228b73ffdc70b0dc666fc7d2b69c33eec75a35c4421157c0e220c || goto error
-call :downloadfile "shaderc-%SHADERC%.zip" "https://github.com/stenzek/shaderc/archive/%SHADERC%.zip" d24760f3c20a0ca39dfa85b84b6cf65eee077cd168e7aa08502e60c168ef05f6 || goto error
+call :downloadfile "shaderc-%SHADERC%.zip" "https://github.com/stenzek/shaderc/archive/%SHADERC%.zip" 1fe2da5a003a1954005ab88b668b0d7b0ce1f6a049ae3f0a8b1beb8bac8824e3 || goto error
 call :downloadfile "soundtouch-%SOUNDTOUCH%.zip" "https://github.com/stenzek/soundtouch/archive/%SOUNDTOUCH%.zip" 107a1941181a69abe28018b9ad26fc0218625758ac193bc979abc9e26b7c0c3a || goto error
 
 if not exist SPIRV-Cross\ (

scripts/deps/build-dependencies-windows-x64.bat

@@ -55,7 +55,7 @@ set ZSTD=1.5.6
 set CPUINFO=7524ad504fdcfcf75a18a133da6abd75c5d48053
 set DISCORD_RPC=144f3a3f1209994d8d9e8a87964a989cb9911c1e
 set LUNASVG=9af1ac7b90658a279b372add52d6f77a4ebb482c
-set SHADERC=f60bb80e255144e71776e2ad570d89b78ea2ab4f
+set SHADERC=3a655d0f8d3c946efd690edea31e138d4efef417
 set SOUNDTOUCH=463ade388f3a51da078dc9ed062bf28e4ba29da7
 set SPIRV_CROSS=vulkan-sdk-1.3.290.0
 
@@ -77,7 +77,7 @@ call :downloadfile "zstd-fd5f8106a58601a963ee816e6a57aa7c61fafc53.patch" https:/
 call :downloadfile "cpuinfo-%CPUINFO%.zip" "https://github.com/pytorch/cpuinfo/archive/%CPUINFO%.zip" 13146ae7983d767a678dd01b0d6af591e77cec82babd41264b9164ab808d7d41 || goto error
 call :downloadfile "discord-rpc-%DISCORD_RPC%.zip" "https://github.com/stenzek/discord-rpc/archive/%DISCORD_RPC%.zip" 61e185e75d37b360c314125bcdf4697192d15e2d5209db3306ed6cd736d508b3 || goto error
 call :downloadfile "lunasvg-%LUNASVG%.zip" "https://github.com/stenzek/lunasvg/archive/%LUNASVG%.zip" 1425ec2bda0228b73ffdc70b0dc666fc7d2b69c33eec75a35c4421157c0e220c || goto error
-call :downloadfile "shaderc-%SHADERC%.zip" "https://github.com/stenzek/shaderc/archive/%SHADERC%.zip" d24760f3c20a0ca39dfa85b84b6cf65eee077cd168e7aa08502e60c168ef05f6 || goto error
+call :downloadfile "shaderc-%SHADERC%.zip" "https://github.com/stenzek/shaderc/archive/%SHADERC%.zip" 1fe2da5a003a1954005ab88b668b0d7b0ce1f6a049ae3f0a8b1beb8bac8824e3 || goto error
 call :downloadfile "soundtouch-%SOUNDTOUCH%.zip" "https://github.com/stenzek/soundtouch/archive/%SOUNDTOUCH%.zip" 107a1941181a69abe28018b9ad26fc0218625758ac193bc979abc9e26b7c0c3a || goto error
 
 if not exist SPIRV-Cross\ (

scripts/flatpak/modules/22-shaderc.yaml

@@ -11,7 +11,7 @@ build-options:
 sources:
   - type: git
     url: "https://github.com/stenzek/shaderc.git"
-    commit: "f60bb80e255144e71776e2ad570d89b78ea2ab4f"
+    commit: "3a655d0f8d3c946efd690edea31e138d4efef417"
 cleanup:
   - /bin
   - /include

src/util/gpu_device.cpp

@@ -1244,7 +1244,8 @@ std::unique_ptr<GPUDevice> GPUDevice::CreateDeviceForAPI(RenderAPI api)
   X(shaderc_result_get_num_warnings)                                                                                   \
   X(shaderc_result_get_bytes)                                                                                          \
   X(shaderc_result_get_compilation_status)                                                                             \
-  X(shaderc_result_get_error_message)
+  X(shaderc_result_get_error_message)                                                                                  \
+  X(shaderc_optimize_spv)
 
 #define SPIRV_CROSS_FUNCTIONS(X)                                                                                       \
   X(spvc_context_create)                                                                                               \
@@ -1413,6 +1414,63 @@ void dyn_libs::CloseAll()
 #undef SPIRV_CROSS_FUNCTIONS
 #undef SHADERC_FUNCTIONS
 
+std::optional<DynamicHeapArray<u8>> GPUDevice::OptimizeVulkanSpv(const std::span<const u8> spirv, Error* error)
+{
+  std::optional<DynamicHeapArray<u8>> ret;
+
+  if (spirv.size() < sizeof(u32) * 2)
+  {
+    Error::SetStringView(error, "Invalid SPIR-V input size.");
+    return ret;
+  }
+
+  // Need to set environment based on version.
+  u32 magic_word, spirv_version;
+  shaderc_target_env target_env = shaderc_target_env_vulkan;
+  shaderc_env_version target_version = shaderc_env_version_vulkan_1_0;
+  std::memcpy(&magic_word, spirv.data(), sizeof(magic_word));
+  std::memcpy(&spirv_version, spirv.data() + sizeof(magic_word), sizeof(spirv_version));
+  if (magic_word != 0x07230203u)
+  {
+    Error::SetStringView(error, "Invalid SPIR-V magic word.");
+    return ret;
+  }
+  if (spirv_version < 0x10300)
+    target_version = shaderc_env_version_vulkan_1_0;
+  else
+    target_version = shaderc_env_version_vulkan_1_1;
+
+  if (!dyn_libs::OpenShaderc(error))
+    return ret;
+
+  const shaderc_compile_options_t options = dyn_libs::shaderc_compile_options_initialize();
+  AssertMsg(options, "shaderc_compile_options_initialize() failed");
+  dyn_libs::shaderc_compile_options_set_target_env(options, target_env, target_version);
+  dyn_libs::shaderc_compile_options_set_optimization_level(options, shaderc_optimization_level_performance);
+
+  const shaderc_compilation_result_t result =
+    dyn_libs::shaderc_optimize_spv(dyn_libs::s_shaderc_compiler, spirv.data(), spirv.size(), options);
+  const shaderc_compilation_status status =
+    result ? dyn_libs::shaderc_result_get_compilation_status(result) : shaderc_compilation_status_internal_error;
+  if (status != shaderc_compilation_status_success)
+  {
+    const std::string_view errors(result ? dyn_libs::shaderc_result_get_error_message(result) : "null result object");
+    Error::SetStringFmt(error, "Failed to optimize SPIR-V: {}\n{}",
+                        dyn_libs::shaderc_compilation_status_to_string(status), errors);
+  }
+  else
+  {
+    const size_t spirv_size = dyn_libs::shaderc_result_get_length(result);
+    DebugAssert(spirv_size > 0);
+    ret = DynamicHeapArray<u8>(spirv_size);
+    std::memcpy(ret->data(), dyn_libs::shaderc_result_get_bytes(result), spirv_size);
+  }
+
+  dyn_libs::shaderc_result_release(result);
+  dyn_libs::shaderc_compile_options_release(options);
+  return ret;
+}
+
 bool GPUDevice::CompileGLSLShaderToVulkanSpv(GPUShaderStage stage, GPUShaderLanguage source_language,
                                              std::string_view source, const char* entry_point, bool optimization,
                                              bool nonsemantic_debug_info, DynamicHeapArray<u8>* out_binary,
@@ -1548,6 +1606,8 @@ bool GPUDevice::TranslateVulkanSpvToLanguage(const std::span<const u8> spirv, GP
   }
 
   [[maybe_unused]] const SpvExecutionModel execmodel = dyn_libs::spvc_compiler_get_execution_model(scompiler);
+  [[maybe_unused]] static constexpr u32 UBO_DESCRIPTOR_SET = 0;
+  [[maybe_unused]] static constexpr u32 TEXTURE_DESCRIPTOR_SET = 1;
 
   switch (target_language)
   {
@@ -1572,11 +1632,19 @@ bool GPUDevice::TranslateVulkanSpvToLanguage(const std::span<const u8> spirv, GP
         return {};
       }
 
-      u32 start_set = 0;
+      if ((sres = dyn_libs::spvc_compiler_options_set_bool(soptions, SPVC_COMPILER_OPTION_HLSL_POINT_SIZE_COMPAT,
+                                                           true)) != SPVC_SUCCESS)
+      {
+        Error::SetStringFmt(error,
+                            "spvc_compiler_options_set_bool(SPVC_COMPILER_OPTION_HLSL_POINT_SIZE_COMPAT) failed: {}",
+                            static_cast<int>(sres));
+        return {};
+      }
+
       if (ubos_count > 0)
       {
         const spvc_hlsl_resource_binding rb = {.stage = execmodel,
-                                               .desc_set = start_set++,
+                                               .desc_set = UBO_DESCRIPTOR_SET,
                                                .binding = 0,
                                                .cbv = {.register_space = 0, .register_binding = 0}};
         if ((sres = dyn_libs::spvc_compiler_hlsl_add_resource_binding(scompiler, &rb)) != SPVC_SUCCESS)
@@ -1588,10 +1656,10 @@ bool GPUDevice::TranslateVulkanSpvToLanguage(const std::span<const u8> spirv, GP
 
       if (textures_count > 0)
       {
-        for (u32 i = 0; i < MAX_TEXTURE_SAMPLERS; i++)
+        for (u32 i = 0; i < textures_count; i++)
         {
           const spvc_hlsl_resource_binding rb = {.stage = execmodel,
-                                                 .desc_set = start_set++,
+                                                 .desc_set = TEXTURE_DESCRIPTOR_SET,
                                                  .binding = i,
                                                  .srv = {.register_space = 0, .register_binding = i},
                                                  .sampler = {.register_space = 0, .register_binding = i}};
@@ -1740,13 +1808,49 @@ std::unique_ptr<GPUShader> GPUDevice::TranspileAndCreateShaderFromSource(
 {
   // Disable optimization when targeting OpenGL GLSL, otherwise, the name-based linking will fail.
   const bool optimization =
-    (target_language != GPUShaderLanguage::GLSL && target_language != GPUShaderLanguage::GLSLES);
+    (!m_debug_device && target_language != GPUShaderLanguage::GLSL && target_language != GPUShaderLanguage::GLSLES);
-  DynamicHeapArray<u8> spv;
+
-  if (!CompileGLSLShaderToVulkanSpv(stage, source_language, source, entry_point, optimization, false, &spv, error))
+  std::span<const u8> spv;
+  DynamicHeapArray<u8> intermediate_spv;
+  if (source_language == GPUShaderLanguage::GLSLVK)
+  {
+    if (!CompileGLSLShaderToVulkanSpv(stage, source_language, source, entry_point, optimization, false,
+                                      &intermediate_spv, error))
+    {
       return {};
+    }
+
+    spv = intermediate_spv.cspan();
+  }
+  else if (source_language == GPUShaderLanguage::SPV)
+  {
+    spv = std::span<const u8>(reinterpret_cast<const u8*>(source.data()), source.size());
+
+    if (optimization)
+    {
+      Error optimize_error;
+      std::optional<DynamicHeapArray<u8>> optimized_spv = GPUDevice::OptimizeVulkanSpv(spv, &optimize_error);
+      if (!optimized_spv.has_value())
+      {
+        WARNING_LOG("Failed to optimize SPIR-V: {}", optimize_error.GetDescription());
+      }
+      else
+      {
+        DEV_LOG("SPIR-V optimized from {} bytes to {} bytes", source.length(), optimized_spv->size());
+        intermediate_spv = std::move(optimized_spv.value());
+        spv = intermediate_spv.cspan();
+      }
+    }
+  }
+  else
+  {
+    Error::SetStringFmt(error, "Unsupported source language for transpile: {}",
+                        ShaderLanguageToString(source_language));
+    return {};
+  }
 
   std::string dest_source;
-  if (!TranslateVulkanSpvToLanguage(spv.cspan(), stage, target_language, target_version, &dest_source, error))
+  if (!TranslateVulkanSpvToLanguage(spv, stage, target_language, target_version, &dest_source, error))
     return {};
 
   // TODO: MSL needs entry point suffixed.

src/util/gpu_device.h

@@ -773,6 +773,7 @@ protected:
                                                                 std::string_view source, const char* entry_point,
                                                                 GPUShaderLanguage target_language, u32 target_version,
                                                                 DynamicHeapArray<u8>* out_binary, Error* error);
+  static std::optional<DynamicHeapArray<u8>> OptimizeVulkanSpv(const std::span<const u8> spirv, Error* error);
 
   Features m_features = {};
   u32 m_max_texture_size = 0;

src/util/vulkan_pipeline.cpp

@@ -60,6 +60,19 @@ std::unique_ptr<GPUShader> VulkanDevice::CreateShaderFromSource(GPUShaderStage s
 {
   if (language == GPUShaderLanguage::SPV)
   {
+    // Optimize the SPIR-V if we're not using a debug device.
+    std::optional<DynamicHeapArray<u8>> optimized_spv;
+    if (!m_debug_device)
+    {
+      Error optimize_error;
+      optimized_spv = GPUDevice::OptimizeVulkanSpv(
+        std::span<const u8>(reinterpret_cast<const u8*>(source.data()), source.size()), &optimize_error);
+      if (!optimized_spv.has_value())
+        WARNING_LOG("Failed to optimize SPIR-V: {}", optimize_error.GetDescription());
+      else
+        source = std::string_view(reinterpret_cast<const char*>(optimized_spv->data()), optimized_spv->size());
+    }
+
     if (out_binary)
       out_binary->assign(reinterpret_cast<const u8*>(source.data()), source.length());