From 4ec5ca4319dfd2d486397064ca55f4cca8c99e8c Mon Sep 17 00:00:00 2001 From: Stenzek Date: Sun, 8 Sep 2024 22:11:58 +1000 Subject: [PATCH] GPUDevice: Support ingesting SPIR-V Will be transpiled to HLSL -> DXBC for DirectX backends. --- scripts/deps/build-dependencies-linux.sh | 4 +- scripts/deps/build-dependencies-mac.sh | 4 +- .../deps/build-dependencies-windows-arm64.bat | 4 +- .../deps/build-dependencies-windows-x64.bat | 4 +- scripts/flatpak/modules/22-shaderc.yaml | 2 +- src/util/gpu_device.cpp | 122 ++++++++++++++++-- src/util/gpu_device.h | 1 + src/util/vulkan_pipeline.cpp | 13 ++ 8 files changed, 136 insertions(+), 18 deletions(-) diff --git a/scripts/deps/build-dependencies-linux.sh b/scripts/deps/build-dependencies-linux.sh index c1b67238b..cbf204129 100755 --- a/scripts/deps/build-dependencies-linux.sh +++ b/scripts/deps/build-dependencies-linux.sh @@ -67,7 +67,7 @@ ZSTD=1.5.6 CPUINFO=7524ad504fdcfcf75a18a133da6abd75c5d48053 DISCORD_RPC=144f3a3f1209994d8d9e8a87964a989cb9911c1e LUNASVG=9af1ac7b90658a279b372add52d6f77a4ebb482c -SHADERC=f60bb80e255144e71776e2ad570d89b78ea2ab4f +SHADERC=3a655d0f8d3c946efd690edea31e138d4efef417 SOUNDTOUCH=463ade388f3a51da078dc9ed062bf28e4ba29da7 SPIRV_CROSS=vulkan-sdk-1.3.290.0 @@ -91,7 +91,7 @@ fd6f417fe9e3a071cf1424a5152d926a34c4a3c5070745470be6cf12a404ed79 $LIBBACKTRACE. e1351218d270db49c3dddcba04fb2153b09731ea3fa6830e423f5952f44585be cpuinfo-$CPUINFO.tar.gz 3eea5ccce6670c126282f1ba4d32c19d486db49a1a5cbfb8d6f48774784d310c discord-rpc-$DISCORD_RPC.tar.gz 3998b024b0d442614a9ee270e76e018bb37a17b8c6941212171731123cbbcac7 lunasvg-$LUNASVG.tar.gz -4c1780b6c65c27c4dcb109f08ab632241c98b77fe2e22be726c151ff514482bf shaderc-$SHADERC.tar.gz +93aa93c087aadd2d1e97db58399d5cc8aaca750fd5d9004520c7426a4eb1fa82 shaderc-$SHADERC.tar.gz fe45c2af99f6102d2704277d392c1c83b55180a70bfd17fb888cc84a54b70573 soundtouch-$SOUNDTOUCH.tar.gz EOF diff --git a/scripts/deps/build-dependencies-mac.sh b/scripts/deps/build-dependencies-mac.sh index 02a310acc..b2cfbe71f 100755 --- a/scripts/deps/build-dependencies-mac.sh +++ b/scripts/deps/build-dependencies-mac.sh @@ -48,7 +48,7 @@ QT=6.7.2 CPUINFO=7524ad504fdcfcf75a18a133da6abd75c5d48053 DISCORD_RPC=144f3a3f1209994d8d9e8a87964a989cb9911c1e LUNASVG=9af1ac7b90658a279b372add52d6f77a4ebb482c -SHADERC=f60bb80e255144e71776e2ad570d89b78ea2ab4f +SHADERC=3a655d0f8d3c946efd690edea31e138d4efef417 SOUNDTOUCH=463ade388f3a51da078dc9ed062bf28e4ba29da7 SPIRV_CROSS=vulkan-sdk-1.3.290.0 @@ -88,7 +88,7 @@ fb0d1286a35be3583fee34aeb5843c94719e07193bdf1d4d8b0dc14009caef01 qtsvg-everywhe e1351218d270db49c3dddcba04fb2153b09731ea3fa6830e423f5952f44585be cpuinfo-$CPUINFO.tar.gz 3eea5ccce6670c126282f1ba4d32c19d486db49a1a5cbfb8d6f48774784d310c discord-rpc-$DISCORD_RPC.tar.gz 3998b024b0d442614a9ee270e76e018bb37a17b8c6941212171731123cbbcac7 lunasvg-$LUNASVG.tar.gz -4c1780b6c65c27c4dcb109f08ab632241c98b77fe2e22be726c151ff514482bf shaderc-$SHADERC.tar.gz +93aa93c087aadd2d1e97db58399d5cc8aaca750fd5d9004520c7426a4eb1fa82 shaderc-$SHADERC.tar.gz fe45c2af99f6102d2704277d392c1c83b55180a70bfd17fb888cc84a54b70573 soundtouch-$SOUNDTOUCH.tar.gz EOF diff --git a/scripts/deps/build-dependencies-windows-arm64.bat b/scripts/deps/build-dependencies-windows-arm64.bat index 01ad376e5..63afaeb8e 100644 --- a/scripts/deps/build-dependencies-windows-arm64.bat +++ b/scripts/deps/build-dependencies-windows-arm64.bat @@ -57,7 +57,7 @@ set ZSTD=1.5.6 set CPUINFO=7524ad504fdcfcf75a18a133da6abd75c5d48053 set DISCORD_RPC=144f3a3f1209994d8d9e8a87964a989cb9911c1e set LUNASVG=9af1ac7b90658a279b372add52d6f77a4ebb482c -set SHADERC=f60bb80e255144e71776e2ad570d89b78ea2ab4f +set SHADERC=3a655d0f8d3c946efd690edea31e138d4efef417 set SOUNDTOUCH=463ade388f3a51da078dc9ed062bf28e4ba29da7 set SPIRV_CROSS=vulkan-sdk-1.3.290.0 @@ -79,7 +79,7 @@ call :downloadfile "zstd-fd5f8106a58601a963ee816e6a57aa7c61fafc53.patch" https:/ call :downloadfile "cpuinfo-%CPUINFO%.zip" "https://github.com/pytorch/cpuinfo/archive/%CPUINFO%.zip" 13146ae7983d767a678dd01b0d6af591e77cec82babd41264b9164ab808d7d41 || goto error call :downloadfile "discord-rpc-%DISCORD_RPC%.zip" "https://github.com/stenzek/discord-rpc/archive/%DISCORD_RPC%.zip" 61e185e75d37b360c314125bcdf4697192d15e2d5209db3306ed6cd736d508b3 || goto error call :downloadfile "lunasvg-%LUNASVG%.zip" "https://github.com/stenzek/lunasvg/archive/%LUNASVG%.zip" 1425ec2bda0228b73ffdc70b0dc666fc7d2b69c33eec75a35c4421157c0e220c || goto error -call :downloadfile "shaderc-%SHADERC%.zip" "https://github.com/stenzek/shaderc/archive/%SHADERC%.zip" d24760f3c20a0ca39dfa85b84b6cf65eee077cd168e7aa08502e60c168ef05f6 || goto error +call :downloadfile "shaderc-%SHADERC%.zip" "https://github.com/stenzek/shaderc/archive/%SHADERC%.zip" 1fe2da5a003a1954005ab88b668b0d7b0ce1f6a049ae3f0a8b1beb8bac8824e3 || goto error call :downloadfile "soundtouch-%SOUNDTOUCH%.zip" "https://github.com/stenzek/soundtouch/archive/%SOUNDTOUCH%.zip" 107a1941181a69abe28018b9ad26fc0218625758ac193bc979abc9e26b7c0c3a || goto error if not exist SPIRV-Cross\ ( diff --git a/scripts/deps/build-dependencies-windows-x64.bat b/scripts/deps/build-dependencies-windows-x64.bat index 5b89ed5c7..c2ed8632b 100644 --- a/scripts/deps/build-dependencies-windows-x64.bat +++ b/scripts/deps/build-dependencies-windows-x64.bat @@ -55,7 +55,7 @@ set ZSTD=1.5.6 set CPUINFO=7524ad504fdcfcf75a18a133da6abd75c5d48053 set DISCORD_RPC=144f3a3f1209994d8d9e8a87964a989cb9911c1e set LUNASVG=9af1ac7b90658a279b372add52d6f77a4ebb482c -set SHADERC=f60bb80e255144e71776e2ad570d89b78ea2ab4f +set SHADERC=3a655d0f8d3c946efd690edea31e138d4efef417 set SOUNDTOUCH=463ade388f3a51da078dc9ed062bf28e4ba29da7 set SPIRV_CROSS=vulkan-sdk-1.3.290.0 @@ -77,7 +77,7 @@ call :downloadfile "zstd-fd5f8106a58601a963ee816e6a57aa7c61fafc53.patch" https:/ call :downloadfile "cpuinfo-%CPUINFO%.zip" "https://github.com/pytorch/cpuinfo/archive/%CPUINFO%.zip" 13146ae7983d767a678dd01b0d6af591e77cec82babd41264b9164ab808d7d41 || goto error call :downloadfile "discord-rpc-%DISCORD_RPC%.zip" "https://github.com/stenzek/discord-rpc/archive/%DISCORD_RPC%.zip" 61e185e75d37b360c314125bcdf4697192d15e2d5209db3306ed6cd736d508b3 || goto error call :downloadfile "lunasvg-%LUNASVG%.zip" "https://github.com/stenzek/lunasvg/archive/%LUNASVG%.zip" 1425ec2bda0228b73ffdc70b0dc666fc7d2b69c33eec75a35c4421157c0e220c || goto error -call :downloadfile "shaderc-%SHADERC%.zip" "https://github.com/stenzek/shaderc/archive/%SHADERC%.zip" d24760f3c20a0ca39dfa85b84b6cf65eee077cd168e7aa08502e60c168ef05f6 || goto error +call :downloadfile "shaderc-%SHADERC%.zip" "https://github.com/stenzek/shaderc/archive/%SHADERC%.zip" 1fe2da5a003a1954005ab88b668b0d7b0ce1f6a049ae3f0a8b1beb8bac8824e3 || goto error call :downloadfile "soundtouch-%SOUNDTOUCH%.zip" "https://github.com/stenzek/soundtouch/archive/%SOUNDTOUCH%.zip" 107a1941181a69abe28018b9ad26fc0218625758ac193bc979abc9e26b7c0c3a || goto error if not exist SPIRV-Cross\ ( diff --git a/scripts/flatpak/modules/22-shaderc.yaml b/scripts/flatpak/modules/22-shaderc.yaml index a94237685..70476e974 100644 --- a/scripts/flatpak/modules/22-shaderc.yaml +++ b/scripts/flatpak/modules/22-shaderc.yaml @@ -11,7 +11,7 @@ build-options: sources: - type: git url: "https://github.com/stenzek/shaderc.git" - commit: "f60bb80e255144e71776e2ad570d89b78ea2ab4f" + commit: "3a655d0f8d3c946efd690edea31e138d4efef417" cleanup: - /bin - /include diff --git a/src/util/gpu_device.cpp b/src/util/gpu_device.cpp index d7b4609c8..770c440a8 100644 --- a/src/util/gpu_device.cpp +++ b/src/util/gpu_device.cpp @@ -1244,7 +1244,8 @@ std::unique_ptr GPUDevice::CreateDeviceForAPI(RenderAPI api) X(shaderc_result_get_num_warnings) \ X(shaderc_result_get_bytes) \ X(shaderc_result_get_compilation_status) \ - X(shaderc_result_get_error_message) + X(shaderc_result_get_error_message) \ + X(shaderc_optimize_spv) #define SPIRV_CROSS_FUNCTIONS(X) \ X(spvc_context_create) \ @@ -1413,6 +1414,63 @@ void dyn_libs::CloseAll() #undef SPIRV_CROSS_FUNCTIONS #undef SHADERC_FUNCTIONS +std::optional> GPUDevice::OptimizeVulkanSpv(const std::span spirv, Error* error) +{ + std::optional> ret; + + if (spirv.size() < sizeof(u32) * 2) + { + Error::SetStringView(error, "Invalid SPIR-V input size."); + return ret; + } + + // Need to set environment based on version. + u32 magic_word, spirv_version; + shaderc_target_env target_env = shaderc_target_env_vulkan; + shaderc_env_version target_version = shaderc_env_version_vulkan_1_0; + std::memcpy(&magic_word, spirv.data(), sizeof(magic_word)); + std::memcpy(&spirv_version, spirv.data() + sizeof(magic_word), sizeof(spirv_version)); + if (magic_word != 0x07230203u) + { + Error::SetStringView(error, "Invalid SPIR-V magic word."); + return ret; + } + if (spirv_version < 0x10300) + target_version = shaderc_env_version_vulkan_1_0; + else + target_version = shaderc_env_version_vulkan_1_1; + + if (!dyn_libs::OpenShaderc(error)) + return ret; + + const shaderc_compile_options_t options = dyn_libs::shaderc_compile_options_initialize(); + AssertMsg(options, "shaderc_compile_options_initialize() failed"); + dyn_libs::shaderc_compile_options_set_target_env(options, target_env, target_version); + dyn_libs::shaderc_compile_options_set_optimization_level(options, shaderc_optimization_level_performance); + + const shaderc_compilation_result_t result = + dyn_libs::shaderc_optimize_spv(dyn_libs::s_shaderc_compiler, spirv.data(), spirv.size(), options); + const shaderc_compilation_status status = + result ? dyn_libs::shaderc_result_get_compilation_status(result) : shaderc_compilation_status_internal_error; + if (status != shaderc_compilation_status_success) + { + const std::string_view errors(result ? dyn_libs::shaderc_result_get_error_message(result) : "null result object"); + Error::SetStringFmt(error, "Failed to optimize SPIR-V: {}\n{}", + dyn_libs::shaderc_compilation_status_to_string(status), errors); + } + else + { + const size_t spirv_size = dyn_libs::shaderc_result_get_length(result); + DebugAssert(spirv_size > 0); + ret = DynamicHeapArray(spirv_size); + std::memcpy(ret->data(), dyn_libs::shaderc_result_get_bytes(result), spirv_size); + } + + dyn_libs::shaderc_result_release(result); + dyn_libs::shaderc_compile_options_release(options); + return ret; +} + bool GPUDevice::CompileGLSLShaderToVulkanSpv(GPUShaderStage stage, GPUShaderLanguage source_language, std::string_view source, const char* entry_point, bool optimization, bool nonsemantic_debug_info, DynamicHeapArray* out_binary, @@ -1548,6 +1606,8 @@ bool GPUDevice::TranslateVulkanSpvToLanguage(const std::span spirv, GP } [[maybe_unused]] const SpvExecutionModel execmodel = dyn_libs::spvc_compiler_get_execution_model(scompiler); + [[maybe_unused]] static constexpr u32 UBO_DESCRIPTOR_SET = 0; + [[maybe_unused]] static constexpr u32 TEXTURE_DESCRIPTOR_SET = 1; switch (target_language) { @@ -1572,11 +1632,19 @@ bool GPUDevice::TranslateVulkanSpvToLanguage(const std::span spirv, GP return {}; } - u32 start_set = 0; + if ((sres = dyn_libs::spvc_compiler_options_set_bool(soptions, SPVC_COMPILER_OPTION_HLSL_POINT_SIZE_COMPAT, + true)) != SPVC_SUCCESS) + { + Error::SetStringFmt(error, + "spvc_compiler_options_set_bool(SPVC_COMPILER_OPTION_HLSL_POINT_SIZE_COMPAT) failed: {}", + static_cast(sres)); + return {}; + } + if (ubos_count > 0) { const spvc_hlsl_resource_binding rb = {.stage = execmodel, - .desc_set = start_set++, + .desc_set = UBO_DESCRIPTOR_SET, .binding = 0, .cbv = {.register_space = 0, .register_binding = 0}}; if ((sres = dyn_libs::spvc_compiler_hlsl_add_resource_binding(scompiler, &rb)) != SPVC_SUCCESS) @@ -1588,10 +1656,10 @@ bool GPUDevice::TranslateVulkanSpvToLanguage(const std::span spirv, GP if (textures_count > 0) { - for (u32 i = 0; i < MAX_TEXTURE_SAMPLERS; i++) + for (u32 i = 0; i < textures_count; i++) { const spvc_hlsl_resource_binding rb = {.stage = execmodel, - .desc_set = start_set++, + .desc_set = TEXTURE_DESCRIPTOR_SET, .binding = i, .srv = {.register_space = 0, .register_binding = i}, .sampler = {.register_space = 0, .register_binding = i}}; @@ -1740,13 +1808,49 @@ std::unique_ptr GPUDevice::TranspileAndCreateShaderFromSource( { // Disable optimization when targeting OpenGL GLSL, otherwise, the name-based linking will fail. const bool optimization = - (target_language != GPUShaderLanguage::GLSL && target_language != GPUShaderLanguage::GLSLES); - DynamicHeapArray spv; - if (!CompileGLSLShaderToVulkanSpv(stage, source_language, source, entry_point, optimization, false, &spv, error)) + (!m_debug_device && target_language != GPUShaderLanguage::GLSL && target_language != GPUShaderLanguage::GLSLES); + + std::span spv; + DynamicHeapArray intermediate_spv; + if (source_language == GPUShaderLanguage::GLSLVK) + { + if (!CompileGLSLShaderToVulkanSpv(stage, source_language, source, entry_point, optimization, false, + &intermediate_spv, error)) + { + return {}; + } + + spv = intermediate_spv.cspan(); + } + else if (source_language == GPUShaderLanguage::SPV) + { + spv = std::span(reinterpret_cast(source.data()), source.size()); + + if (optimization) + { + Error optimize_error; + std::optional> optimized_spv = GPUDevice::OptimizeVulkanSpv(spv, &optimize_error); + if (!optimized_spv.has_value()) + { + WARNING_LOG("Failed to optimize SPIR-V: {}", optimize_error.GetDescription()); + } + else + { + DEV_LOG("SPIR-V optimized from {} bytes to {} bytes", source.length(), optimized_spv->size()); + intermediate_spv = std::move(optimized_spv.value()); + spv = intermediate_spv.cspan(); + } + } + } + else + { + Error::SetStringFmt(error, "Unsupported source language for transpile: {}", + ShaderLanguageToString(source_language)); return {}; + } std::string dest_source; - if (!TranslateVulkanSpvToLanguage(spv.cspan(), stage, target_language, target_version, &dest_source, error)) + if (!TranslateVulkanSpvToLanguage(spv, stage, target_language, target_version, &dest_source, error)) return {}; // TODO: MSL needs entry point suffixed. diff --git a/src/util/gpu_device.h b/src/util/gpu_device.h index baa4e010a..2d46c333f 100644 --- a/src/util/gpu_device.h +++ b/src/util/gpu_device.h @@ -773,6 +773,7 @@ protected: std::string_view source, const char* entry_point, GPUShaderLanguage target_language, u32 target_version, DynamicHeapArray* out_binary, Error* error); + static std::optional> OptimizeVulkanSpv(const std::span spirv, Error* error); Features m_features = {}; u32 m_max_texture_size = 0; diff --git a/src/util/vulkan_pipeline.cpp b/src/util/vulkan_pipeline.cpp index 17bbf32ed..aa0980c02 100644 --- a/src/util/vulkan_pipeline.cpp +++ b/src/util/vulkan_pipeline.cpp @@ -60,6 +60,19 @@ std::unique_ptr VulkanDevice::CreateShaderFromSource(GPUShaderStage s { if (language == GPUShaderLanguage::SPV) { + // Optimize the SPIR-V if we're not using a debug device. + std::optional> optimized_spv; + if (!m_debug_device) + { + Error optimize_error; + optimized_spv = GPUDevice::OptimizeVulkanSpv( + std::span(reinterpret_cast(source.data()), source.size()), &optimize_error); + if (!optimized_spv.has_value()) + WARNING_LOG("Failed to optimize SPIR-V: {}", optimize_error.GetDescription()); + else + source = std::string_view(reinterpret_cast(optimized_spv->data()), optimized_spv->size()); + } + if (out_binary) out_binary->assign(reinterpret_cast(source.data()), source.length());