From 03f97089117c420740195d76d10b0ed59e8fd3a6 Mon Sep 17 00:00:00 2001 From: Stenzek Date: Sun, 12 May 2024 22:56:58 +1000 Subject: [PATCH] GPUDevice: Move SPIR-V compilation to base class --- CMakeModules/DuckStationDependencies.cmake | 6 + scripts/appimage/make-appimage.sh | 26 +++- src/util/CMakeLists.txt | 3 +- src/util/gpu_device.cpp | 170 ++++++++++++++++++++- src/util/gpu_device.h | 5 + src/util/metal_device.mm | 48 +----- src/util/util.props | 2 +- src/util/vulkan_pipeline.cpp | 56 +------ 8 files changed, 219 insertions(+), 97 deletions(-) diff --git a/CMakeModules/DuckStationDependencies.cmake b/CMakeModules/DuckStationDependencies.cmake index 219566bc8..f2a261989 100644 --- a/CMakeModules/DuckStationDependencies.cmake +++ b/CMakeModules/DuckStationDependencies.cmake @@ -36,6 +36,12 @@ endif() if(ENABLE_VULKAN OR APPLE) find_package(Shaderc REQUIRED) + + if(LINUX AND ENABLE_VULKAN) + # We need to add the rpath for shaderc to the executable. + get_filename_component(SHADERC_LIBRARY_DIRECTORY ${SHADERC_LIBRARY} DIRECTORY) + list(APPEND CMAKE_BUILD_RPATH ${SHADERC_LIBRARY_DIRECTORY}) + endif() endif() if(APPLE) diff --git a/scripts/appimage/make-appimage.sh b/scripts/appimage/make-appimage.sh index 6a75b5e3b..309878529 100755 --- a/scripts/appimage/make-appimage.sh +++ b/scripts/appimage/make-appimage.sh @@ -55,6 +55,10 @@ BINARY=duckstation-qt APPDIRNAME=DuckStation.AppDir STRIP=strip +declare -a MANUAL_LIBS=( + "libshaderc_shared.so" +) + declare -a MANUAL_QT_LIBS=( "libQt6WaylandEglClientHwIntegration.so.6" ) @@ -90,6 +94,24 @@ fi OUTDIR=$(realpath "./$APPDIRNAME") rm -fr "$OUTDIR" +echo "Locating extra libraries..." +EXTRA_LIBS_ARGS="" +for lib in "${MANUAL_LIBS[@]}"; do + srcpath=$(find "$DEPSDIR" -name "$lib") + if [ ! -f "$srcpath" ]; then + echo "Missinge extra library $lib. Exiting." + exit 1 + fi + + echo "Found $lib at $srcpath." + + if [ "$EXTRA_LIBS_ARGS" == "" ]; then + EXTRA_LIBS_ARGS="--library=$srcpath" + else + EXTRA_LIBS_ARGS="$EXTRA_LIBS_ARGS,$srcpath" + fi +done + # Why the nastyness? linuxdeploy strips our main binary, and there's no option to turn it off. # It also doesn't strip the Qt libs. We can't strip them after running linuxdeploy, because # patchelf corrupts the libraries (but they still work), but patchelf+strip makes them crash @@ -112,9 +134,9 @@ EXTRA_PLATFORM_PLUGINS="libqwayland-egl.so;libqwayland-generic.so" \ DEPLOY_PLATFORM_THEMES="1" \ QMAKE="$DEPSDIR/bin/qmake" \ NO_STRIP="1" \ -$LINUXDEPLOY --plugin qt --appdir="$OUTDIR" --executable="$BUILDDIR/bin/duckstation-qt" \ +$LINUXDEPLOY --plugin qt --appdir="$OUTDIR" --executable="$BUILDDIR/bin/duckstation-qt" $EXTRA_LIBS_ARGS \ --desktop-file="$ROOTDIR/scripts/org.duckstation.DuckStation.desktop" \ ---icon-file="$ROOTDIR/scripts/org.duckstation.DuckStation.png" +--icon-file="$ROOTDIR/scripts/org.duckstation.DuckStation.png" \ echo "Copying resources into AppDir..." cp -a "$BUILDDIR/bin/resources" "$OUTDIR/usr/bin" diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index b9e84e8d6..275c36e62 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -169,7 +169,8 @@ if(ENABLE_VULKAN) endif() if(ENABLE_VULKAN OR APPLE) - target_link_libraries(util PUBLIC Shaderc::shaderc_shared) + # shaderc is loaded dynamically to reduce module loads on startup. + target_include_directories(util PUBLIC ${SHADERC_INCLUDE_DIR}) endif() if(NOT ANDROID) diff --git a/src/util/gpu_device.cpp b/src/util/gpu_device.cpp index d94762505..b68933216 100644 --- a/src/util/gpu_device.cpp +++ b/src/util/gpu_device.cpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2019-2023 Connor McLaughlin +// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin // SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0) #include "gpu_device.h" @@ -8,6 +8,7 @@ #include "shadergen.h" #include "common/assert.h" +#include "common/dynamic_library.h" #include "common/error.h" #include "common/file_system.h" #include "common/log.h" @@ -36,6 +37,10 @@ Log_SetChannel(GPUDevice); #include "vulkan_device.h" #endif +#if defined(ENABLE_VULKAN) || defined(__APPLE__) +#include "shaderc/shaderc.h" +#endif + std::unique_ptr g_gpu_device; static std::string s_pipeline_cache_path; @@ -1105,3 +1110,166 @@ std::unique_ptr GPUDevice::CreateDeviceForAPI(RenderAPI api) return {}; } } + +#if defined(ENABLE_VULKAN) || defined(__APPLE__) +#define SHADERC_INIT_FUNCTIONS(X) \ + X(shaderc_compiler_initialize) \ + X(shaderc_compiler_release) + +#define SHADERC_FUNCTIONS(X) \ + X(shaderc_compile_options_initialize) \ + X(shaderc_compile_options_release) \ + X(shaderc_compile_options_set_source_language) \ + X(shaderc_compile_options_set_generate_debug_info) \ + X(shaderc_compile_options_set_emit_non_semantic_debug_info) \ + X(shaderc_compile_options_set_optimization_level) \ + X(shaderc_compile_options_set_target_env) \ + X(shaderc_compile_into_spv) \ + X(shaderc_result_release) \ + X(shaderc_result_get_length) \ + X(shaderc_result_get_num_warnings) \ + X(shaderc_result_get_compilation_status) \ + X(shaderc_result_get_bytes) \ + X(shaderc_result_get_error_message) + +// TODO: NOT thread safe, yet. +namespace dyn_shaderc { +static bool Open(); + +static DynamicLibrary s_library; +static std::unique_ptr s_compiler(nullptr, nullptr); + +#define ADD_FUNC(F) static decltype(&::F) F; +SHADERC_FUNCTIONS(ADD_FUNC) +#undef ADD_FUNC + +} // namespace dyn_shaderc + +bool dyn_shaderc::Open() +{ + if (s_library.IsOpen()) + return true; + + Error error; + if (!s_library.Open(DynamicLibrary::GetVersionedFilename("shaderc_shared").c_str(), &error)) + { + Log_ErrorFmt("Failed to load shaderc: {}", error.GetDescription()); + return false; + } + +#define LOAD_FUNC(F) \ + if (!s_library.GetSymbol(#F, &F)) \ + { \ + Log_ErrorFmt("Failed to find function {}", #F); \ + s_library.Close(); \ + return false; \ + } +#define LOAD_INIT_FUNC(F) \ + decltype(&::F) p##F; \ + if (!s_library.GetSymbol(#F, &p##F)) \ + { \ + Log_ErrorFmt("Failed to find function {}", #F); \ + s_library.Close(); \ + return false; \ + } + + SHADERC_FUNCTIONS(LOAD_FUNC) + SHADERC_INIT_FUNCTIONS(LOAD_INIT_FUNC) +#undef LOAD_FUNC +#undef LOAD_INIT_FUNC + + s_compiler = decltype(s_compiler)(pshaderc_compiler_initialize(), pshaderc_compiler_release); + return true; +} + +#undef SHADERC_FUNCTIONS +#undef SHADERC_INIT_FUNCTIONS + +bool GPUDevice::CompileGLSLShaderToVulkanSpv(GPUShaderStage stage, std::string_view source, const char* entry_point, + bool nonsemantic_debug_info, DynamicHeapArray* out_binary) +{ + static constexpr const std::array(GPUShaderStage::MaxCount)> stage_kinds = {{ + shaderc_glsl_vertex_shader, + shaderc_glsl_fragment_shader, + shaderc_glsl_geometry_shader, + shaderc_glsl_compute_shader, + }}; + + // TODO: Move to library. + static constexpr const std::pair status_names[] = { + {shaderc_compilation_status_success, "shaderc_compilation_status_success"}, + {shaderc_compilation_status_invalid_stage, "shaderc_compilation_status_invalid_stage"}, + {shaderc_compilation_status_compilation_error, "shaderc_compilation_status_compilation_error"}, + {shaderc_compilation_status_internal_error, "shaderc_compilation_status_internal_error"}, + {shaderc_compilation_status_null_result_object, "shaderc_compilation_status_null_result_object"}, + {shaderc_compilation_status_invalid_assembly, "shaderc_compilation_status_invalid_assembly"}, + {shaderc_compilation_status_validation_error, "shaderc_compilation_status_validation_error"}, + {shaderc_compilation_status_transformation_error, "shaderc_compilation_status_transformation_error"}, + {shaderc_compilation_status_configuration_error, "shaderc_compilation_status_configuration_error"}, + }; + + if (!dyn_shaderc::Open()) + return false; + + const std::unique_ptr options( + dyn_shaderc::shaderc_compile_options_initialize(), dyn_shaderc::shaderc_compile_options_release); + if (!options) + { + Log_ErrorPrint("Failed to create shaderc options."); + return false; + } + + dyn_shaderc::shaderc_compile_options_set_source_language(options.get(), shaderc_source_language_glsl); + dyn_shaderc::shaderc_compile_options_set_target_env(options.get(), shaderc_target_env_vulkan, 0); + + if (m_debug_device) + { + dyn_shaderc::shaderc_compile_options_set_generate_debug_info(options.get()); + if (nonsemantic_debug_info) + dyn_shaderc::shaderc_compile_options_set_emit_non_semantic_debug_info(options.get()); + + dyn_shaderc::shaderc_compile_options_set_optimization_level(options.get(), shaderc_optimization_level_zero); + } + else + { + dyn_shaderc::shaderc_compile_options_set_optimization_level(options.get(), shaderc_optimization_level_performance); + } + + const std::unique_ptr result( + dyn_shaderc::shaderc_compile_into_spv(dyn_shaderc::s_compiler.get(), source.data(), source.length(), + stage_kinds[static_cast(stage)], "source", entry_point, + options.get()), + dyn_shaderc::shaderc_result_release); + const shaderc_compilation_status status = result ? dyn_shaderc::shaderc_result_get_compilation_status(result.get()) : + shaderc_compilation_status_null_result_object; + if (status != shaderc_compilation_status_success) + { + const char* status_name = "UNKNOWN"; + for (const auto& it : status_names) + { + if (status == it.first) + { + status_name = it.second; + break; + } + } + + const std::string_view errors(result ? dyn_shaderc::shaderc_result_get_error_message(result.get()) : + "null result object"); + Log_ErrorFmt("Failed to compile shader to SPIR-V: {}\n{}", status_name, errors); + DumpBadShader(source, errors); + return false; + } + + const size_t num_warnings = dyn_shaderc::shaderc_result_get_num_warnings(result.get()); + if (num_warnings > 0) + Log_WarningFmt("Shader compiled with warnings:\n{}", dyn_shaderc::shaderc_result_get_error_message(result.get())); + + const size_t spirv_size = dyn_shaderc::shaderc_result_get_length(result.get()); + DebugAssert(spirv_size > 0); + out_binary->resize(spirv_size); + std::memcpy(out_binary->data(), dyn_shaderc::shaderc_result_get_bytes(result.get()), spirv_size); + return true; +} + +#endif diff --git a/src/util/gpu_device.h b/src/util/gpu_device.h index 8790d3dd5..37331a209 100644 --- a/src/util/gpu_device.h +++ b/src/util/gpu_device.h @@ -719,6 +719,11 @@ protected: void TrimTexturePool(); +#if defined(ENABLE_VULKAN) || defined(__APPLE__) + bool CompileGLSLShaderToVulkanSpv(GPUShaderStage stage, std::string_view source, const char* entry_point, + bool nonsemantic_debug_info, DynamicHeapArray* out_binary); +#endif + Features m_features = {}; u32 m_max_texture_size = 0; u32 m_max_multisamples = 0; diff --git a/src/util/metal_device.mm b/src/util/metal_device.mm index 013511a59..2cebe65ee 100644 --- a/src/util/metal_device.mm +++ b/src/util/metal_device.mm @@ -16,7 +16,6 @@ #define FMT_EXCEPTIONS 0 #include "fmt/format.h" -#include "shaderc/shaderc.hpp" #include "spirv_cross_c.h" #include @@ -59,8 +58,6 @@ static constexpr std::array(GPUTexture::Format: MTLPixelFormatBGR10A2Unorm, // RGB10A2 }; -static std::unique_ptr s_shaderc_compiler; - static NSString* StringViewToNSString(std::string_view str) { if (str.empty()) @@ -657,44 +654,12 @@ std::unique_ptr MetalDevice::CreateShaderFromSource(GPUShaderStage st { static constexpr bool dump_shaders = false; - static constexpr const std::array(GPUShaderStage::MaxCount)> stage_kinds = {{ - shaderc_glsl_vertex_shader, - shaderc_glsl_fragment_shader, - shaderc_glsl_geometry_shader, - shaderc_glsl_compute_shader, - }}; - - // TODO: NOT thread safe, yet. - if (!s_shaderc_compiler) - s_shaderc_compiler = std::make_unique(); - - shaderc::CompileOptions spv_options; - spv_options.SetSourceLanguage(shaderc_source_language_glsl); - spv_options.SetTargetEnvironment(shaderc_target_env_vulkan, 0); - - if (m_debug_device) - { - spv_options.SetOptimizationLevel(shaderc_optimization_level_zero); - spv_options.SetGenerateDebugInfo(); - } - else - { - spv_options.SetOptimizationLevel(shaderc_optimization_level_performance); - } - - const shaderc::SpvCompilationResult result = s_shaderc_compiler->CompileGlslToSpv( - source.data(), source.length(), stage_kinds[static_cast(stage)], "source", entry_point, spv_options); - if (result.GetCompilationStatus() != shaderc_compilation_status_success) - { - const std::string errors = result.GetErrorMessage(); - DumpBadShader(source, errors); - Log_ErrorFmt("Failed to compile shader to SPIR-V:\n{}", errors); + DynamicHeapArray local_binary; + DynamicHeapArray* dest_binary = out_binary ? out_binary : &local_binary; + if (!CompileGLSLShaderToVulkanSpv(stage, source, entry_point, false, dest_binary)) return {}; - } - else if (result.GetNumWarnings() > 0) - { - Log_WarningFmt("Shader compiled with warnings:\n{}", result.GetErrorMessage()); - } + + AssertMsg((dest_binary->size() % 4) == 0, "Compile result should be 4 byte aligned."); spvc_context sctx; spvc_result sres; @@ -710,8 +675,7 @@ std::unique_ptr MetalDevice::CreateShaderFromSource(GPUShaderStage st sctx, [](void*, const char* error) { Log_ErrorFmt("SPIRV-Cross reported an error: {}", error); }, nullptr); spvc_parsed_ir sir; - if ((sres = spvc_context_parse_spirv(sctx, result.cbegin(), std::distance(result.cbegin(), result.cend()), &sir)) != - SPVC_SUCCESS) + if ((sres = spvc_context_parse_spirv(sctx, reinterpret_cast(dest_binary->data()), dest_binary->size() / 4, &sir)) != SPVC_SUCCESS) { Log_ErrorFmt("spvc_context_parse_spirv() failed: {}", static_cast(sres)); DumpBadShader(source, std::string_view()); diff --git a/src/util/util.props b/src/util/util.props index 7855c2811..c47fd6d2a 100644 --- a/src/util/util.props +++ b/src/util/util.props @@ -27,7 +27,7 @@ %(AdditionalIncludeDirectories);$(DepsIncludeDir)SDL2 - %(AdditionalDependencies);freetype.lib;libjpeg.lib;libpng16.lib;libwebp.lib;SDL2.lib;shaderc_shared.lib;zlib.lib;zstd.lib + %(AdditionalDependencies);freetype.lib;libjpeg.lib;libpng16.lib;libwebp.lib;SDL2.lib;zlib.lib;zstd.lib diff --git a/src/util/vulkan_pipeline.cpp b/src/util/vulkan_pipeline.cpp index 434764a5d..69756a2b5 100644 --- a/src/util/vulkan_pipeline.cpp +++ b/src/util/vulkan_pipeline.cpp @@ -8,12 +8,8 @@ #include "common/assert.h" #include "common/log.h" -#include "shaderc/shaderc.hpp" - Log_SetChannel(VulkanDevice); -static std::unique_ptr s_shaderc_compiler; - VulkanShader::VulkanShader(GPUShaderStage stage, VkShaderModule mod) : GPUShader(stage), m_module(mod) { } @@ -48,57 +44,17 @@ std::unique_ptr VulkanDevice::CreateShaderFromSource(GPUShaderStage s const char* entry_point, DynamicHeapArray* out_binary) { - static constexpr const std::array(GPUShaderStage::MaxCount)> stage_kinds = {{ - shaderc_glsl_vertex_shader, - shaderc_glsl_fragment_shader, - shaderc_glsl_geometry_shader, - shaderc_glsl_compute_shader, - }}; - - // TODO: NOT thread safe, yet. - if (!s_shaderc_compiler) - s_shaderc_compiler = std::make_unique(); - - shaderc::CompileOptions options; - options.SetSourceLanguage(shaderc_source_language_glsl); - options.SetTargetEnvironment(shaderc_target_env_vulkan, 0); - - if (m_debug_device) + DynamicHeapArray local_binary; + DynamicHeapArray* dest_binary = out_binary ? out_binary : &local_binary; + if (!CompileGLSLShaderToVulkanSpv(stage, source, entry_point, m_optional_extensions.vk_khr_shader_non_semantic_info, + dest_binary)) { - options.SetGenerateDebugInfo(); - if (m_optional_extensions.vk_khr_shader_non_semantic_info) - options.SetEmitNonSemanticDebugInfo(); - - options.SetOptimizationLevel(shaderc_optimization_level_zero); - } - else - { - options.SetOptimizationLevel(shaderc_optimization_level_performance); - } - - const shaderc::SpvCompilationResult result = s_shaderc_compiler->CompileGlslToSpv( - source.data(), source.length(), stage_kinds[static_cast(stage)], "source", entry_point, options); - if (result.GetCompilationStatus() != shaderc_compilation_status_success) - { - const std::string errors = result.GetErrorMessage(); - DumpBadShader(source, errors); - Log_ErrorFmt("Failed to compile shader to SPIR-V:\n{}", errors); return {}; } - else if (result.GetNumWarnings() > 0) - { - Log_WarningFmt("Shader compiled with warnings:\n{}", result.GetErrorMessage()); - } - const size_t spirv_size = std::distance(result.cbegin(), result.cend()) * sizeof(*result.cbegin()); - DebugAssert(spirv_size > 0); - if (out_binary) - { - out_binary->resize(spirv_size); - std::copy(result.cbegin(), result.cend(), reinterpret_cast(out_binary->data())); - } + AssertMsg((dest_binary->size() % 4) == 0, "Compile result should be 4 byte aligned."); - return CreateShaderFromBinary(stage, std::span(reinterpret_cast(result.cbegin()), spirv_size)); + return CreateShaderFromBinary(stage, dest_binary->cspan()); } //////////////////////////////////////////////////////////////////////////