GPUDevice: Move SPIR-V compilation to base class

CMakeModules/DuckStationDependencies.cmake

@@ -36,6 +36,12 @@ endif()
 
 if(ENABLE_VULKAN OR APPLE)
   find_package(Shaderc REQUIRED)
+
+  if(LINUX AND ENABLE_VULKAN)
+    # We need to add the rpath for shaderc to the executable.
+    get_filename_component(SHADERC_LIBRARY_DIRECTORY ${SHADERC_LIBRARY} DIRECTORY)
+    list(APPEND CMAKE_BUILD_RPATH ${SHADERC_LIBRARY_DIRECTORY})
+  endif()
 endif()
 
 if(APPLE)

scripts/appimage/make-appimage.sh

@@ -55,6 +55,10 @@ BINARY=duckstation-qt
 APPDIRNAME=DuckStation.AppDir
 STRIP=strip
 
+declare -a MANUAL_LIBS=(
+	"libshaderc_shared.so"
+)
+
 declare -a MANUAL_QT_LIBS=(
 	"libQt6WaylandEglClientHwIntegration.so.6"
 )
@@ -90,6 +94,24 @@ fi
 OUTDIR=$(realpath "./$APPDIRNAME")
 rm -fr "$OUTDIR"
 
+echo "Locating extra libraries..."
+EXTRA_LIBS_ARGS=""
+for lib in "${MANUAL_LIBS[@]}"; do
+	srcpath=$(find "$DEPSDIR" -name "$lib")
+	if [ ! -f "$srcpath" ]; then
+		echo "Missinge extra library $lib. Exiting."
+		exit 1
+	fi
+
+	echo "Found $lib at $srcpath."
+
+	if [ "$EXTRA_LIBS_ARGS" == "" ]; then
+		EXTRA_LIBS_ARGS="--library=$srcpath"
+	else
+		EXTRA_LIBS_ARGS="$EXTRA_LIBS_ARGS,$srcpath"
+	fi
+done
+
 # Why the nastyness? linuxdeploy strips our main binary, and there's no option to turn it off.
 # It also doesn't strip the Qt libs. We can't strip them after running linuxdeploy, because
 # patchelf corrupts the libraries (but they still work), but patchelf+strip makes them crash
@@ -112,9 +134,9 @@ EXTRA_PLATFORM_PLUGINS="libqwayland-egl.so;libqwayland-generic.so" \
 DEPLOY_PLATFORM_THEMES="1" \
 QMAKE="$DEPSDIR/bin/qmake" \
 NO_STRIP="1" \
-$LINUXDEPLOY --plugin qt --appdir="$OUTDIR" --executable="$BUILDDIR/bin/duckstation-qt" \
+$LINUXDEPLOY --plugin qt --appdir="$OUTDIR" --executable="$BUILDDIR/bin/duckstation-qt" $EXTRA_LIBS_ARGS \
 --desktop-file="$ROOTDIR/scripts/org.duckstation.DuckStation.desktop" \
---icon-file="$ROOTDIR/scripts/org.duckstation.DuckStation.png"
+--icon-file="$ROOTDIR/scripts/org.duckstation.DuckStation.png" \
 
 echo "Copying resources into AppDir..."
 cp -a "$BUILDDIR/bin/resources" "$OUTDIR/usr/bin"

src/util/CMakeLists.txt

@@ -169,7 +169,8 @@ if(ENABLE_VULKAN)
 endif()
 
 if(ENABLE_VULKAN OR APPLE)
-  target_link_libraries(util PUBLIC Shaderc::shaderc_shared)
+  # shaderc is loaded dynamically to reduce module loads on startup.
+  target_include_directories(util PUBLIC ${SHADERC_INCLUDE_DIR})
 endif()
 
 if(NOT ANDROID)

src/util/gpu_device.cpp

@@ -1,4 +1,4 @@
-// SPDX-FileCopyrightText: 2019-2023 Connor McLaughlin <stenzek@gmail.com>
+// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
 // SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
 
 #include "gpu_device.h"
@@ -8,6 +8,7 @@
 #include "shadergen.h"
 
 #include "common/assert.h"
+#include "common/dynamic_library.h"
 #include "common/error.h"
 #include "common/file_system.h"
 #include "common/log.h"
@@ -36,6 +37,10 @@ Log_SetChannel(GPUDevice);
 #include "vulkan_device.h"
 #endif
 
+#if defined(ENABLE_VULKAN) || defined(__APPLE__)
+#include "shaderc/shaderc.h"
+#endif
+
 std::unique_ptr<GPUDevice> g_gpu_device;
 
 static std::string s_pipeline_cache_path;
@@ -1105,3 +1110,166 @@ std::unique_ptr<GPUDevice> GPUDevice::CreateDeviceForAPI(RenderAPI api)
       return {};
   }
 }
+
+#if defined(ENABLE_VULKAN) || defined(__APPLE__)
+#define SHADERC_INIT_FUNCTIONS(X)                                                                                      \
+  X(shaderc_compiler_initialize)                                                                                       \
+  X(shaderc_compiler_release)
+
+#define SHADERC_FUNCTIONS(X)                                                                                           \
+  X(shaderc_compile_options_initialize)                                                                                \
+  X(shaderc_compile_options_release)                                                                                   \
+  X(shaderc_compile_options_set_source_language)                                                                       \
+  X(shaderc_compile_options_set_generate_debug_info)                                                                   \
+  X(shaderc_compile_options_set_emit_non_semantic_debug_info)                                                          \
+  X(shaderc_compile_options_set_optimization_level)                                                                    \
+  X(shaderc_compile_options_set_target_env)                                                                            \
+  X(shaderc_compile_into_spv)                                                                                          \
+  X(shaderc_result_release)                                                                                            \
+  X(shaderc_result_get_length)                                                                                         \
+  X(shaderc_result_get_num_warnings)                                                                                   \
+  X(shaderc_result_get_compilation_status)                                                                             \
+  X(shaderc_result_get_bytes)                                                                                          \
+  X(shaderc_result_get_error_message)
+
+// TODO: NOT thread safe, yet.
+namespace dyn_shaderc {
+static bool Open();
+
+static DynamicLibrary s_library;
+static std::unique_ptr<struct shaderc_compiler, void (*)(shaderc_compiler_t)> s_compiler(nullptr, nullptr);
+
+#define ADD_FUNC(F) static decltype(&::F) F;
+SHADERC_FUNCTIONS(ADD_FUNC)
+#undef ADD_FUNC
+
+} // namespace dyn_shaderc
+
+bool dyn_shaderc::Open()
+{
+  if (s_library.IsOpen())
+    return true;
+
+  Error error;
+  if (!s_library.Open(DynamicLibrary::GetVersionedFilename("shaderc_shared").c_str(), &error))
+  {
+    Log_ErrorFmt("Failed to load shaderc: {}", error.GetDescription());
+    return false;
+  }
+
+#define LOAD_FUNC(F)                                                                                                   \
+  if (!s_library.GetSymbol(#F, &F))                                                                                    \
+  {                                                                                                                    \
+    Log_ErrorFmt("Failed to find function {}", #F);                                                                    \
+    s_library.Close();                                                                                                 \
+    return false;                                                                                                      \
+  }
+#define LOAD_INIT_FUNC(F)                                                                                              \
+  decltype(&::F) p##F;                                                                                                 \
+  if (!s_library.GetSymbol(#F, &p##F))                                                                                 \
+  {                                                                                                                    \
+    Log_ErrorFmt("Failed to find function {}", #F);                                                                    \
+    s_library.Close();                                                                                                 \
+    return false;                                                                                                      \
+  }
+
+  SHADERC_FUNCTIONS(LOAD_FUNC)
+  SHADERC_INIT_FUNCTIONS(LOAD_INIT_FUNC)
+#undef LOAD_FUNC
+#undef LOAD_INIT_FUNC
+
+  s_compiler = decltype(s_compiler)(pshaderc_compiler_initialize(), pshaderc_compiler_release);
+  return true;
+}
+
+#undef SHADERC_FUNCTIONS
+#undef SHADERC_INIT_FUNCTIONS
+
+bool GPUDevice::CompileGLSLShaderToVulkanSpv(GPUShaderStage stage, std::string_view source, const char* entry_point,
+                                             bool nonsemantic_debug_info, DynamicHeapArray<u8>* out_binary)
+{
+  static constexpr const std::array<shaderc_shader_kind, static_cast<size_t>(GPUShaderStage::MaxCount)> stage_kinds = {{
+    shaderc_glsl_vertex_shader,
+    shaderc_glsl_fragment_shader,
+    shaderc_glsl_geometry_shader,
+    shaderc_glsl_compute_shader,
+  }};
+
+  // TODO: Move to library.
+  static constexpr const std::pair<shaderc_compilation_status, const char*> status_names[] = {
+    {shaderc_compilation_status_success, "shaderc_compilation_status_success"},
+    {shaderc_compilation_status_invalid_stage, "shaderc_compilation_status_invalid_stage"},
+    {shaderc_compilation_status_compilation_error, "shaderc_compilation_status_compilation_error"},
+    {shaderc_compilation_status_internal_error, "shaderc_compilation_status_internal_error"},
+    {shaderc_compilation_status_null_result_object, "shaderc_compilation_status_null_result_object"},
+    {shaderc_compilation_status_invalid_assembly, "shaderc_compilation_status_invalid_assembly"},
+    {shaderc_compilation_status_validation_error, "shaderc_compilation_status_validation_error"},
+    {shaderc_compilation_status_transformation_error, "shaderc_compilation_status_transformation_error"},
+    {shaderc_compilation_status_configuration_error, "shaderc_compilation_status_configuration_error"},
+  };
+
+  if (!dyn_shaderc::Open())
+    return false;
+
+  const std::unique_ptr<struct shaderc_compile_options, void (*)(shaderc_compile_options_t)> options(
+    dyn_shaderc::shaderc_compile_options_initialize(), dyn_shaderc::shaderc_compile_options_release);
+  if (!options)
+  {
+    Log_ErrorPrint("Failed to create shaderc options.");
+    return false;
+  }
+
+  dyn_shaderc::shaderc_compile_options_set_source_language(options.get(), shaderc_source_language_glsl);
+  dyn_shaderc::shaderc_compile_options_set_target_env(options.get(), shaderc_target_env_vulkan, 0);
+
+  if (m_debug_device)
+  {
+    dyn_shaderc::shaderc_compile_options_set_generate_debug_info(options.get());
+    if (nonsemantic_debug_info)
+      dyn_shaderc::shaderc_compile_options_set_emit_non_semantic_debug_info(options.get());
+
+    dyn_shaderc::shaderc_compile_options_set_optimization_level(options.get(), shaderc_optimization_level_zero);
+  }
+  else
+  {
+    dyn_shaderc::shaderc_compile_options_set_optimization_level(options.get(), shaderc_optimization_level_performance);
+  }
+
+  const std::unique_ptr<struct shaderc_compilation_result, void (*)(shaderc_compilation_result_t)> result(
+    dyn_shaderc::shaderc_compile_into_spv(dyn_shaderc::s_compiler.get(), source.data(), source.length(),
+                                          stage_kinds[static_cast<size_t>(stage)], "source", entry_point,
+                                          options.get()),
+    dyn_shaderc::shaderc_result_release);
+  const shaderc_compilation_status status = result ? dyn_shaderc::shaderc_result_get_compilation_status(result.get()) :
+                                                     shaderc_compilation_status_null_result_object;
+  if (status != shaderc_compilation_status_success)
+  {
+    const char* status_name = "UNKNOWN";
+    for (const auto& it : status_names)
+    {
+      if (status == it.first)
+      {
+        status_name = it.second;
+        break;
+      }
+    }
+
+    const std::string_view errors(result ? dyn_shaderc::shaderc_result_get_error_message(result.get()) :
+                                           "null result object");
+    Log_ErrorFmt("Failed to compile shader to SPIR-V: {}\n{}", status_name, errors);
+    DumpBadShader(source, errors);
+    return false;
+  }
+
+  const size_t num_warnings = dyn_shaderc::shaderc_result_get_num_warnings(result.get());
+  if (num_warnings > 0)
+    Log_WarningFmt("Shader compiled with warnings:\n{}", dyn_shaderc::shaderc_result_get_error_message(result.get()));
+
+  const size_t spirv_size = dyn_shaderc::shaderc_result_get_length(result.get());
+  DebugAssert(spirv_size > 0);
+  out_binary->resize(spirv_size);
+  std::memcpy(out_binary->data(), dyn_shaderc::shaderc_result_get_bytes(result.get()), spirv_size);
+  return true;
+}
+
+#endif

src/util/gpu_device.h

@@ -719,6 +719,11 @@ protected:
 
   void TrimTexturePool();
 
+#if defined(ENABLE_VULKAN) || defined(__APPLE__)
+  bool CompileGLSLShaderToVulkanSpv(GPUShaderStage stage, std::string_view source, const char* entry_point,
+                                    bool nonsemantic_debug_info, DynamicHeapArray<u8>* out_binary);
+#endif
+
   Features m_features = {};
   u32 m_max_texture_size = 0;
   u32 m_max_multisamples = 0;

src/util/metal_device.mm

@@ -16,7 +16,6 @@
 #define FMT_EXCEPTIONS 0
 #include "fmt/format.h"
 
-#include "shaderc/shaderc.hpp"
 #include "spirv_cross_c.h"
 
 #include <array>
@@ -59,8 +58,6 @@ static constexpr std::array<MTLPixelFormat, static_cast<u32>(GPUTexture::Format:
   MTLPixelFormatBGR10A2Unorm, // RGB10A2
 };
 
-static std::unique_ptr<shaderc::Compiler> s_shaderc_compiler;
-
 static NSString* StringViewToNSString(std::string_view str)
 {
   if (str.empty())
@@ -657,44 +654,12 @@ std::unique_ptr<GPUShader> MetalDevice::CreateShaderFromSource(GPUShaderStage st
 {
   static constexpr bool dump_shaders = false;
 
-  static constexpr const std::array<shaderc_shader_kind, static_cast<size_t>(GPUShaderStage::MaxCount)> stage_kinds = {{
+  DynamicHeapArray<u8> local_binary;
-    shaderc_glsl_vertex_shader,
+  DynamicHeapArray<u8>* dest_binary = out_binary ? out_binary : &local_binary;
-    shaderc_glsl_fragment_shader,
+  if (!CompileGLSLShaderToVulkanSpv(stage, source, entry_point, false, dest_binary))
-    shaderc_glsl_geometry_shader,
-    shaderc_glsl_compute_shader,
-  }};
-
-  // TODO: NOT thread safe, yet.
-  if (!s_shaderc_compiler)
-    s_shaderc_compiler = std::make_unique<shaderc::Compiler>();
-
-  shaderc::CompileOptions spv_options;
-  spv_options.SetSourceLanguage(shaderc_source_language_glsl);
-  spv_options.SetTargetEnvironment(shaderc_target_env_vulkan, 0);
-
-  if (m_debug_device)
-  {
-    spv_options.SetOptimizationLevel(shaderc_optimization_level_zero);
-    spv_options.SetGenerateDebugInfo();
-  }
-  else
-  {
-    spv_options.SetOptimizationLevel(shaderc_optimization_level_performance);
-  }
-
-  const shaderc::SpvCompilationResult result = s_shaderc_compiler->CompileGlslToSpv(
-    source.data(), source.length(), stage_kinds[static_cast<size_t>(stage)], "source", entry_point, spv_options);
-  if (result.GetCompilationStatus() != shaderc_compilation_status_success)
-  {
-    const std::string errors = result.GetErrorMessage();
-    DumpBadShader(source, errors);
-    Log_ErrorFmt("Failed to compile shader to SPIR-V:\n{}", errors);
     return {};
-  }
+
-  else if (result.GetNumWarnings() > 0)
+  AssertMsg((dest_binary->size() % 4) == 0, "Compile result should be 4 byte aligned.");
-  {
-    Log_WarningFmt("Shader compiled with warnings:\n{}", result.GetErrorMessage());
-  }
 
   spvc_context sctx;
   spvc_result sres;
@@ -710,8 +675,7 @@ std::unique_ptr<GPUShader> MetalDevice::CreateShaderFromSource(GPUShaderStage st
     sctx, [](void*, const char* error) { Log_ErrorFmt("SPIRV-Cross reported an error: {}", error); }, nullptr);
 
   spvc_parsed_ir sir;
-  if ((sres = spvc_context_parse_spirv(sctx, result.cbegin(), std::distance(result.cbegin(), result.cend()), &sir)) !=
+  if ((sres = spvc_context_parse_spirv(sctx, reinterpret_cast<const u32*>(dest_binary->data()), dest_binary->size() / 4, &sir)) != SPVC_SUCCESS)
-      SPVC_SUCCESS)
   {
     Log_ErrorFmt("spvc_context_parse_spirv() failed: {}", static_cast<int>(sres));
     DumpBadShader(source, std::string_view());

src/util/util.props

@@ -27,7 +27,7 @@
       <AdditionalIncludeDirectories>%(AdditionalIncludeDirectories);$(DepsIncludeDir)SDL2</AdditionalIncludeDirectories>
     </ClCompile>
     <Link>
-      <AdditionalDependencies>%(AdditionalDependencies);freetype.lib;libjpeg.lib;libpng16.lib;libwebp.lib;SDL2.lib;shaderc_shared.lib;zlib.lib;zstd.lib</AdditionalDependencies>
+      <AdditionalDependencies>%(AdditionalDependencies);freetype.lib;libjpeg.lib;libpng16.lib;libwebp.lib;SDL2.lib;zlib.lib;zstd.lib</AdditionalDependencies>
     </Link>
   </ItemDefinitionGroup>
   <ItemGroup>

src/util/vulkan_pipeline.cpp

@@ -8,12 +8,8 @@
 #include "common/assert.h"
 #include "common/log.h"
 
-#include "shaderc/shaderc.hpp"
-
 Log_SetChannel(VulkanDevice);
 
-static std::unique_ptr<shaderc::Compiler> s_shaderc_compiler;
-
 VulkanShader::VulkanShader(GPUShaderStage stage, VkShaderModule mod) : GPUShader(stage), m_module(mod)
 {
 }
@@ -48,57 +44,17 @@ std::unique_ptr<GPUShader> VulkanDevice::CreateShaderFromSource(GPUShaderStage s
                                                                 const char* entry_point,
                                                                 DynamicHeapArray<u8>* out_binary)
 {
-  static constexpr const std::array<shaderc_shader_kind, static_cast<size_t>(GPUShaderStage::MaxCount)> stage_kinds = {{
+  DynamicHeapArray<u8> local_binary;
-    shaderc_glsl_vertex_shader,
+  DynamicHeapArray<u8>* dest_binary = out_binary ? out_binary : &local_binary;
-    shaderc_glsl_fragment_shader,
+  if (!CompileGLSLShaderToVulkanSpv(stage, source, entry_point, m_optional_extensions.vk_khr_shader_non_semantic_info,
-    shaderc_glsl_geometry_shader,
+                                    dest_binary))
-    shaderc_glsl_compute_shader,
-  }};
-
-  // TODO: NOT thread safe, yet.
-  if (!s_shaderc_compiler)
-    s_shaderc_compiler = std::make_unique<shaderc::Compiler>();
-
-  shaderc::CompileOptions options;
-  options.SetSourceLanguage(shaderc_source_language_glsl);
-  options.SetTargetEnvironment(shaderc_target_env_vulkan, 0);
-
-  if (m_debug_device)
   {
-    options.SetGenerateDebugInfo();
-    if (m_optional_extensions.vk_khr_shader_non_semantic_info)
-      options.SetEmitNonSemanticDebugInfo();
-
-    options.SetOptimizationLevel(shaderc_optimization_level_zero);
-  }
-  else
-  {
-    options.SetOptimizationLevel(shaderc_optimization_level_performance);
-  }
-
-  const shaderc::SpvCompilationResult result = s_shaderc_compiler->CompileGlslToSpv(
-    source.data(), source.length(), stage_kinds[static_cast<size_t>(stage)], "source", entry_point, options);
-  if (result.GetCompilationStatus() != shaderc_compilation_status_success)
-  {
-    const std::string errors = result.GetErrorMessage();
-    DumpBadShader(source, errors);
-    Log_ErrorFmt("Failed to compile shader to SPIR-V:\n{}", errors);
     return {};
   }
-  else if (result.GetNumWarnings() > 0)
-  {
-    Log_WarningFmt("Shader compiled with warnings:\n{}", result.GetErrorMessage());
-  }
 
-  const size_t spirv_size = std::distance(result.cbegin(), result.cend()) * sizeof(*result.cbegin());
+  AssertMsg((dest_binary->size() % 4) == 0, "Compile result should be 4 byte aligned.");
-  DebugAssert(spirv_size > 0);
-  if (out_binary)
-  {
-    out_binary->resize(spirv_size);
-    std::copy(result.cbegin(), result.cend(), reinterpret_cast<uint32_t*>(out_binary->data()));
-  }
 
-  return CreateShaderFromBinary(stage, std::span<const u8>(reinterpret_cast<const u8*>(result.cbegin()), spirv_size));
+  return CreateShaderFromBinary(stage, dest_binary->cspan());
 }
 
 //////////////////////////////////////////////////////////////////////////