From fd166a4485bea981e1e9f596261e818a190f6801 Mon Sep 17 00:00:00 2001 From: Connor McLaughlin <stenzek@gmail.com> Date: Mon, 11 Jan 2021 14:50:25 +1000 Subject: [PATCH] ShaderCache: Add a data version field We can increment this to prevent people's shader caches from growing too large with shader changes. --- src/common/d3d11/shader_cache.cpp | 14 +++++++++----- src/common/d3d11/shader_cache.h | 8 +++++--- src/common/gl/shader_cache.cpp | 14 +++++++++----- src/common/gl/shader_cache.h | 5 +++-- src/common/vulkan/shader_cache.cpp | 16 ++++++++++------ src/common/vulkan/shader_cache.h | 7 ++++--- src/core/CMakeLists.txt | 1 + src/core/core.vcxproj | 1 + src/core/core.vcxproj.filters | 3 ++- src/core/gpu_hw_d3d11.cpp | 3 ++- src/core/gpu_hw_opengl.cpp | 3 ++- src/core/shader_cache_version.h | 4 ++++ src/frontend-common/d3d11_host_display.cpp | 3 ++- src/frontend-common/vulkan_host_display.cpp | 3 ++- 14 files changed, 56 insertions(+), 29 deletions(-) create mode 100644 src/core/shader_cache_version.h diff --git a/src/common/d3d11/shader_cache.cpp b/src/common/d3d11/shader_cache.cpp index 1c293a064..7ed06c32c 100644 --- a/src/common/d3d11/shader_cache.cpp +++ b/src/common/d3d11/shader_cache.cpp @@ -42,9 +42,10 @@ bool ShaderCache::CacheIndexKey::operator!=(const CacheIndexKey& key) const source_length != key.source_length || shader_type != key.shader_type); } -void ShaderCache::Open(std::string_view base_path, D3D_FEATURE_LEVEL feature_level, bool debug) +void ShaderCache::Open(std::string_view base_path, D3D_FEATURE_LEVEL feature_level, u32 version, bool debug) { m_feature_level = feature_level; + m_version = version; m_debug = debug; if (!base_path.empty()) @@ -79,7 +80,8 @@ bool ShaderCache::CreateNew(const std::string& index_filename, const std::string } const u32 index_version = FILE_VERSION; - if (std::fwrite(&index_version, sizeof(index_version), 1, m_index_file) != 1) + if (std::fwrite(&index_version, sizeof(index_version), 1, m_index_file) != 1 || + std::fwrite(&m_version, sizeof(m_version), 1, m_index_file) != 1) { Log_ErrorPrintf("Failed to write version to index file '%s'", index_filename.c_str()); std::fclose(m_index_file); @@ -107,10 +109,12 @@ bool ShaderCache::ReadExisting(const std::string& index_filename, const std::str if (!m_index_file) return false; - u32 file_version; - if (std::fread(&file_version, sizeof(file_version), 1, m_index_file) != 1 || file_version != FILE_VERSION) + u32 file_version = 0; + u32 data_version = 0; + if (std::fread(&file_version, sizeof(file_version), 1, m_index_file) != 1 || file_version != FILE_VERSION || + std::fread(&data_version, sizeof(data_version), 1, m_index_file) != 1 || data_version != m_version) { - Log_ErrorPrintf("Bad file version in '%s'", index_filename.c_str()); + Log_ErrorPrintf("Bad file/data version in '%s'", index_filename.c_str()); std::fclose(m_index_file); m_index_file = nullptr; return false; diff --git a/src/common/d3d11/shader_cache.h b/src/common/d3d11/shader_cache.h index d658fc91b..3b24ffe0c 100644 --- a/src/common/d3d11/shader_cache.h +++ b/src/common/d3d11/shader_cache.h @@ -21,7 +21,7 @@ public: ShaderCache(); ~ShaderCache(); - void Open(std::string_view base_path, D3D_FEATURE_LEVEL feature_level, bool debug); + void Open(std::string_view base_path, D3D_FEATURE_LEVEL feature_level, u32 version, bool debug); ComPtr<ID3DBlob> GetShaderBlob(ShaderCompiler::Type type, std::string_view shader_code); @@ -31,7 +31,7 @@ public: ComPtr<ID3D11ComputeShader> GetComputeShader(ID3D11Device* device, std::string_view shader_code); private: - static constexpr u32 FILE_VERSION = 1; + static constexpr u32 FILE_VERSION = 2; struct CacheIndexKey { @@ -62,7 +62,8 @@ private: using CacheIndex = std::unordered_map<CacheIndexKey, CacheIndexData, CacheIndexEntryHasher>; - static std::string GetCacheBaseFileName(const std::string_view& base_path, D3D_FEATURE_LEVEL feature_level, bool debug); + static std::string GetCacheBaseFileName(const std::string_view& base_path, D3D_FEATURE_LEVEL feature_level, + bool debug); static CacheIndexKey GetCacheKey(ShaderCompiler::Type type, const std::string_view& shader_code); bool CreateNew(const std::string& index_filename, const std::string& blob_filename); @@ -77,6 +78,7 @@ private: CacheIndex m_index; D3D_FEATURE_LEVEL m_feature_level = D3D_FEATURE_LEVEL_11_0; + u32 m_version = 0; bool m_debug = false; }; diff --git a/src/common/gl/shader_cache.cpp b/src/common/gl/shader_cache.cpp index 90166bd78..3306c4b79 100644 --- a/src/common/gl/shader_cache.cpp +++ b/src/common/gl/shader_cache.cpp @@ -52,9 +52,10 @@ bool ShaderCache::CacheIndexKey::operator!=(const CacheIndexKey& key) const fragment_source_hash_high != key.fragment_source_hash_high || fragment_source_length != key.fragment_source_length); } -void ShaderCache::Open(bool is_gles, std::string_view base_path) +void ShaderCache::Open(bool is_gles, std::string_view base_path, u32 version) { m_base_path = base_path; + m_version = version; m_program_binary_supported = is_gles || GLAD_GL_ARB_get_program_binary; if (m_program_binary_supported) { @@ -103,7 +104,8 @@ bool ShaderCache::CreateNew(const std::string& index_filename, const std::string } const u32 index_version = FILE_VERSION; - if (std::fwrite(&index_version, sizeof(index_version), 1, m_index_file) != 1) + if (std::fwrite(&index_version, sizeof(index_version), 1, m_index_file) != 1 || + std::fwrite(&m_version, sizeof(m_version), 1, m_index_file) != 1) { Log_ErrorPrintf("Failed to write version to index file '%s'", index_filename.c_str()); std::fclose(m_index_file); @@ -131,10 +133,12 @@ bool ShaderCache::ReadExisting(const std::string& index_filename, const std::str if (!m_index_file) return false; - u32 file_version; - if (std::fread(&file_version, sizeof(file_version), 1, m_index_file) != 1 || file_version != FILE_VERSION) + u32 file_version = 0; + u32 data_version = 0; + if (std::fread(&file_version, sizeof(file_version), 1, m_index_file) != 1 || file_version != FILE_VERSION || + std::fread(&data_version, sizeof(data_version), 1, m_index_file) != 1 || data_version != m_version) { - Log_ErrorPrintf("Bad file version in '%s'", index_filename.c_str()); + Log_ErrorPrintf("Bad file/data version in '%s'", index_filename.c_str()); std::fclose(m_index_file); m_index_file = nullptr; return false; diff --git a/src/common/gl/shader_cache.h b/src/common/gl/shader_cache.h index 304c6a5f0..28fc8f367 100644 --- a/src/common/gl/shader_cache.h +++ b/src/common/gl/shader_cache.h @@ -20,13 +20,13 @@ public: ShaderCache(); ~ShaderCache(); - void Open(bool is_gles, std::string_view base_path); + void Open(bool is_gles, std::string_view base_path, u32 version); std::optional<Program> GetProgram(const std::string_view vertex_shader, const std::string_view geometry_shader, const std::string_view fragment_shader, const PreLinkCallback& callback = {}); private: - static constexpr u32 FILE_VERSION = 2; + static constexpr u32 FILE_VERSION = 3; struct CacheIndexKey { @@ -88,6 +88,7 @@ private: std::FILE* m_blob_file = nullptr; CacheIndex m_index; + u32 m_version = 0; bool m_program_binary_supported = false; }; diff --git a/src/common/vulkan/shader_cache.cpp b/src/common/vulkan/shader_cache.cpp index 80951cdab..16e1f57ae 100644 --- a/src/common/vulkan/shader_cache.cpp +++ b/src/common/vulkan/shader_cache.cpp @@ -105,11 +105,11 @@ bool ShaderCache::CacheIndexKey::operator!=(const CacheIndexKey& key) const source_length != key.source_length || shader_type != key.shader_type); } -void ShaderCache::Create(std::string_view base_path, bool debug) +void ShaderCache::Create(std::string_view base_path, u32 version, bool debug) { Assert(!g_vulkan_shader_cache); g_vulkan_shader_cache.reset(new ShaderCache()); - g_vulkan_shader_cache->Open(base_path, debug); + g_vulkan_shader_cache->Open(base_path, debug, version); } void ShaderCache::Destroy() @@ -117,8 +117,9 @@ void ShaderCache::Destroy() g_vulkan_shader_cache.reset(); } -void ShaderCache::Open(std::string_view base_path, bool debug) +void ShaderCache::Open(std::string_view base_path, u32 version, bool debug) { + m_version = version; m_debug = debug; if (!base_path.empty()) @@ -175,6 +176,7 @@ bool ShaderCache::CreateNewShaderCache(const std::string& index_filename, const FillPipelineCacheHeader(&header); if (std::fwrite(&index_version, sizeof(index_version), 1, m_index_file) != 1 || + std::fwrite(&m_version, sizeof(m_version), 1, m_index_file) != 1 || std::fwrite(&header, sizeof(header), 1, m_index_file) != 1) { Log_ErrorPrintf("Failed to write header to index file '%s'", index_filename.c_str()); @@ -203,10 +205,12 @@ bool ShaderCache::ReadExistingShaderCache(const std::string& index_filename, con if (!m_index_file) return false; - u32 file_version; - if (std::fread(&file_version, sizeof(file_version), 1, m_index_file) != 1 || file_version != FILE_VERSION) + u32 file_version = 0; + u32 data_version = 0; + if (std::fread(&file_version, sizeof(file_version), 1, m_index_file) != 1 || file_version != FILE_VERSION || + std::fread(&data_version, sizeof(data_version), 1, m_index_file) != 1 || data_version != m_version) { - Log_ErrorPrintf("Bad file version in '%s'", index_filename.c_str()); + Log_ErrorPrintf("Bad file/data version in '%s'", index_filename.c_str()); std::fclose(m_index_file); m_index_file = nullptr; return false; diff --git a/src/common/vulkan/shader_cache.h b/src/common/vulkan/shader_cache.h index 7fdefc002..961272d4c 100644 --- a/src/common/vulkan/shader_cache.h +++ b/src/common/vulkan/shader_cache.h @@ -18,7 +18,7 @@ class ShaderCache public: ~ShaderCache(); - static void Create(std::string_view base_path, bool debug); + static void Create(std::string_view base_path, u32 version, bool debug); static void Destroy(); /// Returns a handle to the pipeline cache. Set set_dirty to true if you are planning on writing to it externally. @@ -36,7 +36,7 @@ public: VkShaderModule GetComputeShader(std::string_view shader_code); private: - static constexpr u32 FILE_VERSION = 1; + static constexpr u32 FILE_VERSION = 2; struct CacheIndexKey { @@ -73,7 +73,7 @@ private: static std::string GetPipelineCacheBaseFileName(const std::string_view& base_path, bool debug); static CacheIndexKey GetCacheKey(ShaderCompiler::Type type, const std::string_view& shader_code); - void Open(std::string_view base_path, bool debug); + void Open(std::string_view base_path, u32 version, bool debug); bool CreateNewShaderCache(const std::string& index_filename, const std::string& blob_filename); bool ReadExistingShaderCache(const std::string& index_filename, const std::string& blob_filename); @@ -93,6 +93,7 @@ private: CacheIndex m_index; VkPipelineCache m_pipeline_cache = VK_NULL_HANDLE; + u32 m_version = 0; bool m_debug = false; bool m_pipeline_cache_dirty = false; }; diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 059ef1838..4ece26be9 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -84,6 +84,7 @@ add_library(core save_state_version.h settings.cpp settings.h + shader_cache_version.h shadergen.cpp shadergen.h sio.cpp diff --git a/src/core/core.vcxproj b/src/core/core.vcxproj index 374f81ffb..6142f7c8c 100644 --- a/src/core/core.vcxproj +++ b/src/core/core.vcxproj @@ -224,6 +224,7 @@ <ClInclude Include="save_state_version.h" /> <ClInclude Include="settings.h" /> <ClInclude Include="shadergen.h" /> + <ClInclude Include="shader_cache_version.h" /> <ClInclude Include="sio.h" /> <ClInclude Include="spu.h" /> <ClInclude Include="system.h" /> diff --git a/src/core/core.vcxproj.filters b/src/core/core.vcxproj.filters index b50bf245a..91d6d1d25 100644 --- a/src/core/core.vcxproj.filters +++ b/src/core/core.vcxproj.filters @@ -56,6 +56,7 @@ <ClCompile Include="gpu_sw_backend.cpp" /> <ClCompile Include="libcrypt_game_codes.cpp" /> <ClCompile Include="texture_replacements.cpp" /> + <ClCompile Include="gdb_protocol.h" /> </ItemGroup> <ItemGroup> <ClInclude Include="types.h" /> @@ -66,7 +67,6 @@ <ClInclude Include="cpu_disasm.h" /> <ClInclude Include="bus.h" /> <ClInclude Include="dma.h" /> - <ClInclude Include="gdb_protocol.h" /> <ClInclude Include="gpu.h" /> <ClInclude Include="gpu_hw_opengl.h" /> <ClInclude Include="gpu_hw.h" /> @@ -115,5 +115,6 @@ <ClInclude Include="gpu_sw_backend.h" /> <ClInclude Include="libcrypt_game_codes.h" /> <ClInclude Include="texture_replacements.h" /> + <ClInclude Include="shader_cache_version.h" /> </ItemGroup> </Project> \ No newline at end of file diff --git a/src/core/gpu_hw_d3d11.cpp b/src/core/gpu_hw_d3d11.cpp index c7204b628..9cd98ef70 100644 --- a/src/core/gpu_hw_d3d11.cpp +++ b/src/core/gpu_hw_d3d11.cpp @@ -6,6 +6,7 @@ #include "gpu_hw_shadergen.h" #include "host_display.h" #include "host_interface.h" +#include "shader_cache_version.h" #include "system.h" Log_SetChannel(GPU_HW_D3D11); @@ -447,7 +448,7 @@ void GPU_HW_D3D11::DestroyStateObjects() bool GPU_HW_D3D11::CompileShaders() { D3D11::ShaderCache shader_cache; - shader_cache.Open(g_host_interface->GetShaderCacheBasePath(), m_device->GetFeatureLevel(), + shader_cache.Open(g_host_interface->GetShaderCacheBasePath(), m_device->GetFeatureLevel(), SHADER_CACHE_VERSION, g_settings.gpu_use_debug_device); GPU_HW_ShaderGen shadergen(m_host_display->GetRenderAPI(), m_resolution_scale, m_multisamples, m_per_sample_shading, diff --git a/src/core/gpu_hw_opengl.cpp b/src/core/gpu_hw_opengl.cpp index 5fa00ccc1..1e0980f1d 100644 --- a/src/core/gpu_hw_opengl.cpp +++ b/src/core/gpu_hw_opengl.cpp @@ -4,6 +4,7 @@ #include "common/timer.h" #include "gpu_hw_shadergen.h" #include "host_display.h" +#include "shader_cache_version.h" #include "system.h" #include "texture_replacements.h" Log_SetChannel(GPU_HW_OpenGL); @@ -398,7 +399,7 @@ bool GPU_HW_OpenGL::CreateTextureBuffer() bool GPU_HW_OpenGL::CompilePrograms() { GL::ShaderCache shader_cache; - shader_cache.Open(IsGLES(), g_host_interface->GetShaderCacheBasePath()); + shader_cache.Open(IsGLES(), g_host_interface->GetShaderCacheBasePath(), SHADER_CACHE_VERSION); const bool use_binding_layout = GPU_HW_ShaderGen::UseGLSLBindingLayout(); GPU_HW_ShaderGen shadergen(m_host_display->GetRenderAPI(), m_resolution_scale, m_multisamples, m_per_sample_shading, diff --git a/src/core/shader_cache_version.h b/src/core/shader_cache_version.h new file mode 100644 index 000000000..d643c1d92 --- /dev/null +++ b/src/core/shader_cache_version.h @@ -0,0 +1,4 @@ +#pragma once +#include "types.h" + +static constexpr u32 SHADER_CACHE_VERSION = 1; \ No newline at end of file diff --git a/src/frontend-common/d3d11_host_display.cpp b/src/frontend-common/d3d11_host_display.cpp index e635d3b95..4b9831724 100644 --- a/src/frontend-common/d3d11_host_display.cpp +++ b/src/frontend-common/d3d11_host_display.cpp @@ -6,6 +6,7 @@ #include "common/string_util.h" #include "core/host_interface.h" #include "core/settings.h" +#include "core/shader_cache_version.h" #include "display_ps.hlsl.h" #include "display_vs.hlsl.h" #include "frontend-common/postprocessing_shadergen.h" @@ -930,7 +931,7 @@ bool D3D11HostDisplay::SetPostProcessingChain(const std::string_view& config) m_post_processing_stages.clear(); D3D11::ShaderCache shader_cache; - shader_cache.Open(g_host_interface->GetShaderCacheBasePath(), m_device->GetFeatureLevel(), + shader_cache.Open(g_host_interface->GetShaderCacheBasePath(), m_device->GetFeatureLevel(), SHADER_CACHE_VERSION, g_settings.gpu_use_debug_device); FrontendCommon::PostProcessingShaderGen shadergen(HostDisplay::RenderAPI::D3D11, true); diff --git a/src/frontend-common/vulkan_host_display.cpp b/src/frontend-common/vulkan_host_display.cpp index 3fb7dba92..1cafab374 100644 --- a/src/frontend-common/vulkan_host_display.cpp +++ b/src/frontend-common/vulkan_host_display.cpp @@ -9,6 +9,7 @@ #include "common/vulkan/stream_buffer.h" #include "common/vulkan/swap_chain.h" #include "common/vulkan/util.h" +#include "core/shader_cache_version.h" #include "postprocessing_shadergen.h" #include <array> #ifdef WITH_IMGUI @@ -331,7 +332,7 @@ bool VulkanHostDisplay::CreateRenderDevice(const WindowInfo& wi, std::string_vie bool VulkanHostDisplay::InitializeRenderDevice(std::string_view shader_cache_directory, bool debug_device, bool threaded_presentation) { - Vulkan::ShaderCache::Create(shader_cache_directory, debug_device); + Vulkan::ShaderCache::Create(shader_cache_directory, SHADER_CACHE_VERSION, debug_device); if (!CreateResources()) return false;