From 2275aa85d9c4dd72590887544c9fcc168e01da7e Mon Sep 17 00:00:00 2001 From: Stenzek Date: Mon, 26 Aug 2024 21:33:28 +1000 Subject: [PATCH] GPUDevice: Use CompressHelpers And compress the pipeline cache. Saves a fair bit of disk space. --- src/util/d3d12_device.cpp | 12 +--- src/util/d3d12_device.h | 2 +- src/util/gpu_device.cpp | 73 ++++++++++++++++++++----- src/util/gpu_device.h | 4 +- src/util/gpu_shader_cache.cpp | 64 +++++++++++----------- src/util/gpu_shader_cache.h | 3 +- src/util/opengl_context.cpp | 2 +- src/util/opengl_context.h | 2 +- src/util/opengl_context_agl.h | 2 +- src/util/opengl_context_agl.mm | 2 +- src/util/opengl_context_egl.cpp | 2 +- src/util/opengl_context_egl.h | 2 +- src/util/opengl_context_egl_wayland.cpp | 2 +- src/util/opengl_context_egl_wayland.h | 2 +- src/util/opengl_context_egl_x11.cpp | 2 +- src/util/opengl_context_egl_x11.h | 2 +- src/util/opengl_context_wgl.cpp | 2 +- src/util/opengl_context_wgl.h | 2 +- src/util/opengl_device.cpp | 4 +- src/util/opengl_device.h | 4 +- src/util/opengl_loader.h | 2 +- src/util/opengl_pipeline.cpp | 41 +++++++------- src/util/opengl_pipeline.h | 4 +- src/util/opengl_stream_buffer.cpp | 4 +- src/util/opengl_stream_buffer.h | 4 +- src/util/opengl_texture.cpp | 2 +- src/util/opengl_texture.h | 4 +- src/util/vulkan_device.cpp | 28 ++++------ src/util/vulkan_device.h | 2 +- 29 files changed, 161 insertions(+), 120 deletions(-) diff --git a/src/util/d3d12_device.cpp b/src/util/d3d12_device.cpp index f37ef4f74..b911d58fa 100644 --- a/src/util/d3d12_device.cpp +++ b/src/util/d3d12_device.cpp @@ -283,10 +283,8 @@ void D3D12Device::DestroyDevice() m_dxgi_factory.Reset(); } -bool D3D12Device::ReadPipelineCache(const std::string& filename) +bool D3D12Device::ReadPipelineCache(std::optional> data) { - std::optional> data = FileSystem::ReadBinaryFile(filename.c_str()); - HRESULT hr = m_device->CreatePipelineLibrary(data.has_value() ? data->data() : nullptr, data.has_value() ? data->size() : 0, IID_PPV_ARGS(m_pipeline_library.ReleaseAndGetAddressOf())); @@ -306,11 +304,7 @@ bool D3D12Device::ReadPipelineCache(const std::string& filename) hr = m_device->CreatePipelineLibrary(nullptr, 0, IID_PPV_ARGS(m_pipeline_library.ReleaseAndGetAddressOf())); if (SUCCEEDED(hr)) - { - // Delete cache file, it's no longer relevant. - INFO_LOG("Deleting pipeline cache file {}", filename); - FileSystem::DeleteFile(filename.c_str()); - } + return true; } if (FAILED(hr)) @@ -332,7 +326,7 @@ bool D3D12Device::GetPipelineCacheData(DynamicHeapArray* data) if (size == 0) { WARNING_LOG("Empty serialized pipeline state returned."); - return false; + return true; } data->resize(size); diff --git a/src/util/d3d12_device.h b/src/util/d3d12_device.h index 495d3b723..c411b3b69 100644 --- a/src/util/d3d12_device.h +++ b/src/util/d3d12_device.h @@ -190,7 +190,7 @@ protected: Error* error) override; void DestroyDevice() override; - bool ReadPipelineCache(const std::string& filename) override; + bool ReadPipelineCache(std::optional> data) override; bool GetPipelineCacheData(DynamicHeapArray* data) override; private: diff --git a/src/util/gpu_device.cpp b/src/util/gpu_device.cpp index 72bceb963..fd353a844 100644 --- a/src/util/gpu_device.cpp +++ b/src/util/gpu_device.cpp @@ -2,6 +2,7 @@ // SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0) #include "gpu_device.h" +#include "compress_helpers.h" #include "core/host.h" // TODO: Remove, needed for getting fullscreen mode. #include "core/settings.h" // TODO: Remove, needed for dump directory. #include "gpu_framebuffer_manager.h" @@ -14,6 +15,7 @@ #include "common/log.h" #include "common/path.h" #include "common/scoped_guard.h" +#include "common/sha1_digest.h" #include "common/string_util.h" #include "common/timer.h" @@ -43,6 +45,8 @@ Log_SetChannel(GPUDevice); std::unique_ptr g_gpu_device; static std::string s_pipeline_cache_path; +static size_t s_pipeline_cache_size; +static std::array s_pipeline_cache_hash; size_t GPUDevice::s_total_vram_usage = 0; GPUDevice::Statistics GPUDevice::s_stats = {}; @@ -427,7 +431,7 @@ void GPUDevice::OpenShaderCache(std::string_view base_path, u32 version) { const std::string basename = GetShaderCacheBaseName("pipelines"); std::string filename = Path::Combine(base_path, TinyString::from_format("{}.bin", basename)); - if (ReadPipelineCache(filename)) + if (OpenPipelineCache(filename)) s_pipeline_cache_path = std::move(filename); else WARNING_LOG("Failed to read pipeline cache."); @@ -444,12 +448,17 @@ void GPUDevice::CloseShaderCache() if (GetPipelineCacheData(&data)) { // Save disk writes if it hasn't changed, think of the poor SSDs. - FILESYSTEM_STAT_DATA sd; - if (!FileSystem::StatFile(s_pipeline_cache_path.c_str(), &sd) || sd.Size != static_cast(data.size())) + if (s_pipeline_cache_size != static_cast(data.size()) || + s_pipeline_cache_hash != SHA1Digest::GetDigest(data.cspan())) { - INFO_LOG("Writing {} bytes to '{}'", data.size(), Path::GetFileName(s_pipeline_cache_path)); - if (!FileSystem::WriteBinaryFile(s_pipeline_cache_path.c_str(), data.data(), data.size())) - ERROR_LOG("Failed to write pipeline cache to '{}'", Path::GetFileName(s_pipeline_cache_path)); + Error error; + INFO_LOG("Compressing and writing {} bytes to '{}'", data.size(), Path::GetFileName(s_pipeline_cache_path)); + if (!CompressHelpers::CompressToFile(CompressHelpers::CompressType::Zstandard, s_pipeline_cache_path.c_str(), + data.cspan(), -1, true, &error)) + { + ERROR_LOG("Failed to write pipeline cache to '{}': {}", Path::GetFileName(s_pipeline_cache_path), + error.GetDescription()); + } } else { @@ -505,7 +514,43 @@ std::string GPUDevice::GetShaderCacheBaseName(std::string_view type) const return ret; } -bool GPUDevice::ReadPipelineCache(const std::string& filename) +bool GPUDevice::OpenPipelineCache(const std::string& filename) +{ + if (FileSystem::GetPathFileSize(filename.c_str()) <= 0) + return false; + + Error error; + CompressHelpers::OptionalByteBuffer data = + CompressHelpers::DecompressFile(CompressHelpers::CompressType::Zstandard, filename.c_str(), std::nullopt, &error); + if (!data.has_value()) + { + ERROR_LOG("Failed to load pipeline cache from '{}': {}", Path::GetFileName(filename), error.GetDescription()); + data.reset(); + } + + if (data.has_value()) + { + s_pipeline_cache_size = data->size(); + s_pipeline_cache_hash = SHA1Digest::GetDigest(data->cspan()); + } + else + { + s_pipeline_cache_size = 0; + s_pipeline_cache_hash = {}; + } + + if (!ReadPipelineCache(std::move(data))) + { + s_pipeline_cache_size = 0; + s_pipeline_cache_hash = {}; + return false; + } + + INFO_LOG("Pipeline cache hash: {}", SHA1Digest::DigestToString(s_pipeline_cache_hash)); + return true; +} + +bool GPUDevice::ReadPipelineCache(std::optional> data) { return false; } @@ -744,25 +789,27 @@ std::unique_ptr GPUDevice::CreateShader(GPUShaderStage stage, GPUShad } const GPUShaderCache::CacheIndexKey key = m_shader_cache.GetCacheKey(stage, language, source, entry_point); - DynamicHeapArray binary; - if (m_shader_cache.Lookup(key, &binary)) + std::optional binary = m_shader_cache.Lookup(key); + if (binary.has_value()) { - shader = CreateShaderFromBinary(stage, binary, error); + shader = CreateShaderFromBinary(stage, binary->cspan(), error); if (shader) return shader; ERROR_LOG("Failed to create shader from binary (driver changed?). Clearing cache."); m_shader_cache.Clear(); + binary.reset(); } - shader = CreateShaderFromSource(stage, language, source, entry_point, &binary, error); + GPUShaderCache::ShaderBinary new_binary; + shader = CreateShaderFromSource(stage, language, source, entry_point, &new_binary, error); if (!shader) return shader; // Don't insert empty shaders into the cache... - if (!binary.empty()) + if (!new_binary.empty()) { - if (!m_shader_cache.Insert(key, binary.data(), static_cast(binary.size()))) + if (!m_shader_cache.Insert(key, new_binary.data(), static_cast(new_binary.size()))) m_shader_cache.Close(); } diff --git a/src/util/gpu_device.h b/src/util/gpu_device.h index fee9ec7cf..d4759e97d 100644 --- a/src/util/gpu_device.h +++ b/src/util/gpu_device.h @@ -533,6 +533,7 @@ public: static constexpr u32 MAX_RENDER_TARGETS = 4; static constexpr u32 MAX_IMAGE_RENDER_TARGETS = 2; static constexpr u32 DEFAULT_CLEAR_COLOR = 0xFF000000u; + static constexpr u32 PIPELINE_CACHE_HASH_SIZE = 20; static_assert(sizeof(GPUPipeline::GraphicsConfig::color_formats) == sizeof(GPUTexture::Format) * MAX_RENDER_TARGETS); GPUDevice(); @@ -741,7 +742,8 @@ protected: virtual void DestroyDevice() = 0; std::string GetShaderCacheBaseName(std::string_view type) const; - virtual bool ReadPipelineCache(const std::string& filename); + virtual bool OpenPipelineCache(const std::string& filename); + virtual bool ReadPipelineCache(std::optional> data); virtual bool GetPipelineCacheData(DynamicHeapArray* data); virtual std::unique_ptr CreateShaderFromBinary(GPUShaderStage stage, std::span data, diff --git a/src/util/gpu_shader_cache.cpp b/src/util/gpu_shader_cache.cpp index 2bb7e04b7..5e7e1cf1a 100644 --- a/src/util/gpu_shader_cache.cpp +++ b/src/util/gpu_shader_cache.cpp @@ -4,6 +4,7 @@ #include "gpu_shader_cache.h" #include "gpu_device.h" +#include "common/error.h" #include "common/file_system.h" #include "common/heap_array.h" #include "common/log.h" @@ -12,8 +13,7 @@ #include "fmt/format.h" -#include "zstd.h" -#include "zstd_errors.h" +#include "compress_helpers.h" Log_SetChannel(GPUShaderCache); @@ -251,42 +251,43 @@ GPUShaderCache::CacheIndexKey GPUShaderCache::GetCacheKey(GPUShaderStage stage, return key; } -bool GPUShaderCache::Lookup(const CacheIndexKey& key, ShaderBinary* binary) +std::optional GPUShaderCache::Lookup(const CacheIndexKey& key) { - auto iter = m_index.find(key); - if (iter == m_index.end()) - return false; - - binary->resize(iter->second.uncompressed_size); - - DynamicHeapArray compressed_data(iter->second.compressed_size); + std::optional ret; - if (std::fseek(m_blob_file, iter->second.file_offset, SEEK_SET) != 0 || - std::fread(compressed_data.data(), iter->second.compressed_size, 1, m_blob_file) != 1) [[unlikely]] + auto iter = m_index.find(key); + if (iter != m_index.end()) { - ERROR_LOG("Read {} byte {} shader from file failed", iter->second.compressed_size, - GPUShader::GetStageName(static_cast(key.shader_type))); - return false; - } + DynamicHeapArray compressed_data(iter->second.compressed_size); - const size_t decompress_result = - ZSTD_decompress(binary->data(), binary->size(), compressed_data.data(), compressed_data.size()); - if (ZSTD_isError(decompress_result)) [[unlikely]] - { - ERROR_LOG("Failed to decompress shader: {}", ZSTD_getErrorName(decompress_result)); - return false; + if (std::fseek(m_blob_file, iter->second.file_offset, SEEK_SET) != 0 || + std::fread(compressed_data.data(), iter->second.compressed_size, 1, m_blob_file) != 1) [[unlikely]] + { + ERROR_LOG("Read {} byte {} shader from file failed", iter->second.compressed_size, + GPUShader::GetStageName(static_cast(key.shader_type))); + } + else + { + Error error; + ret = CompressHelpers::DecompressBuffer(CompressHelpers::CompressType::Zstandard, + CompressHelpers::OptionalByteBuffer(std::move(compressed_data)), + iter->second.uncompressed_size, &error); + if (!ret.has_value()) [[unlikely]] + ERROR_LOG("Failed to decompress shader: {}", error.GetDescription()); + } } - return true; + return ret; } bool GPUShaderCache::Insert(const CacheIndexKey& key, const void* data, u32 data_size) { - DynamicHeapArray compress_buffer(ZSTD_compressBound(data_size)); - const size_t compress_result = ZSTD_compress(compress_buffer.data(), compress_buffer.size(), data, data_size, 0); - if (ZSTD_isError(compress_result)) [[unlikely]] + Error error; + CompressHelpers::OptionalByteBuffer compress_buffer = + CompressHelpers::CompressToBuffer(CompressHelpers::CompressType::Zstandard, data, data_size, -1, &error); + if (!compress_buffer.has_value()) [[unlikely]] { - ERROR_LOG("Failed to compress shader: {}", ZSTD_getErrorName(compress_result)); + ERROR_LOG("Failed to compress shader: {}", error.GetDescription()); return false; } @@ -295,7 +296,7 @@ bool GPUShaderCache::Insert(const CacheIndexKey& key, const void* data, u32 data CacheIndexData idata; idata.file_offset = static_cast(std::ftell(m_blob_file)); - idata.compressed_size = static_cast(compress_result); + idata.compressed_size = static_cast(compress_buffer->size()); idata.uncompressed_size = data_size; CacheIndexEntry entry = {}; @@ -310,8 +311,9 @@ bool GPUShaderCache::Insert(const CacheIndexKey& key, const void* data, u32 data entry.compressed_size = idata.compressed_size; entry.uncompressed_size = idata.uncompressed_size; - if (std::fwrite(compress_buffer.data(), compress_result, 1, m_blob_file) != 1 || std::fflush(m_blob_file) != 0 || - std::fwrite(&entry, sizeof(entry), 1, m_index_file) != 1 || std::fflush(m_index_file) != 0) [[unlikely]] + if (std::fwrite(compress_buffer->data(), compress_buffer->size(), 1, m_blob_file) != 1 || + std::fflush(m_blob_file) != 0 || std::fwrite(&entry, sizeof(entry), 1, m_index_file) != 1 || + std::fflush(m_index_file) != 0) [[unlikely]] { ERROR_LOG("Failed to write {} byte {} shader blob to file", data_size, GPUShader::GetStageName(static_cast(key.shader_type))); @@ -319,7 +321,7 @@ bool GPUShaderCache::Insert(const CacheIndexKey& key, const void* data, u32 data } DEV_LOG("Cached compressed {} shader: {} -> {} bytes", - GPUShader::GetStageName(static_cast(key.shader_type)), data_size, compress_result); + GPUShader::GetStageName(static_cast(key.shader_type)), data_size, compress_buffer->size()); m_index.emplace(key, idata); return true; } diff --git a/src/util/gpu_shader_cache.h b/src/util/gpu_shader_cache.h index 85bfcd8f3..4b7ea683e 100644 --- a/src/util/gpu_shader_cache.h +++ b/src/util/gpu_shader_cache.h @@ -7,6 +7,7 @@ #include "common/heap_array.h" #include "common/types.h" +#include #include #include #include @@ -56,7 +57,7 @@ public: static CacheIndexKey GetCacheKey(GPUShaderStage stage, GPUShaderLanguage language, std::string_view shader_code, std::string_view entry_point); - bool Lookup(const CacheIndexKey& key, ShaderBinary* binary); + std::optional Lookup(const CacheIndexKey& key); bool Insert(const CacheIndexKey& key, const void* data, u32 data_size); void Clear(); diff --git a/src/util/opengl_context.cpp b/src/util/opengl_context.cpp index c6e2323c5..c07dc4182 100644 --- a/src/util/opengl_context.cpp +++ b/src/util/opengl_context.cpp @@ -1,5 +1,5 @@ // SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin -// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0) +// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0) #include "opengl_context.h" #include "opengl_loader.h" diff --git a/src/util/opengl_context.h b/src/util/opengl_context.h index f7e5cedb3..b18f9e084 100644 --- a/src/util/opengl_context.h +++ b/src/util/opengl_context.h @@ -1,5 +1,5 @@ // SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin -// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0) +// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0) #pragma once diff --git a/src/util/opengl_context_agl.h b/src/util/opengl_context_agl.h index 953cae6c5..5b1fbfcd4 100644 --- a/src/util/opengl_context_agl.h +++ b/src/util/opengl_context_agl.h @@ -1,5 +1,5 @@ // SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin -// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0) +// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0) #pragma once diff --git a/src/util/opengl_context_agl.mm b/src/util/opengl_context_agl.mm index 96de6a2f0..55124919d 100644 --- a/src/util/opengl_context_agl.mm +++ b/src/util/opengl_context_agl.mm @@ -1,5 +1,5 @@ // SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin -// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0) +// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0) #include "opengl_context_agl.h" diff --git a/src/util/opengl_context_egl.cpp b/src/util/opengl_context_egl.cpp index 7fd6b219d..b5d2f206a 100644 --- a/src/util/opengl_context_egl.cpp +++ b/src/util/opengl_context_egl.cpp @@ -1,5 +1,5 @@ // SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin -// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0) +// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0) #include "opengl_context_egl.h" diff --git a/src/util/opengl_context_egl.h b/src/util/opengl_context_egl.h index 43abd7e4f..9919cae6d 100644 --- a/src/util/opengl_context_egl.h +++ b/src/util/opengl_context_egl.h @@ -1,5 +1,5 @@ // SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin -// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0) +// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0) #pragma once diff --git a/src/util/opengl_context_egl_wayland.cpp b/src/util/opengl_context_egl_wayland.cpp index 51834335b..4d8228f99 100644 --- a/src/util/opengl_context_egl_wayland.cpp +++ b/src/util/opengl_context_egl_wayland.cpp @@ -1,5 +1,5 @@ // SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin -// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0) +// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0) #include "opengl_context_egl_wayland.h" diff --git a/src/util/opengl_context_egl_wayland.h b/src/util/opengl_context_egl_wayland.h index f5a8cff8f..73781dd0f 100644 --- a/src/util/opengl_context_egl_wayland.h +++ b/src/util/opengl_context_egl_wayland.h @@ -1,5 +1,5 @@ // SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin -// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0) +// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0) #pragma once diff --git a/src/util/opengl_context_egl_x11.cpp b/src/util/opengl_context_egl_x11.cpp index af608acac..c3a8269f0 100644 --- a/src/util/opengl_context_egl_x11.cpp +++ b/src/util/opengl_context_egl_x11.cpp @@ -1,5 +1,5 @@ // SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin -// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0) +// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0) #include "opengl_context_egl_x11.h" diff --git a/src/util/opengl_context_egl_x11.h b/src/util/opengl_context_egl_x11.h index 9337a626d..5b25f7bac 100644 --- a/src/util/opengl_context_egl_x11.h +++ b/src/util/opengl_context_egl_x11.h @@ -1,5 +1,5 @@ // SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin -// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0) +// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0) #pragma once diff --git a/src/util/opengl_context_wgl.cpp b/src/util/opengl_context_wgl.cpp index a44e07bc3..070bdf211 100644 --- a/src/util/opengl_context_wgl.cpp +++ b/src/util/opengl_context_wgl.cpp @@ -1,5 +1,5 @@ // SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin -// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0) +// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0) #include "opengl_context_wgl.h" #include "opengl_loader.h" diff --git a/src/util/opengl_context_wgl.h b/src/util/opengl_context_wgl.h index 836bd0f58..23d4ff144 100644 --- a/src/util/opengl_context_wgl.h +++ b/src/util/opengl_context_wgl.h @@ -1,5 +1,5 @@ // SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin -// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0) +// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0) #pragma once diff --git a/src/util/opengl_device.cpp b/src/util/opengl_device.cpp index 57a6be524..669d9e1c1 100644 --- a/src/util/opengl_device.cpp +++ b/src/util/opengl_device.cpp @@ -1,5 +1,5 @@ -// SPDX-FileCopyrightText: 2019-2023 Connor McLaughlin -// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0) +// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin +// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0) #include "opengl_device.h" #include "opengl_pipeline.h" diff --git a/src/util/opengl_device.h b/src/util/opengl_device.h index 5bfd83892..b90416043 100644 --- a/src/util/opengl_device.h +++ b/src/util/opengl_device.h @@ -1,5 +1,5 @@ // SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin -// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0) +// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0) #pragma once @@ -137,7 +137,7 @@ protected: Error* error) override; void DestroyDevice() override; - bool ReadPipelineCache(const std::string& filename) override; + bool OpenPipelineCache(const std::string& filename) override; bool GetPipelineCacheData(DynamicHeapArray* data) override; private: diff --git a/src/util/opengl_loader.h b/src/util/opengl_loader.h index 6043ec896..1190374a2 100644 --- a/src/util/opengl_loader.h +++ b/src/util/opengl_loader.h @@ -1,5 +1,5 @@ // SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin -// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0) +// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0) #pragma once diff --git a/src/util/opengl_pipeline.cpp b/src/util/opengl_pipeline.cpp index 4e3b65db4..35778aa64 100644 --- a/src/util/opengl_pipeline.cpp +++ b/src/util/opengl_pipeline.cpp @@ -1,7 +1,8 @@ // SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin -// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0) +// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0) #include "opengl_pipeline.h" +#include "compress_helpers.h" #include "opengl_device.h" #include "opengl_stream_buffer.h" #include "shadergen.h" @@ -17,8 +18,6 @@ #include "common/string_util.h" #include "fmt/format.h" -#include "zstd.h" -#include "zstd_errors.h" #include @@ -754,7 +753,7 @@ void OpenGLDevice::SetPipeline(GPUPipeline* pipeline) } } -bool OpenGLDevice::ReadPipelineCache(const std::string& filename) +bool OpenGLDevice::OpenPipelineCache(const std::string& filename) { DebugAssert(!m_pipeline_disk_cache_file); @@ -863,7 +862,6 @@ bool OpenGLDevice::GetPipelineCacheData(DynamicHeapArray* data) GLuint OpenGLDevice::CreateProgramFromPipelineCache(const OpenGLPipeline::ProgramCacheItem& it, const GPUPipeline::GraphicsConfig& plconfig) { - DynamicHeapArray data(it.file_uncompressed_size); DynamicHeapArray compressed_data(it.file_compressed_size); if (FileSystem::FSeek64(m_pipeline_disk_cache_file, it.file_offset, SEEK_SET) != 0 || @@ -873,14 +871,15 @@ GLuint OpenGLDevice::CreateProgramFromPipelineCache(const OpenGLPipeline::Progra return 0; } - const size_t decompress_result = - ZSTD_decompress(data.data(), data.size(), compressed_data.data(), compressed_data.size()); - if (ZSTD_isError(decompress_result)) [[unlikely]] + Error error; + CompressHelpers::OptionalByteBuffer data = CompressHelpers::DecompressBuffer( + CompressHelpers::CompressType::Zstandard, CompressHelpers::OptionalByteBuffer(std::move(compressed_data)), + it.file_uncompressed_size, &error); + if (!data.has_value()) { - ERROR_LOG("Failed to decompress program from disk cache: {}", ZSTD_getErrorName(decompress_result)); + ERROR_LOG("Failed to decompress program from disk cache: {}", error.GetDescription()); return 0; } - compressed_data.deallocate(); glGetError(); GLuint prog = glCreateProgram(); @@ -890,7 +889,7 @@ GLuint OpenGLDevice::CreateProgramFromPipelineCache(const OpenGLPipeline::Progra return 0; } - glProgramBinary(prog, it.file_format, data.data(), it.file_uncompressed_size); + glProgramBinary(prog, it.file_format, data->data(), it.file_uncompressed_size); GLint link_status; glGetProgramiv(prog, GL_LINK_STATUS, &link_status); @@ -932,19 +931,21 @@ void OpenGLDevice::AddToPipelineCache(OpenGLPipeline::ProgramCacheItem* it) WARNING_LOG("Size changed from {} to {} after glGetProgramBinary()", uncompressed_data.size(), binary_size); } - DynamicHeapArray compressed_data(ZSTD_compressBound(binary_size)); - const size_t compress_result = - ZSTD_compress(compressed_data.data(), compressed_data.size(), uncompressed_data.data(), binary_size, 0); - if (ZSTD_isError(compress_result)) [[unlikely]] + Error error; + CompressHelpers::OptionalByteBuffer compressed_data = + CompressHelpers::CompressToBuffer(CompressHelpers::CompressType::Zstandard, + CompressHelpers::OptionalByteBuffer(std::move(uncompressed_data)), -1, &error); + if (!compressed_data.has_value()) [[unlikely]] { - ERROR_LOG("Failed to compress program: {}", ZSTD_getErrorName(compress_result)); + ERROR_LOG("Failed to compress program: {}", error.GetDescription()); return; } - DEV_LOG("Program binary retrieved and compressed, {} -> {} bytes, format {}", binary_size, compress_result, format); + DEV_LOG("Program binary retrieved and compressed, {} -> {} bytes, format {}", binary_size, compressed_data->size(), + format); if (FileSystem::FSeek64(m_pipeline_disk_cache_file, m_pipeline_disk_cache_data_end, SEEK_SET) != 0 || - std::fwrite(compressed_data.data(), compress_result, 1, m_pipeline_disk_cache_file) != 1) + std::fwrite(compressed_data->data(), compressed_data->size(), 1, m_pipeline_disk_cache_file) != 1) { ERROR_LOG("Failed to write binary to disk cache."); } @@ -952,8 +953,8 @@ void OpenGLDevice::AddToPipelineCache(OpenGLPipeline::ProgramCacheItem* it) it->file_format = format; it->file_offset = m_pipeline_disk_cache_data_end; it->file_uncompressed_size = static_cast(binary_size); - it->file_compressed_size = static_cast(compress_result); - m_pipeline_disk_cache_data_end += static_cast(compress_result); + it->file_compressed_size = static_cast(compressed_data->size()); + m_pipeline_disk_cache_data_end += static_cast(compressed_data->size()); m_pipeline_disk_cache_changed = true; } diff --git a/src/util/opengl_pipeline.h b/src/util/opengl_pipeline.h index 8deb64bc4..c39e5b789 100644 --- a/src/util/opengl_pipeline.h +++ b/src/util/opengl_pipeline.h @@ -1,5 +1,5 @@ -// SPDX-FileCopyrightText: 2019-2023 Connor McLaughlin -// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0) +// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin +// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0) #pragma once diff --git a/src/util/opengl_stream_buffer.cpp b/src/util/opengl_stream_buffer.cpp index 36f41f6df..7a4fef1e5 100644 --- a/src/util/opengl_stream_buffer.cpp +++ b/src/util/opengl_stream_buffer.cpp @@ -1,5 +1,5 @@ -// SPDX-FileCopyrightText: 2019-2023 Connor McLaughlin -// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0) +// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin +// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0) #include "opengl_stream_buffer.h" diff --git a/src/util/opengl_stream_buffer.h b/src/util/opengl_stream_buffer.h index 6164b6dba..2fe8f9b7e 100644 --- a/src/util/opengl_stream_buffer.h +++ b/src/util/opengl_stream_buffer.h @@ -1,5 +1,5 @@ -// SPDX-FileCopyrightText: 2019-2023 Connor McLaughlin -// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0) +// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin +// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0) #pragma once diff --git a/src/util/opengl_texture.cpp b/src/util/opengl_texture.cpp index 9882ab469..fbdb6a88d 100644 --- a/src/util/opengl_texture.cpp +++ b/src/util/opengl_texture.cpp @@ -1,5 +1,5 @@ // SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin -// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0) +// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0) #include "opengl_texture.h" #include "opengl_device.h" diff --git a/src/util/opengl_texture.h b/src/util/opengl_texture.h index 99aaeaabc..3d0702a65 100644 --- a/src/util/opengl_texture.h +++ b/src/util/opengl_texture.h @@ -1,10 +1,12 @@ // SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin -// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0) +// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0) #pragma once + #include "gpu_device.h" #include "gpu_texture.h" #include "opengl_loader.h" + #include class OpenGLDevice; diff --git a/src/util/vulkan_device.cpp b/src/util/vulkan_device.cpp index a2b830c77..810dc3071 100644 --- a/src/util/vulkan_device.cpp +++ b/src/util/vulkan_device.cpp @@ -2242,28 +2242,20 @@ void VulkanDevice::FillPipelineCacheHeader(VK_PIPELINE_CACHE_HEADER* header) std::memcpy(header->uuid, m_device_properties.pipelineCacheUUID, VK_UUID_SIZE); } -bool VulkanDevice::ReadPipelineCache(const std::string& filename) +bool VulkanDevice::ReadPipelineCache(std::optional> data) { - std::optional> data; - - auto fp = FileSystem::OpenManagedCFile(filename.c_str(), "rb"); - if (fp) + if (data.has_value()) { - data = FileSystem::ReadBinaryFile(fp.get()); - - if (data.has_value()) + if (data->size() < sizeof(VK_PIPELINE_CACHE_HEADER)) { - if (data->size() < sizeof(VK_PIPELINE_CACHE_HEADER)) - { - ERROR_LOG("Pipeline cache at '{}' is too small", Path::GetFileName(filename)); - return false; - } - - VK_PIPELINE_CACHE_HEADER header; - std::memcpy(&header, data->data(), sizeof(header)); - if (!ValidatePipelineCacheHeader(header)) - data.reset(); + ERROR_LOG("Pipeline cache is too small, ignoring."); + data.reset(); } + + VK_PIPELINE_CACHE_HEADER header; + std::memcpy(&header, data->data(), sizeof(header)); + if (!ValidatePipelineCacheHeader(header)) + data.reset(); } const VkPipelineCacheCreateInfo ci{VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO, nullptr, 0, diff --git a/src/util/vulkan_device.h b/src/util/vulkan_device.h index d23e0e7e6..994efb530 100644 --- a/src/util/vulkan_device.h +++ b/src/util/vulkan_device.h @@ -239,7 +239,7 @@ protected: Error* error) override; void DestroyDevice() override; - bool ReadPipelineCache(const std::string& filename) override; + bool ReadPipelineCache(std::optional> data) override; bool GetPipelineCacheData(DynamicHeapArray* data) override; private: