Xargon/Duckstation
1
0
Fork 0
mirror of https://github.com/RetroDECK/Duckstation.git synced 2024-09-17 08:02:41 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

GPUDevice: Use CompressHelpers

Browse source 
And compress the pipeline cache. Saves a fair bit of disk space.
This commit is contained in:
Stenzek 2024-08-26 21:33:28 +10:00
parent 8e11dd42b4
commit 2275aa85d9
No known key found for this signature in database
29 changed files with 161 additions and 120 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
src/util/d3d12_device.cpp
View file

@ -283,10 +283,8 @@ void D3D12Device::DestroyDevice()
m_dxgi_factory.Reset();
}
bool D3D12Device::ReadPipelineCache(const std::string& filename)
bool D3D12Device::ReadPipelineCache(std::optional<DynamicHeapArray<u8>> data)
{
std::optional<DynamicHeapArray<u8>> data = FileSystem::ReadBinaryFile(filename.c_str());
HRESULT hr =
m_device->CreatePipelineLibrary(data.has_value() ? data->data() : nullptr, data.has_value() ? data->size() : 0,
IID_PPV_ARGS(m_pipeline_library.ReleaseAndGetAddressOf()));
@ -306,11 +304,7 @@ bool D3D12Device::ReadPipelineCache(const std::string& filename)
hr = m_device->CreatePipelineLibrary(nullptr, 0, IID_PPV_ARGS(m_pipeline_library.ReleaseAndGetAddressOf()));
if (SUCCEEDED(hr))
{
// Delete cache file, it's no longer relevant.
INFO_LOG("Deleting pipeline cache file {}", filename);
FileSystem::DeleteFile(filename.c_str());
}
return true;
}
if (FAILED(hr))
@ -332,7 +326,7 @@ bool D3D12Device::GetPipelineCacheData(DynamicHeapArray<u8>* data)
if (size == 0)
{
WARNING_LOG("Empty serialized pipeline state returned.");
return false;
return true;
}
data->resize(size);

2
src/util/d3d12_device.h
View file

@ -190,7 +190,7 @@ protected:
Error* error) override;
void DestroyDevice() override;
bool ReadPipelineCache(const std::string& filename) override;
bool ReadPipelineCache(std::optional<DynamicHeapArray<u8>> data) override;
bool GetPipelineCacheData(DynamicHeapArray<u8>* data) override;
private:

73
src/util/gpu_device.cpp
View file

@ -2,6 +2,7 @@
// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
#include "gpu_device.h"
#include "compress_helpers.h"
#include "core/host.h" // TODO: Remove, needed for getting fullscreen mode.
#include "core/settings.h" // TODO: Remove, needed for dump directory.
#include "gpu_framebuffer_manager.h"
@ -14,6 +15,7 @@
#include "common/log.h"
#include "common/path.h"
#include "common/scoped_guard.h"
#include "common/sha1_digest.h"
#include "common/string_util.h"
#include "common/timer.h"
@ -43,6 +45,8 @@ Log_SetChannel(GPUDevice);
std::unique_ptr<GPUDevice> g_gpu_device;
static std::string s_pipeline_cache_path;
static size_t s_pipeline_cache_size;
static std::array<u8, SHA1Digest::DIGEST_SIZE> s_pipeline_cache_hash;
size_t GPUDevice::s_total_vram_usage = 0;
GPUDevice::Statistics GPUDevice::s_stats = {};
@ -427,7 +431,7 @@ void GPUDevice::OpenShaderCache(std::string_view base_path, u32 version)
{
const std::string basename = GetShaderCacheBaseName("pipelines");
std::string filename = Path::Combine(base_path, TinyString::from_format("{}.bin", basename));
if (ReadPipelineCache(filename))
if (OpenPipelineCache(filename))
s_pipeline_cache_path = std::move(filename);
else
WARNING_LOG("Failed to read pipeline cache.");
@ -444,12 +448,17 @@ void GPUDevice::CloseShaderCache()
if (GetPipelineCacheData(&data))
{
// Save disk writes if it hasn't changed, think of the poor SSDs.
FILESYSTEM_STAT_DATA sd;
if (!FileSystem::StatFile(s_pipeline_cache_path.c_str(), &sd) || sd.Size != static_cast<s64>(data.size()))
if (s_pipeline_cache_size != static_cast<s64>(data.size()) ||
s_pipeline_cache_hash != SHA1Digest::GetDigest(data.cspan()))
{
INFO_LOG("Writing {} bytes to '{}'", data.size(), Path::GetFileName(s_pipeline_cache_path));
if (!FileSystem::WriteBinaryFile(s_pipeline_cache_path.c_str(), data.data(), data.size()))
ERROR_LOG("Failed to write pipeline cache to '{}'", Path::GetFileName(s_pipeline_cache_path));
Error error;
INFO_LOG("Compressing and writing {} bytes to '{}'", data.size(), Path::GetFileName(s_pipeline_cache_path));
if (!CompressHelpers::CompressToFile(CompressHelpers::CompressType::Zstandard, s_pipeline_cache_path.c_str(),
data.cspan(), -1, true, &error))
{
ERROR_LOG("Failed to write pipeline cache to '{}': {}", Path::GetFileName(s_pipeline_cache_path),
error.GetDescription());
}
}
else
{
@ -505,7 +514,43 @@ std::string GPUDevice::GetShaderCacheBaseName(std::string_view type) const
return ret;
}
bool GPUDevice::ReadPipelineCache(const std::string& filename)
bool GPUDevice::OpenPipelineCache(const std::string& filename)
{
if (FileSystem::GetPathFileSize(filename.c_str()) <= 0)
return false;
Error error;
CompressHelpers::OptionalByteBuffer data =
CompressHelpers::DecompressFile(CompressHelpers::CompressType::Zstandard, filename.c_str(), std::nullopt, &error);
if (!data.has_value())
{
ERROR_LOG("Failed to load pipeline cache from '{}': {}", Path::GetFileName(filename), error.GetDescription());
data.reset();
}
if (data.has_value())
{
s_pipeline_cache_size = data->size();
s_pipeline_cache_hash = SHA1Digest::GetDigest(data->cspan());
}
else
{
s_pipeline_cache_size = 0;
s_pipeline_cache_hash = {};
}
if (!ReadPipelineCache(std::move(data)))
{
s_pipeline_cache_size = 0;
s_pipeline_cache_hash = {};
return false;
}
INFO_LOG("Pipeline cache hash: {}", SHA1Digest::DigestToString(s_pipeline_cache_hash));
return true;
}
bool GPUDevice::ReadPipelineCache(std::optional<DynamicHeapArray<u8>> data)
{
return false;
}
@ -744,25 +789,27 @@ std::unique_ptr<GPUShader> GPUDevice::CreateShader(GPUShaderStage stage, GPUShad
}
const GPUShaderCache::CacheIndexKey key = m_shader_cache.GetCacheKey(stage, language, source, entry_point);
DynamicHeapArray<u8> binary;
if (m_shader_cache.Lookup(key, &binary))
std::optional<GPUShaderCache::ShaderBinary> binary = m_shader_cache.Lookup(key);
if (binary.has_value())
{
shader = CreateShaderFromBinary(stage, binary, error);
shader = CreateShaderFromBinary(stage, binary->cspan(), error);
if (shader)
return shader;
ERROR_LOG("Failed to create shader from binary (driver changed?). Clearing cache.");
m_shader_cache.Clear();
binary.reset();
}
shader = CreateShaderFromSource(stage, language, source, entry_point, &binary, error);
GPUShaderCache::ShaderBinary new_binary;
shader = CreateShaderFromSource(stage, language, source, entry_point, &new_binary, error);
if (!shader)
return shader;
// Don't insert empty shaders into the cache...
if (!binary.empty())
if (!new_binary.empty())
{
if (!m_shader_cache.Insert(key, binary.data(), static_cast<u32>(binary.size())))
if (!m_shader_cache.Insert(key, new_binary.data(), static_cast<u32>(new_binary.size())))
m_shader_cache.Close();
}

4
src/util/gpu_device.h
View file

@ -533,6 +533,7 @@ public:
static constexpr u32 MAX_RENDER_TARGETS = 4;
static constexpr u32 MAX_IMAGE_RENDER_TARGETS = 2;
static constexpr u32 DEFAULT_CLEAR_COLOR = 0xFF000000u;
static constexpr u32 PIPELINE_CACHE_HASH_SIZE = 20;
static_assert(sizeof(GPUPipeline::GraphicsConfig::color_formats) == sizeof(GPUTexture::Format) * MAX_RENDER_TARGETS);
GPUDevice();
@ -741,7 +742,8 @@ protected:
virtual void DestroyDevice() = 0;
std::string GetShaderCacheBaseName(std::string_view type) const;
virtual bool ReadPipelineCache(const std::string& filename);
virtual bool OpenPipelineCache(const std::string& filename);
virtual bool ReadPipelineCache(std::optional<DynamicHeapArray<u8>> data);
virtual bool GetPipelineCacheData(DynamicHeapArray<u8>* data);
virtual std::unique_ptr<GPUShader> CreateShaderFromBinary(GPUShaderStage stage, std::span<const u8> data,

64
src/util/gpu_shader_cache.cpp
View file

@ -4,6 +4,7 @@
#include "gpu_shader_cache.h"
#include "gpu_device.h"
#include "common/error.h"
#include "common/file_system.h"
#include "common/heap_array.h"
#include "common/log.h"
@ -12,8 +13,7 @@
#include "fmt/format.h"
#include "zstd.h"
#include "zstd_errors.h"
#include "compress_helpers.h"
Log_SetChannel(GPUShaderCache);
@ -251,42 +251,43 @@ GPUShaderCache::CacheIndexKey GPUShaderCache::GetCacheKey(GPUShaderStage stage,
return key;
}
bool GPUShaderCache::Lookup(const CacheIndexKey& key, ShaderBinary* binary)
std::optional<GPUShaderCache::ShaderBinary> GPUShaderCache::Lookup(const CacheIndexKey& key)
{
auto iter = m_index.find(key);
if (iter == m_index.end())
return false;
binary->resize(iter->second.uncompressed_size);
DynamicHeapArray<u8> compressed_data(iter->second.compressed_size);
std::optional<ShaderBinary> ret;
if (std::fseek(m_blob_file, iter->second.file_offset, SEEK_SET) != 0 ||
std::fread(compressed_data.data(), iter->second.compressed_size, 1, m_blob_file) != 1) [[unlikely]]
auto iter = m_index.find(key);
if (iter != m_index.end())
{
ERROR_LOG("Read {} byte {} shader from file failed", iter->second.compressed_size,
GPUShader::GetStageName(static_cast<GPUShaderStage>(key.shader_type)));
return false;
}
DynamicHeapArray<u8> compressed_data(iter->second.compressed_size);
const size_t decompress_result =
ZSTD_decompress(binary->data(), binary->size(), compressed_data.data(), compressed_data.size());
if (ZSTD_isError(decompress_result)) [[unlikely]]
{
ERROR_LOG("Failed to decompress shader: {}", ZSTD_getErrorName(decompress_result));
return false;
if (std::fseek(m_blob_file, iter->second.file_offset, SEEK_SET) != 0 ||
std::fread(compressed_data.data(), iter->second.compressed_size, 1, m_blob_file) != 1) [[unlikely]]
{
ERROR_LOG("Read {} byte {} shader from file failed", iter->second.compressed_size,
GPUShader::GetStageName(static_cast<GPUShaderStage>(key.shader_type)));
}
else
{
Error error;
ret = CompressHelpers::DecompressBuffer(CompressHelpers::CompressType::Zstandard,
CompressHelpers::OptionalByteBuffer(std::move(compressed_data)),
iter->second.uncompressed_size, &error);
if (!ret.has_value()) [[unlikely]]
ERROR_LOG("Failed to decompress shader: {}", error.GetDescription());
}
}
return true;
return ret;
}
bool GPUShaderCache::Insert(const CacheIndexKey& key, const void* data, u32 data_size)
{
DynamicHeapArray<u8> compress_buffer(ZSTD_compressBound(data_size));
const size_t compress_result = ZSTD_compress(compress_buffer.data(), compress_buffer.size(), data, data_size, 0);
if (ZSTD_isError(compress_result)) [[unlikely]]
Error error;
CompressHelpers::OptionalByteBuffer compress_buffer =
CompressHelpers::CompressToBuffer(CompressHelpers::CompressType::Zstandard, data, data_size, -1, &error);
if (!compress_buffer.has_value()) [[unlikely]]
{
ERROR_LOG("Failed to compress shader: {}", ZSTD_getErrorName(compress_result));
ERROR_LOG("Failed to compress shader: {}", error.GetDescription());
return false;
}
@ -295,7 +296,7 @@ bool GPUShaderCache::Insert(const CacheIndexKey& key, const void* data, u32 data
CacheIndexData idata;
idata.file_offset = static_cast<u32>(std::ftell(m_blob_file));
idata.compressed_size = static_cast<u32>(compress_result);
idata.compressed_size = static_cast<u32>(compress_buffer->size());
idata.uncompressed_size = data_size;
CacheIndexEntry entry = {};
@ -310,8 +311,9 @@ bool GPUShaderCache::Insert(const CacheIndexKey& key, const void* data, u32 data
entry.compressed_size = idata.compressed_size;
entry.uncompressed_size = idata.uncompressed_size;
if (std::fwrite(compress_buffer.data(), compress_result, 1, m_blob_file) != 1 || std::fflush(m_blob_file) != 0 ||
std::fwrite(&entry, sizeof(entry), 1, m_index_file) != 1 || std::fflush(m_index_file) != 0) [[unlikely]]
if (std::fwrite(compress_buffer->data(), compress_buffer->size(), 1, m_blob_file) != 1 ||
std::fflush(m_blob_file) != 0 || std::fwrite(&entry, sizeof(entry), 1, m_index_file) != 1 ||
std::fflush(m_index_file) != 0) [[unlikely]]
{
ERROR_LOG("Failed to write {} byte {} shader blob to file", data_size,
GPUShader::GetStageName(static_cast<GPUShaderStage>(key.shader_type)));
@ -319,7 +321,7 @@ bool GPUShaderCache::Insert(const CacheIndexKey& key, const void* data, u32 data
}
DEV_LOG("Cached compressed {} shader: {} -> {} bytes",
GPUShader::GetStageName(static_cast<GPUShaderStage>(key.shader_type)), data_size, compress_result);
GPUShader::GetStageName(static_cast<GPUShaderStage>(key.shader_type)), data_size, compress_buffer->size());
m_index.emplace(key, idata);
return true;
}

3
src/util/gpu_shader_cache.h
View file

@ -7,6 +7,7 @@
#include "common/heap_array.h"
#include "common/types.h"
#include <optional>
#include <string>
#include <string_view>
#include <unordered_map>
@ -56,7 +57,7 @@ public:
static CacheIndexKey GetCacheKey(GPUShaderStage stage, GPUShaderLanguage language, std::string_view shader_code,
std::string_view entry_point);
bool Lookup(const CacheIndexKey& key, ShaderBinary* binary);
std::optional<ShaderBinary> Lookup(const CacheIndexKey& key);
bool Insert(const CacheIndexKey& key, const void* data, u32 data_size);
void Clear();

2
src/util/opengl_context.cpp
View file

@ -1,5 +1,5 @@
// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0)
#include "opengl_context.h"
#include "opengl_loader.h"

2
src/util/opengl_context.h
View file

@ -1,5 +1,5 @@
// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0)
#pragma once

2
src/util/opengl_context_agl.h
View file

@ -1,5 +1,5 @@
// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0)
#pragma once

2
src/util/opengl_context_agl.mm
View file

@ -1,5 +1,5 @@
// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0)
#include "opengl_context_agl.h"

2
src/util/opengl_context_egl.cpp
View file

@ -1,5 +1,5 @@
// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0)
#include "opengl_context_egl.h"

2
src/util/opengl_context_egl.h
View file

@ -1,5 +1,5 @@
// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0)
#pragma once

2
src/util/opengl_context_egl_wayland.cpp
View file

@ -1,5 +1,5 @@
// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0)
#include "opengl_context_egl_wayland.h"

2
src/util/opengl_context_egl_wayland.h
View file

@ -1,5 +1,5 @@
// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0)
#pragma once

2
src/util/opengl_context_egl_x11.cpp
View file

@ -1,5 +1,5 @@
// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0)
#include "opengl_context_egl_x11.h"

2
src/util/opengl_context_egl_x11.h
View file

@ -1,5 +1,5 @@
// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0)
#pragma once

2
src/util/opengl_context_wgl.cpp
View file

@ -1,5 +1,5 @@
// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0)
#include "opengl_context_wgl.h"
#include "opengl_loader.h"

2
src/util/opengl_context_wgl.h
View file

@ -1,5 +1,5 @@
// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0)
#pragma once

4
src/util/opengl_device.cpp
View file

@ -1,5 +1,5 @@
// SPDX-FileCopyrightText: 2019-2023 Connor McLaughlin <stenzek@gmail.com>
// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0)
#include "opengl_device.h"
#include "opengl_pipeline.h"

4
src/util/opengl_device.h
View file

@ -1,5 +1,5 @@
// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0)
#pragma once
@ -137,7 +137,7 @@ protected:
Error* error) override;
void DestroyDevice() override;
bool ReadPipelineCache(const std::string& filename) override;
bool OpenPipelineCache(const std::string& filename) override;
bool GetPipelineCacheData(DynamicHeapArray<u8>* data) override;
private:

2
src/util/opengl_loader.h
View file

@ -1,5 +1,5 @@
// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0)
#pragma once

41
src/util/opengl_pipeline.cpp
View file

@ -1,7 +1,8 @@
// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0)
#include "opengl_pipeline.h"
#include "compress_helpers.h"
#include "opengl_device.h"
#include "opengl_stream_buffer.h"
#include "shadergen.h"
@ -17,8 +18,6 @@
#include "common/string_util.h"
#include "fmt/format.h"
#include "zstd.h"
#include "zstd_errors.h"
#include <cerrno>
@ -754,7 +753,7 @@ void OpenGLDevice::SetPipeline(GPUPipeline* pipeline)
}
}
bool OpenGLDevice::ReadPipelineCache(const std::string& filename)
bool OpenGLDevice::OpenPipelineCache(const std::string& filename)
{
DebugAssert(!m_pipeline_disk_cache_file);
@ -863,7 +862,6 @@ bool OpenGLDevice::GetPipelineCacheData(DynamicHeapArray<u8>* data)
GLuint OpenGLDevice::CreateProgramFromPipelineCache(const OpenGLPipeline::ProgramCacheItem& it,
const GPUPipeline::GraphicsConfig& plconfig)
{
DynamicHeapArray<u8> data(it.file_uncompressed_size);
DynamicHeapArray<u8> compressed_data(it.file_compressed_size);
if (FileSystem::FSeek64(m_pipeline_disk_cache_file, it.file_offset, SEEK_SET) != 0 ||
@ -873,14 +871,15 @@ GLuint OpenGLDevice::CreateProgramFromPipelineCache(const OpenGLPipeline::Progra
return 0;
}
const size_t decompress_result =
ZSTD_decompress(data.data(), data.size(), compressed_data.data(), compressed_data.size());
if (ZSTD_isError(decompress_result)) [[unlikely]]
Error error;
CompressHelpers::OptionalByteBuffer data = CompressHelpers::DecompressBuffer(
CompressHelpers::CompressType::Zstandard, CompressHelpers::OptionalByteBuffer(std::move(compressed_data)),
it.file_uncompressed_size, &error);
if (!data.has_value())
{
ERROR_LOG("Failed to decompress program from disk cache: {}", ZSTD_getErrorName(decompress_result));
ERROR_LOG("Failed to decompress program from disk cache: {}", error.GetDescription());
return 0;
}
compressed_data.deallocate();
glGetError();
GLuint prog = glCreateProgram();
@ -890,7 +889,7 @@ GLuint OpenGLDevice::CreateProgramFromPipelineCache(const OpenGLPipeline::Progra
return 0;
}
glProgramBinary(prog, it.file_format, data.data(), it.file_uncompressed_size);
glProgramBinary(prog, it.file_format, data->data(), it.file_uncompressed_size);
GLint link_status;
glGetProgramiv(prog, GL_LINK_STATUS, &link_status);
@ -932,19 +931,21 @@ void OpenGLDevice::AddToPipelineCache(OpenGLPipeline::ProgramCacheItem* it)
WARNING_LOG("Size changed from {} to {} after glGetProgramBinary()", uncompressed_data.size(), binary_size);
}
DynamicHeapArray<u8> compressed_data(ZSTD_compressBound(binary_size));
const size_t compress_result =
ZSTD_compress(compressed_data.data(), compressed_data.size(), uncompressed_data.data(), binary_size, 0);
if (ZSTD_isError(compress_result)) [[unlikely]]
Error error;
CompressHelpers::OptionalByteBuffer compressed_data =
CompressHelpers::CompressToBuffer(CompressHelpers::CompressType::Zstandard,
CompressHelpers::OptionalByteBuffer(std::move(uncompressed_data)), -1, &error);
if (!compressed_data.has_value()) [[unlikely]]
{
ERROR_LOG("Failed to compress program: {}", ZSTD_getErrorName(compress_result));
ERROR_LOG("Failed to compress program: {}", error.GetDescription());
return;
}
DEV_LOG("Program binary retrieved and compressed, {} -> {} bytes, format {}", binary_size, compress_result, format);
DEV_LOG("Program binary retrieved and compressed, {} -> {} bytes, format {}", binary_size, compressed_data->size(),
format);
if (FileSystem::FSeek64(m_pipeline_disk_cache_file, m_pipeline_disk_cache_data_end, SEEK_SET) != 0 ||
std::fwrite(compressed_data.data(), compress_result, 1, m_pipeline_disk_cache_file) != 1)
std::fwrite(compressed_data->data(), compressed_data->size(), 1, m_pipeline_disk_cache_file) != 1)
{
ERROR_LOG("Failed to write binary to disk cache.");
}
@ -952,8 +953,8 @@ void OpenGLDevice::AddToPipelineCache(OpenGLPipeline::ProgramCacheItem* it)
it->file_format = format;
it->file_offset = m_pipeline_disk_cache_data_end;
it->file_uncompressed_size = static_cast<u32>(binary_size);
it->file_compressed_size = static_cast<u32>(compress_result);
m_pipeline_disk_cache_data_end += static_cast<u32>(compress_result);
it->file_compressed_size = static_cast<u32>(compressed_data->size());
m_pipeline_disk_cache_data_end += static_cast<u32>(compressed_data->size());
m_pipeline_disk_cache_changed = true;
}

4
src/util/opengl_pipeline.h
View file

@ -1,5 +1,5 @@
// SPDX-FileCopyrightText: 2019-2023 Connor McLaughlin <stenzek@gmail.com>
// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0)
#pragma once

4
src/util/opengl_stream_buffer.cpp
View file

@ -1,5 +1,5 @@
// SPDX-FileCopyrightText: 2019-2023 Connor McLaughlin <stenzek@gmail.com>
// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0)
#include "opengl_stream_buffer.h"

4
src/util/opengl_stream_buffer.h
View file

@ -1,5 +1,5 @@
// SPDX-FileCopyrightText: 2019-2023 Connor McLaughlin <stenzek@gmail.com>
// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0)
#pragma once

2
src/util/opengl_texture.cpp
View file

@ -1,5 +1,5 @@
// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0)
#include "opengl_texture.h"
#include "opengl_device.h"

4
src/util/opengl_texture.h
View file

@ -1,10 +1,12 @@
// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
// SPDX-License-Identifier: (GPL-3.0 OR PolyForm-Strict-1.0.0)
#pragma once
#include "gpu_device.h"
#include "gpu_texture.h"
#include "opengl_loader.h"
#include <tuple>
class OpenGLDevice;

28
src/util/vulkan_device.cpp
View file

@ -2242,28 +2242,20 @@ void VulkanDevice::FillPipelineCacheHeader(VK_PIPELINE_CACHE_HEADER* header)
std::memcpy(header->uuid, m_device_properties.pipelineCacheUUID, VK_UUID_SIZE);
}
bool VulkanDevice::ReadPipelineCache(const std::string& filename)
bool VulkanDevice::ReadPipelineCache(std::optional<DynamicHeapArray<u8>> data)
{
std::optional<DynamicHeapArray<u8>> data;
auto fp = FileSystem::OpenManagedCFile(filename.c_str(), "rb");
if (fp)
if (data.has_value())
{
data = FileSystem::ReadBinaryFile(fp.get());
if (data.has_value())
if (data->size() < sizeof(VK_PIPELINE_CACHE_HEADER))
{
if (data->size() < sizeof(VK_PIPELINE_CACHE_HEADER))
{
ERROR_LOG("Pipeline cache at '{}' is too small", Path::GetFileName(filename));
return false;
}
VK_PIPELINE_CACHE_HEADER header;
std::memcpy(&header, data->data(), sizeof(header));
if (!ValidatePipelineCacheHeader(header))
data.reset();
ERROR_LOG("Pipeline cache is too small, ignoring.");
data.reset();
}
VK_PIPELINE_CACHE_HEADER header;
std::memcpy(&header, data->data(), sizeof(header));
if (!ValidatePipelineCacheHeader(header))
data.reset();
}
const VkPipelineCacheCreateInfo ci{VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO, nullptr, 0,

2
src/util/vulkan_device.h
View file

@ -239,7 +239,7 @@ protected:
Error* error) override;
void DestroyDevice() override;
bool ReadPipelineCache(const std::string& filename) override;
bool ReadPipelineCache(std::optional<DynamicHeapArray<u8>> data) override;
bool GetPipelineCacheData(DynamicHeapArray<u8>* data) override;
private: