diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt
index 29869a750..64416c592 100644
--- a/src/common/CMakeLists.txt
+++ b/src/common/CMakeLists.txt
@@ -24,6 +24,8 @@ add_library(common
file_system.h
gl/program.cpp
gl/program.h
+ gl/shader_cache.cpp
+ gl/shader_cache.h
gl/stream_buffer.cpp
gl/stream_buffer.h
gl/texture.cpp
diff --git a/src/common/common.vcxproj b/src/common/common.vcxproj
index b8a9532c2..5c958f25d 100644
--- a/src/common/common.vcxproj
+++ b/src/common/common.vcxproj
@@ -51,6 +51,7 @@
+
@@ -88,6 +89,7 @@
+
diff --git a/src/common/common.vcxproj.filters b/src/common/common.vcxproj.filters
index 631173c7a..bc01bd681 100644
--- a/src/common/common.vcxproj.filters
+++ b/src/common/common.vcxproj.filters
@@ -53,6 +53,9 @@
+
+ gl
+
@@ -102,6 +105,9 @@
+
+ gl
+
diff --git a/src/common/d3d11/shader_cache.cpp b/src/common/d3d11/shader_cache.cpp
index 12286c8e3..82176fbba 100644
--- a/src/common/d3d11/shader_cache.cpp
+++ b/src/common/d3d11/shader_cache.cpp
@@ -149,6 +149,9 @@ bool ShaderCache::ReadExisting(const std::string& index_filename, const std::str
m_index.emplace(key, data);
}
+ // ensure we don't write before seeking
+ std::fseek(m_index_file, 0, SEEK_END);
+
Log_InfoPrintf("Read %zu entries from '%s'", m_index.size(), index_filename.c_str());
return true;
}
diff --git a/src/common/gl/shader_cache.cpp b/src/common/gl/shader_cache.cpp
new file mode 100644
index 000000000..cf9acca42
--- /dev/null
+++ b/src/common/gl/shader_cache.cpp
@@ -0,0 +1,308 @@
+#include "shader_cache.h"
+#include "../file_system.h"
+#include "../log.h"
+#include "../md5_digest.h"
+Log_SetChannel(GL::ShaderCache);
+
+namespace GL {
+
+#pragma pack(push, 1)
+struct CacheIndexEntry
+{
+ u64 vertex_source_hash_low;
+ u64 vertex_source_hash_high;
+ u32 vertex_source_length;
+ u64 fragment_source_hash_low;
+ u64 fragment_source_hash_high;
+ u32 fragment_source_length;
+ u32 file_offset;
+ u32 blob_size;
+ u32 blob_format;
+};
+#pragma pack(pop)
+
+ShaderCache::ShaderCache() = default;
+
+ShaderCache::~ShaderCache()
+{
+ Close();
+}
+
+bool ShaderCache::CacheIndexKey::operator==(const CacheIndexKey& key) const
+{
+ return (
+ vertex_source_hash_low == key.vertex_source_hash_low && vertex_source_hash_high == key.vertex_source_hash_high &&
+ vertex_source_length == key.vertex_source_length && fragment_source_hash_low == key.fragment_source_hash_low &&
+ fragment_source_hash_high == key.fragment_source_hash_high && fragment_source_length == key.fragment_source_length);
+}
+
+bool ShaderCache::CacheIndexKey::operator!=(const CacheIndexKey& key) const
+{
+ return (
+ vertex_source_hash_low != key.vertex_source_hash_low || vertex_source_hash_high != key.vertex_source_hash_high ||
+ vertex_source_length != key.vertex_source_length || fragment_source_hash_low != key.fragment_source_hash_low ||
+ fragment_source_hash_high != key.fragment_source_hash_high || fragment_source_length != key.fragment_source_length);
+}
+
+void ShaderCache::Open(bool is_gles, std::string_view base_path)
+{
+ m_program_binary_supported = is_gles || GLAD_GL_ARB_get_program_binary;
+ if (m_program_binary_supported)
+ {
+ // check that there's at least one format and the extension isn't being "faked"
+ GLint num_formats = 0;
+ glGetIntegerv(GL_NUM_PROGRAM_BINARY_FORMATS, &num_formats);
+ Log_InfoPrintf("%u program binary formats supported by driver", num_formats);
+ m_program_binary_supported = (num_formats > 0);
+ }
+
+ if (!m_program_binary_supported)
+ {
+ Log_WarningPrintf("Your GL driver does not support program binaries. Hopefully it has a built-in cache, otherwise "
+ "startup will be slow due to compiling shaders.");
+ return;
+ }
+
+ const std::string base_filename = GetCacheBaseFileName(base_path);
+ const std::string index_filename = base_filename + ".idx";
+ const std::string blob_filename = base_filename + ".bin";
+
+ if (!ReadExisting(index_filename, blob_filename))
+ CreateNew(index_filename, blob_filename);
+}
+
+bool ShaderCache::CreateNew(const std::string& index_filename, const std::string& blob_filename)
+{
+ if (FileSystem::FileExists(index_filename.c_str()))
+ {
+ Log_WarningPrintf("Removing existing index file '%s'", index_filename.c_str());
+ FileSystem::DeleteFile(index_filename.c_str());
+ }
+ if (FileSystem::FileExists(blob_filename.c_str()))
+ {
+ Log_WarningPrintf("Removing existing blob file '%s'", blob_filename.c_str());
+ FileSystem::DeleteFile(blob_filename.c_str());
+ }
+
+ m_index_file = FileSystem::OpenCFile(index_filename.c_str(), "wb");
+ if (!m_index_file)
+ {
+ Log_ErrorPrintf("Failed to open index file '%s' for writing", index_filename.c_str());
+ return false;
+ }
+
+ const u32 index_version = FILE_VERSION;
+ if (std::fwrite(&index_version, sizeof(index_version), 1, m_index_file) != 1)
+ {
+ Log_ErrorPrintf("Failed to write version to index file '%s'", index_filename.c_str());
+ std::fclose(m_index_file);
+ m_index_file = nullptr;
+ FileSystem::DeleteFile(index_filename.c_str());
+ return false;
+ }
+
+ m_blob_file = FileSystem::OpenCFile(blob_filename.c_str(), "w+b");
+ if (!m_blob_file)
+ {
+ Log_ErrorPrintf("Failed to open blob file '%s' for writing", blob_filename.c_str());
+ std::fclose(m_index_file);
+ m_index_file = nullptr;
+ FileSystem::DeleteFile(index_filename.c_str());
+ return false;
+ }
+
+ return true;
+}
+
+bool ShaderCache::ReadExisting(const std::string& index_filename, const std::string& blob_filename)
+{
+ m_index_file = FileSystem::OpenCFile(index_filename.c_str(), "r+b");
+ if (!m_index_file)
+ return false;
+
+ u32 file_version;
+ if (std::fread(&file_version, sizeof(file_version), 1, m_index_file) != 1 || file_version != FILE_VERSION)
+ {
+ Log_ErrorPrintf("Bad file version in '%s'", index_filename.c_str());
+ std::fclose(m_index_file);
+ m_index_file = nullptr;
+ return false;
+ }
+
+ m_blob_file = FileSystem::OpenCFile(blob_filename.c_str(), "a+b");
+ if (!m_blob_file)
+ {
+ Log_ErrorPrintf("Blob file '%s' is missing", blob_filename.c_str());
+ std::fclose(m_index_file);
+ m_index_file = nullptr;
+ return false;
+ }
+
+ std::fseek(m_blob_file, 0, SEEK_END);
+ const u32 blob_file_size = static_cast(std::ftell(m_blob_file));
+
+ for (;;)
+ {
+ CacheIndexEntry entry;
+ if (std::fread(&entry, sizeof(entry), 1, m_index_file) != 1 ||
+ (entry.file_offset + entry.blob_size) > blob_file_size)
+ {
+ if (std::feof(m_index_file))
+ break;
+
+ Log_ErrorPrintf("Failed to read entry from '%s', corrupt file?", index_filename.c_str());
+ m_index.clear();
+ std::fclose(m_blob_file);
+ m_blob_file = nullptr;
+ std::fclose(m_index_file);
+ m_index_file = nullptr;
+ return false;
+ }
+
+ const CacheIndexKey key{entry.vertex_source_hash_low, entry.vertex_source_hash_high,
+ entry.vertex_source_length, entry.fragment_source_hash_low,
+ entry.fragment_source_hash_high, entry.fragment_source_length};
+ const CacheIndexData data{entry.file_offset, entry.blob_size, entry.blob_format};
+ m_index.emplace(key, data);
+ }
+
+ Log_InfoPrintf("Read %zu entries from '%s'", m_index.size(), index_filename.c_str());
+ return true;
+}
+
+void ShaderCache::Close()
+{
+ if (m_index_file)
+ std::fclose(m_index_file);
+ if (m_blob_file)
+ std::fclose(m_blob_file);
+}
+
+std::string ShaderCache::GetCacheBaseFileName(const std::string_view& base_path)
+{
+ std::string base_filename(base_path);
+ base_filename += FS_OSPATH_SEPERATOR_CHARACTER;
+ base_filename += "gl_programs";
+ return base_filename;
+}
+
+ShaderCache::CacheIndexKey ShaderCache::GetCacheKey(const std::string_view& vertex_shader,
+ const std::string_view& fragment_shader)
+{
+ union ShaderHash
+ {
+ struct
+ {
+ u64 low;
+ u64 high;
+ };
+ u8 bytes[16];
+ };
+
+ ShaderHash vertex_hash;
+ ShaderHash fragment_hash;
+
+ MD5Digest digest;
+ digest.Update(vertex_shader.data(), static_cast(vertex_shader.length()));
+ digest.Final(vertex_hash.bytes);
+
+ digest.Reset();
+ digest.Update(fragment_shader.data(), static_cast(fragment_shader.length()));
+ digest.Final(fragment_hash.bytes);
+
+ return CacheIndexKey{vertex_hash.low, vertex_hash.high, static_cast(vertex_shader.length()),
+ fragment_hash.low, fragment_hash.high, static_cast(fragment_shader.length())};
+}
+
+std::optional ShaderCache::GetProgram(const std::string_view vertex_shader,
+ const std::string_view fragment_shader, const PreLinkCallback& callback)
+{
+ if (!m_program_binary_supported)
+ return CompileProgram(vertex_shader, fragment_shader, callback, false);
+
+ const auto key = GetCacheKey(vertex_shader, fragment_shader);
+ auto iter = m_index.find(key);
+ if (iter == m_index.end())
+ return CompileAndAddProgram(key, vertex_shader, fragment_shader, callback);
+
+ std::vector data(iter->second.blob_size);
+ if (std::fseek(m_blob_file, iter->second.file_offset, SEEK_SET) != 0 ||
+ std::fread(data.data(), 1, iter->second.blob_size, m_blob_file) != iter->second.blob_size)
+ {
+ Log_ErrorPrintf("Read blob from file failed");
+ return {};
+ }
+
+ Program prog;
+ if (prog.CreateFromBinary(data.data(), static_cast(data.size()), iter->second.blob_format))
+ return prog;
+
+ return CompileProgram(vertex_shader, fragment_shader, callback, false);
+}
+
+std::optional ShaderCache::CompileProgram(const std::string_view& vertex_shader,
+ const std::string_view& fragment_shader,
+ const PreLinkCallback& callback, bool set_retrievable)
+{
+ Program prog;
+ if (!prog.Compile(vertex_shader, fragment_shader))
+ return std::nullopt;
+
+ if (callback)
+ callback(prog);
+
+ if (set_retrievable)
+ prog.SetBinaryRetrievableHint();
+
+ if (!prog.Link())
+ return std::nullopt;
+
+ return prog;
+}
+
+std::optional ShaderCache::CompileAndAddProgram(const CacheIndexKey& key,
+ const std::string_view& vertex_shader,
+ const std::string_view& fragment_shader,
+ const PreLinkCallback& callback)
+{
+ std::optional prog = CompileProgram(vertex_shader, fragment_shader, callback, true);
+ if (!prog)
+ return std::nullopt;
+
+ std::vector prog_data;
+ u32 prog_format = 0;
+ if (!prog->GetBinary(&prog_data, &prog_format))
+ return std::nullopt;
+
+ if (!m_blob_file || std::fseek(m_blob_file, 0, SEEK_END) != 0)
+ return prog;
+
+ CacheIndexData data;
+ data.file_offset = static_cast(std::ftell(m_blob_file));
+ data.blob_size = static_cast(prog_data.size());
+ data.blob_format = prog_format;
+
+ CacheIndexEntry entry = {};
+ entry.vertex_source_hash_low = key.vertex_source_hash_low;
+ entry.vertex_source_hash_high = key.vertex_source_hash_high;
+ entry.vertex_source_length = key.vertex_source_length;
+ entry.fragment_source_hash_low = key.fragment_source_hash_low;
+ entry.fragment_source_hash_high = key.fragment_source_hash_high;
+ entry.fragment_source_length = key.fragment_source_length;
+ entry.file_offset = data.file_offset;
+ entry.blob_size = data.blob_size;
+ entry.blob_format = data.blob_format;
+
+ if (std::fwrite(prog_data.data(), 1, entry.blob_size, m_blob_file) != entry.blob_size ||
+ std::fflush(m_blob_file) != 0 || std::fwrite(&entry, sizeof(entry), 1, m_index_file) != 1 ||
+ std::fflush(m_index_file) != 0)
+ {
+ Log_ErrorPrintf("Failed to write shader blob to file");
+ return prog;
+ }
+
+ m_index.emplace(key, data);
+ return prog;
+}
+
+} // namespace GL
\ No newline at end of file
diff --git a/src/common/gl/shader_cache.h b/src/common/gl/shader_cache.h
new file mode 100644
index 000000000..6a139a870
--- /dev/null
+++ b/src/common/gl/shader_cache.h
@@ -0,0 +1,82 @@
+#pragma once
+#include "../hash_combine.h"
+#include "../types.h"
+#include "program.h"
+#include
+#include
+#include
+#include
+#include
+#include
+
+namespace GL {
+
+class ShaderCache
+{
+public:
+ using PreLinkCallback = std::function;
+
+ ShaderCache();
+ ~ShaderCache();
+
+ void Open(bool is_gles, std::string_view base_path);
+
+ std::optional GetProgram(const std::string_view vertex_shader, const std::string_view fragment_shader,
+ const PreLinkCallback& callback = {});
+
+private:
+ static constexpr u32 FILE_VERSION = 1;
+
+ struct CacheIndexKey
+ {
+ u64 vertex_source_hash_low;
+ u64 vertex_source_hash_high;
+ u32 vertex_source_length;
+ u64 fragment_source_hash_low;
+ u64 fragment_source_hash_high;
+ u32 fragment_source_length;
+
+ bool operator==(const CacheIndexKey& key) const;
+ bool operator!=(const CacheIndexKey& key) const;
+ };
+
+ struct CacheIndexEntryHasher
+ {
+ std::size_t operator()(const CacheIndexKey& e) const noexcept
+ {
+ std::size_t h = 0;
+ hash_combine(h, e.vertex_source_hash_low, e.vertex_source_hash_high, e.vertex_source_length,
+ e.fragment_source_hash_low, e.fragment_source_hash_high, e.fragment_source_length);
+ return h;
+ }
+ };
+
+ struct CacheIndexData
+ {
+ u32 file_offset;
+ u32 blob_size;
+ u32 blob_format;
+ };
+
+ using CacheIndex = std::unordered_map;
+
+ static std::string GetCacheBaseFileName(const std::string_view& base_path);
+ static CacheIndexKey GetCacheKey(const std::string_view& vertex_shader, const std::string_view& fragment_shader);
+
+ bool CreateNew(const std::string& index_filename, const std::string& blob_filename);
+ bool ReadExisting(const std::string& index_filename, const std::string& blob_filename);
+ void Close();
+
+ std::optional CompileProgram(const std::string_view& vertex_shader, const std::string_view& fragment_shader,
+ const PreLinkCallback& callback, bool set_retrievable);
+ std::optional CompileAndAddProgram(const CacheIndexKey& key, const std::string_view& vertex_shader,
+ const std::string_view& fragment_shader, const PreLinkCallback& callback);
+
+ std::FILE* m_index_file = nullptr;
+ std::FILE* m_blob_file = nullptr;
+
+ CacheIndex m_index;
+ bool m_program_binary_supported = false;
+};
+
+} // namespace GL