Common: Add GL Shader Cache class

src/common/CMakeLists.txt

@@ -24,6 +24,8 @@ add_library(common
   file_system.h
   gl/program.cpp
   gl/program.h
+  gl/shader_cache.cpp
+  gl/shader_cache.h
   gl/stream_buffer.cpp
   gl/stream_buffer.h
   gl/texture.cpp

src/common/common.vcxproj

@@ -51,6 +51,7 @@
     <ClInclude Include="fifo_queue.h" />
     <ClInclude Include="file_system.h" />
     <ClInclude Include="gl\program.h" />
+    <ClInclude Include="gl\shader_cache.h" />
     <ClInclude Include="gl\stream_buffer.h" />
     <ClInclude Include="gl\texture.h" />
     <ClInclude Include="hash_combine.h" />
@@ -88,6 +89,7 @@
     <ClCompile Include="d3d11\texture.cpp" />
     <ClCompile Include="file_system.cpp" />
     <ClCompile Include="gl\program.cpp" />
+    <ClCompile Include="gl\shader_cache.cpp" />
     <ClCompile Include="gl\stream_buffer.cpp" />
     <ClCompile Include="gl\texture.cpp" />
     <ClCompile Include="iso_reader.cpp" />

src/common/common.vcxproj.filters

@@ -53,6 +53,9 @@
     <ClInclude Include="hash_combine.h" />
     <ClInclude Include="progress_callback.h" />
     <ClInclude Include="wav_writer.h" />
+    <ClInclude Include="gl\shader_cache.h">
+      <Filter>gl</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="jit_code_buffer.cpp" />
@@ -102,6 +105,9 @@
     <ClCompile Include="cd_image_chd.cpp" />
     <ClCompile Include="progress_callback.cpp" />
     <ClCompile Include="wav_writer.cpp" />
+    <ClCompile Include="gl\shader_cache.cpp">
+      <Filter>gl</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <Natvis Include="bitfield.natvis" />

src/common/d3d11/shader_cache.cpp

@@ -149,6 +149,9 @@ bool ShaderCache::ReadExisting(const std::string& index_filename, const std::str
     m_index.emplace(key, data);
   }
 
+  // ensure we don't write before seeking
+  std::fseek(m_index_file, 0, SEEK_END);
+
   Log_InfoPrintf("Read %zu entries from '%s'", m_index.size(), index_filename.c_str());
   return true;
 }

src/common/gl/shader_cache.cpp

@@ -0,0 +1,308 @@
+#include "shader_cache.h"
+#include "../file_system.h"
+#include "../log.h"
+#include "../md5_digest.h"
+Log_SetChannel(GL::ShaderCache);
+
+namespace GL {
+
+#pragma pack(push, 1)
+struct CacheIndexEntry
+{
+  u64 vertex_source_hash_low;
+  u64 vertex_source_hash_high;
+  u32 vertex_source_length;
+  u64 fragment_source_hash_low;
+  u64 fragment_source_hash_high;
+  u32 fragment_source_length;
+  u32 file_offset;
+  u32 blob_size;
+  u32 blob_format;
+};
+#pragma pack(pop)
+
+ShaderCache::ShaderCache() = default;
+
+ShaderCache::~ShaderCache()
+{
+  Close();
+}
+
+bool ShaderCache::CacheIndexKey::operator==(const CacheIndexKey& key) const
+{
+  return (
+    vertex_source_hash_low == key.vertex_source_hash_low && vertex_source_hash_high == key.vertex_source_hash_high &&
+    vertex_source_length == key.vertex_source_length && fragment_source_hash_low == key.fragment_source_hash_low &&
+    fragment_source_hash_high == key.fragment_source_hash_high && fragment_source_length == key.fragment_source_length);
+}
+
+bool ShaderCache::CacheIndexKey::operator!=(const CacheIndexKey& key) const
+{
+  return (
+    vertex_source_hash_low != key.vertex_source_hash_low || vertex_source_hash_high != key.vertex_source_hash_high ||
+    vertex_source_length != key.vertex_source_length || fragment_source_hash_low != key.fragment_source_hash_low ||
+    fragment_source_hash_high != key.fragment_source_hash_high || fragment_source_length != key.fragment_source_length);
+}
+
+void ShaderCache::Open(bool is_gles, std::string_view base_path)
+{
+  m_program_binary_supported = is_gles || GLAD_GL_ARB_get_program_binary;
+  if (m_program_binary_supported)
+  {
+    // check that there's at least one format and the extension isn't being "faked"
+    GLint num_formats = 0;
+    glGetIntegerv(GL_NUM_PROGRAM_BINARY_FORMATS, &num_formats);
+    Log_InfoPrintf("%u program binary formats supported by driver", num_formats);
+    m_program_binary_supported = (num_formats > 0);
+  }
+
+  if (!m_program_binary_supported)
+  {
+    Log_WarningPrintf("Your GL driver does not support program binaries. Hopefully it has a built-in cache, otherwise "
+                      "startup will be slow due to compiling shaders.");
+    return;
+  }
+
+  const std::string base_filename = GetCacheBaseFileName(base_path);
+  const std::string index_filename = base_filename + ".idx";
+  const std::string blob_filename = base_filename + ".bin";
+
+  if (!ReadExisting(index_filename, blob_filename))
+    CreateNew(index_filename, blob_filename);
+}
+
+bool ShaderCache::CreateNew(const std::string& index_filename, const std::string& blob_filename)
+{
+  if (FileSystem::FileExists(index_filename.c_str()))
+  {
+    Log_WarningPrintf("Removing existing index file '%s'", index_filename.c_str());
+    FileSystem::DeleteFile(index_filename.c_str());
+  }
+  if (FileSystem::FileExists(blob_filename.c_str()))
+  {
+    Log_WarningPrintf("Removing existing blob file '%s'", blob_filename.c_str());
+    FileSystem::DeleteFile(blob_filename.c_str());
+  }
+
+  m_index_file = FileSystem::OpenCFile(index_filename.c_str(), "wb");
+  if (!m_index_file)
+  {
+    Log_ErrorPrintf("Failed to open index file '%s' for writing", index_filename.c_str());
+    return false;
+  }
+
+  const u32 index_version = FILE_VERSION;
+  if (std::fwrite(&index_version, sizeof(index_version), 1, m_index_file) != 1)
+  {
+    Log_ErrorPrintf("Failed to write version to index file '%s'", index_filename.c_str());
+    std::fclose(m_index_file);
+    m_index_file = nullptr;
+    FileSystem::DeleteFile(index_filename.c_str());
+    return false;
+  }
+
+  m_blob_file = FileSystem::OpenCFile(blob_filename.c_str(), "w+b");
+  if (!m_blob_file)
+  {
+    Log_ErrorPrintf("Failed to open blob file '%s' for writing", blob_filename.c_str());
+    std::fclose(m_index_file);
+    m_index_file = nullptr;
+    FileSystem::DeleteFile(index_filename.c_str());
+    return false;
+  }
+
+  return true;
+}
+
+bool ShaderCache::ReadExisting(const std::string& index_filename, const std::string& blob_filename)
+{
+  m_index_file = FileSystem::OpenCFile(index_filename.c_str(), "r+b");
+  if (!m_index_file)
+    return false;
+
+  u32 file_version;
+  if (std::fread(&file_version, sizeof(file_version), 1, m_index_file) != 1 || file_version != FILE_VERSION)
+  {
+    Log_ErrorPrintf("Bad file version in '%s'", index_filename.c_str());
+    std::fclose(m_index_file);
+    m_index_file = nullptr;
+    return false;
+  }
+
+  m_blob_file = FileSystem::OpenCFile(blob_filename.c_str(), "a+b");
+  if (!m_blob_file)
+  {
+    Log_ErrorPrintf("Blob file '%s' is missing", blob_filename.c_str());
+    std::fclose(m_index_file);
+    m_index_file = nullptr;
+    return false;
+  }
+
+  std::fseek(m_blob_file, 0, SEEK_END);
+  const u32 blob_file_size = static_cast<u32>(std::ftell(m_blob_file));
+
+  for (;;)
+  {
+    CacheIndexEntry entry;
+    if (std::fread(&entry, sizeof(entry), 1, m_index_file) != 1 ||
+        (entry.file_offset + entry.blob_size) > blob_file_size)
+    {
+      if (std::feof(m_index_file))
+        break;
+
+      Log_ErrorPrintf("Failed to read entry from '%s', corrupt file?", index_filename.c_str());
+      m_index.clear();
+      std::fclose(m_blob_file);
+      m_blob_file = nullptr;
+      std::fclose(m_index_file);
+      m_index_file = nullptr;
+      return false;
+    }
+
+    const CacheIndexKey key{entry.vertex_source_hash_low,    entry.vertex_source_hash_high,
+                            entry.vertex_source_length,      entry.fragment_source_hash_low,
+                            entry.fragment_source_hash_high, entry.fragment_source_length};
+    const CacheIndexData data{entry.file_offset, entry.blob_size, entry.blob_format};
+    m_index.emplace(key, data);
+  }
+
+  Log_InfoPrintf("Read %zu entries from '%s'", m_index.size(), index_filename.c_str());
+  return true;
+}
+
+void ShaderCache::Close()
+{
+  if (m_index_file)
+    std::fclose(m_index_file);
+  if (m_blob_file)
+    std::fclose(m_blob_file);
+}
+
+std::string ShaderCache::GetCacheBaseFileName(const std::string_view& base_path)
+{
+  std::string base_filename(base_path);
+  base_filename += FS_OSPATH_SEPERATOR_CHARACTER;
+  base_filename += "gl_programs";
+  return base_filename;
+}
+
+ShaderCache::CacheIndexKey ShaderCache::GetCacheKey(const std::string_view& vertex_shader,
+                                                    const std::string_view& fragment_shader)
+{
+  union ShaderHash
+  {
+    struct
+    {
+      u64 low;
+      u64 high;
+    };
+    u8 bytes[16];
+  };
+
+  ShaderHash vertex_hash;
+  ShaderHash fragment_hash;
+
+  MD5Digest digest;
+  digest.Update(vertex_shader.data(), static_cast<u32>(vertex_shader.length()));
+  digest.Final(vertex_hash.bytes);
+
+  digest.Reset();
+  digest.Update(fragment_shader.data(), static_cast<u32>(fragment_shader.length()));
+  digest.Final(fragment_hash.bytes);
+
+  return CacheIndexKey{vertex_hash.low,   vertex_hash.high,   static_cast<u32>(vertex_shader.length()),
+                       fragment_hash.low, fragment_hash.high, static_cast<u32>(fragment_shader.length())};
+}
+
+std::optional<Program> ShaderCache::GetProgram(const std::string_view vertex_shader,
+                                               const std::string_view fragment_shader, const PreLinkCallback& callback)
+{
+  if (!m_program_binary_supported)
+    return CompileProgram(vertex_shader, fragment_shader, callback, false);
+
+  const auto key = GetCacheKey(vertex_shader, fragment_shader);
+  auto iter = m_index.find(key);
+  if (iter == m_index.end())
+    return CompileAndAddProgram(key, vertex_shader, fragment_shader, callback);
+
+  std::vector<u8> data(iter->second.blob_size);
+  if (std::fseek(m_blob_file, iter->second.file_offset, SEEK_SET) != 0 ||
+      std::fread(data.data(), 1, iter->second.blob_size, m_blob_file) != iter->second.blob_size)
+  {
+    Log_ErrorPrintf("Read blob from file failed");
+    return {};
+  }
+
+  Program prog;
+  if (prog.CreateFromBinary(data.data(), static_cast<u32>(data.size()), iter->second.blob_format))
+    return prog;
+
+  return CompileProgram(vertex_shader, fragment_shader, callback, false);
+}
+
+std::optional<Program> ShaderCache::CompileProgram(const std::string_view& vertex_shader,
+                                                   const std::string_view& fragment_shader,
+                                                   const PreLinkCallback& callback, bool set_retrievable)
+{
+  Program prog;
+  if (!prog.Compile(vertex_shader, fragment_shader))
+    return std::nullopt;
+
+  if (callback)
+    callback(prog);
+
+  if (set_retrievable)
+    prog.SetBinaryRetrievableHint();
+
+  if (!prog.Link())
+    return std::nullopt;
+
+  return prog;
+}
+
+std::optional<Program> ShaderCache::CompileAndAddProgram(const CacheIndexKey& key,
+                                                         const std::string_view& vertex_shader,
+                                                         const std::string_view& fragment_shader,
+                                                         const PreLinkCallback& callback)
+{
+  std::optional<Program> prog = CompileProgram(vertex_shader, fragment_shader, callback, true);
+  if (!prog)
+    return std::nullopt;
+
+  std::vector<u8> prog_data;
+  u32 prog_format = 0;
+  if (!prog->GetBinary(&prog_data, &prog_format))
+    return std::nullopt;
+
+  if (!m_blob_file || std::fseek(m_blob_file, 0, SEEK_END) != 0)
+    return prog;
+
+  CacheIndexData data;
+  data.file_offset = static_cast<u32>(std::ftell(m_blob_file));
+  data.blob_size = static_cast<u32>(prog_data.size());
+  data.blob_format = prog_format;
+
+  CacheIndexEntry entry = {};
+  entry.vertex_source_hash_low = key.vertex_source_hash_low;
+  entry.vertex_source_hash_high = key.vertex_source_hash_high;
+  entry.vertex_source_length = key.vertex_source_length;
+  entry.fragment_source_hash_low = key.fragment_source_hash_low;
+  entry.fragment_source_hash_high = key.fragment_source_hash_high;
+  entry.fragment_source_length = key.fragment_source_length;
+  entry.file_offset = data.file_offset;
+  entry.blob_size = data.blob_size;
+  entry.blob_format = data.blob_format;
+
+  if (std::fwrite(prog_data.data(), 1, entry.blob_size, m_blob_file) != entry.blob_size ||
+      std::fflush(m_blob_file) != 0 || std::fwrite(&entry, sizeof(entry), 1, m_index_file) != 1 ||
+      std::fflush(m_index_file) != 0)
+  {
+    Log_ErrorPrintf("Failed to write shader blob to file");
+    return prog;
+  }
+
+  m_index.emplace(key, data);
+  return prog;
+}
+
+} // namespace GL

src/common/gl/shader_cache.h

@@ -0,0 +1,82 @@
+#pragma once
+#include "../hash_combine.h"
+#include "../types.h"
+#include "program.h"
+#include <cstdio>
+#include <functional>
+#include <optional>
+#include <string_view>
+#include <unordered_map>
+#include <vector>
+
+namespace GL {
+
+class ShaderCache
+{
+public:
+  using PreLinkCallback = std::function<void(Program&)>;
+
+  ShaderCache();
+  ~ShaderCache();
+
+  void Open(bool is_gles, std::string_view base_path);
+
+  std::optional<Program> GetProgram(const std::string_view vertex_shader, const std::string_view fragment_shader,
+                                    const PreLinkCallback& callback = {});
+
+private:
+  static constexpr u32 FILE_VERSION = 1;
+
+  struct CacheIndexKey
+  {
+    u64 vertex_source_hash_low;
+    u64 vertex_source_hash_high;
+    u32 vertex_source_length;
+    u64 fragment_source_hash_low;
+    u64 fragment_source_hash_high;
+    u32 fragment_source_length;
+
+    bool operator==(const CacheIndexKey& key) const;
+    bool operator!=(const CacheIndexKey& key) const;
+  };
+
+  struct CacheIndexEntryHasher
+  {
+    std::size_t operator()(const CacheIndexKey& e) const noexcept
+    {
+      std::size_t h = 0;
+      hash_combine(h, e.vertex_source_hash_low, e.vertex_source_hash_high, e.vertex_source_length,
+                   e.fragment_source_hash_low, e.fragment_source_hash_high, e.fragment_source_length);
+      return h;
+    }
+  };
+
+  struct CacheIndexData
+  {
+    u32 file_offset;
+    u32 blob_size;
+    u32 blob_format;
+  };
+
+  using CacheIndex = std::unordered_map<CacheIndexKey, CacheIndexData, CacheIndexEntryHasher>;
+
+  static std::string GetCacheBaseFileName(const std::string_view& base_path);
+  static CacheIndexKey GetCacheKey(const std::string_view& vertex_shader, const std::string_view& fragment_shader);
+
+  bool CreateNew(const std::string& index_filename, const std::string& blob_filename);
+  bool ReadExisting(const std::string& index_filename, const std::string& blob_filename);
+  void Close();
+
+  std::optional<Program> CompileProgram(const std::string_view& vertex_shader, const std::string_view& fragment_shader,
+                                        const PreLinkCallback& callback, bool set_retrievable);
+  std::optional<Program> CompileAndAddProgram(const CacheIndexKey& key, const std::string_view& vertex_shader,
+                                              const std::string_view& fragment_shader, const PreLinkCallback& callback);
+
+  std::FILE* m_index_file = nullptr;
+  std::FILE* m_blob_file = nullptr;
+
+  CacheIndex m_index;
+  bool m_program_binary_supported = false;
+};
+
+} // namespace GL