GameList: Support parsing Redump.org dat files

2024-10-23 07:35:57 +00:00 · 2019-11-30 23:55:05 +10:00 · 2019-11-30 23:55:05 +10:00 · ea52b9e8aa
parent 28a179ec1d
commit ea52b9e8aa
4 changed files with 189 additions and 39 deletions
--- a/src/core/CMakeLists.txt
+++ b/src/core/CMakeLists.txt
@ -73,7 +73,7 @@ set(RECOMPILER_SRCS

 target_include_directories(core PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/..")
 target_include_directories(core PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/..")
-target_link_libraries(core PUBLIC Threads::Threads YBaseLib common imgui)
+target_link_libraries(core PUBLIC Threads::Threads YBaseLib common imgui tinyxml2)
 target_link_libraries(core PRIVATE glad simpleini stb)

 if(WIN32)
--- a/src/core/core.vcxproj
+++ b/src/core/core.vcxproj
@ -110,6 +110,9 @@
    <ProjectReference Include="..\..\dep\stb\stb.vcxproj">
      <Project>{ed601289-ac1a-46b8-a8ed-17db9eb73423}</Project>
    </ProjectReference>
+    <ProjectReference Include="..\..\dep\tinyxml2\tinyxml2.vcxproj">
+      <Project>{933118a9-68c5-47b4-b151-b03c93961623}</Project>
+    </ProjectReference>
    <ProjectReference Include="..\..\dep\YBaseLib\Source\YBaseLib.vcxproj">
      <Project>{b56ce698-7300-4fa5-9609-942f1d05c5a2}</Project>
    </ProjectReference>
@ -262,10 +265,10 @@
      </PrecompiledHeader>
      <WarningLevel>Level4</WarningLevel>
      <Optimization>Disabled</Optimization>
-      <PreprocessorDefinitions>WITH_RECOMPILER=1;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>WITH_RECOMPILER=1;TINYXML2_IMPORT;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
      <SDLCheck>true</SDLCheck>
      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
-      <AdditionalIncludeDirectories>$(SolutionDir)dep\msvc\include;$(SolutionDir)dep\YBaseLib\Include;$(SolutionDir)dep\glad\include;$(SolutionDir)dep\stb\include;$(SolutionDir)dep\imgui\include;$(SolutionDir)dep\simpleini\include;$(SolutionDir)dep\xbyak\xbyak;$(SolutionDir)src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>$(SolutionDir)dep\msvc\include;$(SolutionDir)dep\YBaseLib\Include;$(SolutionDir)dep\glad\include;$(SolutionDir)dep\stb\include;$(SolutionDir)dep\imgui\include;$(SolutionDir)dep\simpleini\include;$(SolutionDir)dep\xbyak\xbyak;$(SolutionDir)dep\tinyxml2\include;$(SolutionDir)src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
      <MultiProcessorCompilation>true</MultiProcessorCompilation>
      <MinimalRebuild>false</MinimalRebuild>
      <LanguageStandard>stdcpp17</LanguageStandard>
@ -284,10 +287,10 @@
      </PrecompiledHeader>
      <WarningLevel>Level4</WarningLevel>
      <Optimization>Disabled</Optimization>
-      <PreprocessorDefinitions>WITH_RECOMPILER=1;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>WITH_RECOMPILER=1;TINYXML2_IMPORT;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
      <SDLCheck>true</SDLCheck>
      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
-      <AdditionalIncludeDirectories>$(SolutionDir)dep\msvc\include;$(SolutionDir)dep\YBaseLib\Include;$(SolutionDir)dep\glad\include;$(SolutionDir)dep\stb\include;$(SolutionDir)dep\imgui\include;$(SolutionDir)dep\simpleini\include;$(SolutionDir)dep\xbyak\xbyak;$(SolutionDir)src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>$(SolutionDir)dep\msvc\include;$(SolutionDir)dep\YBaseLib\Include;$(SolutionDir)dep\glad\include;$(SolutionDir)dep\stb\include;$(SolutionDir)dep\imgui\include;$(SolutionDir)dep\simpleini\include;$(SolutionDir)dep\xbyak\xbyak;$(SolutionDir)dep\tinyxml2\include;$(SolutionDir)src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
      <MultiProcessorCompilation>true</MultiProcessorCompilation>
      <MinimalRebuild>false</MinimalRebuild>
      <LanguageStandard>stdcpp17</LanguageStandard>
@ -306,10 +309,10 @@
      </PrecompiledHeader>
      <WarningLevel>Level4</WarningLevel>
      <Optimization>Disabled</Optimization>
-      <PreprocessorDefinitions>WITH_RECOMPILER=1;_ITERATOR_DEBUG_LEVEL=1;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUGFAST;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>WITH_RECOMPILER=1;TINYXML2_IMPORT;_ITERATOR_DEBUG_LEVEL=1;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUGFAST;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
      <SDLCheck>true</SDLCheck>
      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
-      <AdditionalIncludeDirectories>$(SolutionDir)dep\msvc\include;$(SolutionDir)dep\YBaseLib\Include;$(SolutionDir)dep\glad\include;$(SolutionDir)dep\stb\include;$(SolutionDir)dep\imgui\include;$(SolutionDir)dep\simpleini\include;$(SolutionDir)dep\xbyak\xbyak;$(SolutionDir)src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>$(SolutionDir)dep\msvc\include;$(SolutionDir)dep\YBaseLib\Include;$(SolutionDir)dep\glad\include;$(SolutionDir)dep\stb\include;$(SolutionDir)dep\imgui\include;$(SolutionDir)dep\simpleini\include;$(SolutionDir)dep\xbyak\xbyak;$(SolutionDir)dep\tinyxml2\include;$(SolutionDir)src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
      <BasicRuntimeChecks>Default</BasicRuntimeChecks>
      <MultiProcessorCompilation>true</MultiProcessorCompilation>
      <MinimalRebuild>false</MinimalRebuild>
@ -331,10 +334,10 @@
      </PrecompiledHeader>
      <WarningLevel>Level4</WarningLevel>
      <Optimization>Disabled</Optimization>
-      <PreprocessorDefinitions>WITH_RECOMPILER=1;_ITERATOR_DEBUG_LEVEL=1;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUGFAST;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>WITH_RECOMPILER=1;TINYXML2_IMPORT;_ITERATOR_DEBUG_LEVEL=1;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUGFAST;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
      <SDLCheck>true</SDLCheck>
      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
-      <AdditionalIncludeDirectories>$(SolutionDir)dep\msvc\include;$(SolutionDir)dep\YBaseLib\Include;$(SolutionDir)dep\glad\include;$(SolutionDir)dep\stb\include;$(SolutionDir)dep\imgui\include;$(SolutionDir)dep\simpleini\include;$(SolutionDir)dep\xbyak\xbyak;$(SolutionDir)src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>$(SolutionDir)dep\msvc\include;$(SolutionDir)dep\YBaseLib\Include;$(SolutionDir)dep\glad\include;$(SolutionDir)dep\stb\include;$(SolutionDir)dep\imgui\include;$(SolutionDir)dep\simpleini\include;$(SolutionDir)dep\xbyak\xbyak;$(SolutionDir)dep\tinyxml2\include;$(SolutionDir)src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
      <BasicRuntimeChecks>Default</BasicRuntimeChecks>
      <MultiProcessorCompilation>true</MultiProcessorCompilation>
      <MinimalRebuild>false</MinimalRebuild>
@ -357,8 +360,8 @@
      </PrecompiledHeader>
      <Optimization>MaxSpeed</Optimization>
      <IntrinsicFunctions>true</IntrinsicFunctions>
-      <PreprocessorDefinitions>WITH_RECOMPILER=1;_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
-      <AdditionalIncludeDirectories>$(SolutionDir)dep\msvc\include;$(SolutionDir)dep\YBaseLib\Include;$(SolutionDir)dep\glad\include;$(SolutionDir)dep\stb\include;$(SolutionDir)dep\imgui\include;$(SolutionDir)dep\simpleini\include;$(SolutionDir)dep\xbyak\xbyak;$(SolutionDir)src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WITH_RECOMPILER=1;TINYXML2_IMPORT;_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>$(SolutionDir)dep\msvc\include;$(SolutionDir)dep\YBaseLib\Include;$(SolutionDir)dep\glad\include;$(SolutionDir)dep\stb\include;$(SolutionDir)dep\imgui\include;$(SolutionDir)dep\simpleini\include;$(SolutionDir)dep\xbyak\xbyak;$(SolutionDir)dep\tinyxml2\include;$(SolutionDir)src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
      <MultiProcessorCompilation>true</MultiProcessorCompilation>
      <WholeProgramOptimization>false</WholeProgramOptimization>
      <LanguageStandard>stdcpp17</LanguageStandard>
@ -380,8 +383,8 @@
      </PrecompiledHeader>
      <Optimization>MaxSpeed</Optimization>
      <IntrinsicFunctions>true</IntrinsicFunctions>
-      <PreprocessorDefinitions>WITH_RECOMPILER=1;_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
-      <AdditionalIncludeDirectories>$(SolutionDir)dep\msvc\include;$(SolutionDir)dep\YBaseLib\Include;$(SolutionDir)dep\glad\include;$(SolutionDir)dep\stb\include;$(SolutionDir)dep\imgui\include;$(SolutionDir)dep\simpleini\include;$(SolutionDir)dep\xbyak\xbyak;$(SolutionDir)src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WITH_RECOMPILER=1;TINYXML2_IMPORT;_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>$(SolutionDir)dep\msvc\include;$(SolutionDir)dep\YBaseLib\Include;$(SolutionDir)dep\glad\include;$(SolutionDir)dep\stb\include;$(SolutionDir)dep\imgui\include;$(SolutionDir)dep\simpleini\include;$(SolutionDir)dep\xbyak\xbyak;$(SolutionDir)dep\tinyxml2\include;$(SolutionDir)src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
      <MultiProcessorCompilation>true</MultiProcessorCompilation>
      <WholeProgramOptimization>true</WholeProgramOptimization>
      <LanguageStandard>stdcpp17</LanguageStandard>
@ -404,8 +407,8 @@
      </PrecompiledHeader>
      <Optimization>MaxSpeed</Optimization>
      <IntrinsicFunctions>true</IntrinsicFunctions>
-      <PreprocessorDefinitions>WITH_RECOMPILER=1;_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
-      <AdditionalIncludeDirectories>$(SolutionDir)dep\msvc\include;$(SolutionDir)dep\YBaseLib\Include;$(SolutionDir)dep\glad\include;$(SolutionDir)dep\stb\include;$(SolutionDir)dep\imgui\include;$(SolutionDir)dep\simpleini\include;$(SolutionDir)dep\xbyak\xbyak;$(SolutionDir)src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WITH_RECOMPILER=1;TINYXML2_IMPORT;_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>$(SolutionDir)dep\msvc\include;$(SolutionDir)dep\YBaseLib\Include;$(SolutionDir)dep\glad\include;$(SolutionDir)dep\stb\include;$(SolutionDir)dep\imgui\include;$(SolutionDir)dep\simpleini\include;$(SolutionDir)dep\xbyak\xbyak;$(SolutionDir)dep\tinyxml2\include;$(SolutionDir)src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
      <MultiProcessorCompilation>true</MultiProcessorCompilation>
      <WholeProgramOptimization>false</WholeProgramOptimization>
      <LanguageStandard>stdcpp17</LanguageStandard>
@ -427,8 +430,8 @@
      </PrecompiledHeader>
      <Optimization>MaxSpeed</Optimization>
      <IntrinsicFunctions>true</IntrinsicFunctions>
-      <PreprocessorDefinitions>WITH_RECOMPILER=1;_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
-      <AdditionalIncludeDirectories>$(SolutionDir)dep\msvc\include;$(SolutionDir)dep\YBaseLib\Include;$(SolutionDir)dep\glad\include;$(SolutionDir)dep\stb\include;$(SolutionDir)dep\imgui\include;$(SolutionDir)dep\simpleini\include;$(SolutionDir)dep\xbyak\xbyak;$(SolutionDir)src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WITH_RECOMPILER=1;TINYXML2_IMPORT;_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>$(SolutionDir)dep\msvc\include;$(SolutionDir)dep\YBaseLib\Include;$(SolutionDir)dep\glad\include;$(SolutionDir)dep\stb\include;$(SolutionDir)dep\imgui\include;$(SolutionDir)dep\simpleini\include;$(SolutionDir)dep\xbyak\xbyak;$(SolutionDir)dep\tinyxml2\include;$(SolutionDir)src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
      <MultiProcessorCompilation>true</MultiProcessorCompilation>
      <WholeProgramOptimization>true</WholeProgramOptimization>
      <LanguageStandard>stdcpp17</LanguageStandard>
--- a/src/core/game_list.cpp
+++ b/src/core/game_list.cpp
@ -6,6 +6,7 @@
 #include "common/iso_reader.h"
 #include <algorithm>
 #include <cctype>
+#include <tinyxml2.h>
 #include <utility>
 Log_SetChannel(GameList);

@ -78,23 +79,36 @@ std::string GameList::GetGameCodeForImage(CDImage* cdi)
  // cdrom:\SCES_123.45;1
  std::string code = iter->second;
  std::string::size_type pos = code.rfind('\\');
-  if (pos == std::string::npos)
-    return {};
+  if (pos != std::string::npos)
+  {
    code.erase(0, pos + 1);
-  pos = code.find(';');
-  if (pos == std::string::npos)
-    return {};
+  }
+  else
+  {
+    // cdrom:SCES_123.45;1
+    pos = code.rfind(':');
+    if (pos != std::string::npos)
+      code.erase(0, pos + 1);
+  }

+  pos = code.find(';');
+  if (pos != std::string::npos)
    code.erase(pos);

  // SCES_123.45 -> SCES-12345
  for (pos = 0; pos < code.size();)
  {
-    if (code[pos] == '_')
-      code[pos++] = '-';
-    else if (code[pos] == '.')
+    if (code[pos] == '.')
+    {
      code.erase(pos, 1);
+      continue;
+    }
+
+    if (code[pos] == '_')
+      code[pos] = '-';
    else
+      code[pos] = static_cast<char>(std::toupper(code[pos]));
+
    pos++;
  }

@ -137,11 +151,24 @@ bool GameList::GetGameListEntry(const char* path, GameListEntry* entry)
    return false;

  std::string game_code = GetGameCodeForImage(cdi.get());
+  cdi.reset();

  entry->path = path;
-  entry->title = game_code;
  entry->code = game_code;
+
+  auto iter = m_database.find(game_code);
+  if (iter != m_database.end())
+  {
+    entry->title = iter->second.title;
+    entry->region = iter->second.region;
+  }
+  else
+  {
+    Log_WarningPrintf("'%s' not found in database", game_code.c_str());
+    entry->title = game_code;
    entry->region = GetRegionForCode(game_code).value_or(ConsoleRegion::NTSC_U);
+  }
+
  return true;
 }

@ -183,3 +210,105 @@ void GameList::ScanDirectory(const char* path, bool recursive)
    }
  }
 }
+
+class RedumpDatVisitor final : public tinyxml2::XMLVisitor
+{
+public:
+  RedumpDatVisitor(GameList::DatabaseMap& database) : m_database(database) {}
+
+  static std::string FixupSerial(const std::string_view str)
+  {
+    std::string ret;
+    ret.reserve(str.length());
+    for (size_t i = 0; i < str.length(); i++)
+    {
+      if (str[i] == '.' || str[i] == '#')
+        continue;
+      else if (str[i] == ',')
+        break;
+      else if (str[i] == '_' || str[i] == ' ')
+        ret.push_back('-');
+      else
+        ret.push_back(static_cast<char>(std::toupper(str[i])));
+    }
+
+    return ret;
+  }
+
+  bool VisitEnter(const tinyxml2::XMLElement& element, const tinyxml2::XMLAttribute* firstAttribute) override
+  {
+    // recurse into gamelist
+    if (Y_stricmp(element.Name(), "datafile") == 0)
+      return true;
+
+    if (Y_stricmp(element.Name(), "game") != 0)
+      return false;
+
+    const char* name = element.Attribute("name");
+    if (!name)
+      return false;
+
+    const tinyxml2::XMLElement* serial_elem = element.FirstChildElement("serial");
+    if (!serial_elem)
+      return false;
+
+    const char* serial_text = serial_elem->GetText();
+    if (!serial_text)
+      return false;
+
+    // Handle entries like <serial>SCES-00984, SCES-00984#</serial>
+    const char* start = serial_text;
+    const char* end = std::strchr(start, ',');
+    for (;;)
+    {
+      std::string code = FixupSerial(end ? std::string_view(start, end - start) : std::string_view(start));
+      auto iter = m_database.find(code);
+      if (iter == m_database.end())
+      {
+        GameList::GameDatabaseEntry gde;
+        gde.code = std::move(code);
+        gde.region = GameList::GetRegionForCode(gde.code).value_or(ConsoleRegion::NTSC_U);
+        gde.title = name;
+        m_database.emplace(gde.code, std::move(gde));
+      }
+
+      if (!end)
+        break;
+
+      start = end + 1;
+      while (std::isspace(*start))
+        start++;
+
+      end = std::strchr(start, ',');
+    }
+
+    return false;
+  }
+
+private:
+  GameList::DatabaseMap& m_database;
+};
+
+bool GameList::ParseRedumpDatabase(const char* redump_dat_path)
+{
+  tinyxml2::XMLDocument doc;
+  tinyxml2::XMLError error = doc.LoadFile(redump_dat_path);
+  if (error != tinyxml2::XML_SUCCESS)
+  {
+    Log_ErrorPrintf("Failed to parse redump dat '%s': %s", redump_dat_path,
+                    tinyxml2::XMLDocument::ErrorIDToName(error));
+    return false;
+  }
+
+  const tinyxml2::XMLElement* datafile_elem = doc.FirstChildElement("datafile");
+  if (!datafile_elem)
+  {
+    Log_ErrorPrintf("Failed to get datafile element in '%s'", redump_dat_path);
+    return false;
+  }
+
+  RedumpDatVisitor visitor(m_database);
+  datafile_elem->Accept(&visitor);
+  Log_InfoPrintf("Loaded %zu entries from Redump.org database '%s'", m_database.size(), redump_dat_path);
+  return true;
+}
--- a/src/core/game_list.h
+++ b/src/core/game_list.h
@ -10,16 +10,15 @@ class CDImage;
 class GameList
 {
 public:
-  GameList();
-  ~GameList();
+  struct GameDatabaseEntry
+  {
+    std::string code;
+    std::string title;
+    ConsoleRegion region;
+  };

-  static std::string GetGameCodeForImage(CDImage* cdi);
-  static std::string GetGameCodeForPath(const char* image_path);
-  static std::optional<ConsoleRegion> GetRegionForCode(std::string_view code);
+  using DatabaseMap = std::unordered_map<std::string, GameDatabaseEntry>;

-  void AddDirectory(const char* path, bool recursive);
-
-private:
  struct GameListEntry
  {
    std::string path;
@ -28,9 +27,28 @@ private:
    ConsoleRegion region;
  };

+  using EntryList = std::vector<GameListEntry>;
+
+  GameList();
+  ~GameList();
+
+  static std::string GetGameCodeForImage(CDImage* cdi);
+  static std::string GetGameCodeForPath(const char* image_path);
+  static std::optional<ConsoleRegion> GetRegionForCode(std::string_view code);
+
+  const DatabaseMap& GetDatabase() const { return m_database; }
+  const EntryList& GetEntries() const { return m_entries; }
+
+  void AddDirectory(const char* path, bool recursive);
+
+  bool ParseRedumpDatabase(const char* redump_dat_path);
+
+private:
+
  bool GetGameListEntry(const char* path, GameListEntry* entry);

  void ScanDirectory(const char* path, bool recursive);

-  std::vector<GameListEntry> m_entries;
+  DatabaseMap m_database;
+  EntryList m_entries;
 };