GameList: Add cover downloader

CMakeLists.txt

@@ -95,6 +95,9 @@ if(NOT ANDROID)
   if(NOT WIN32 AND USE_SDL2)
     find_package(SDL2 REQUIRED)
   endif()
+  if(NOT WIN32)
+    find_package(CURL REQUIRED)
+  endif()
   if(BUILD_QT_FRONTEND)
     find_package(Qt6 COMPONENTS Core Gui Widgets Network LinguistTools REQUIRED)
   endif()
@@ -129,9 +132,6 @@ if(USE_EVDEV)
 endif()
 if(ENABLE_CHEEVOS)
   message(STATUS "RetroAchievements support enabled")
-  if(NOT WIN32 AND NOT ANDROID)
-    find_package(CURL REQUIRED)
-  endif()
 endif()
 
 

src/common/CMakeLists.txt

@@ -20,6 +20,8 @@ add_library(common
   hash_combine.h
   heap_array.h
   heterogeneous_containers.h
+  http_downloader.cpp
+  http_downloader.h
   layered_settings_interface.cpp
   layered_settings_interface.h
   log.cpp
@@ -86,6 +88,8 @@ if(WIN32)
     d3d11/stream_buffer.h
     d3d11/texture.cpp
     d3d11/texture.h
+    http_downloader_winhttp.cpp
+    http_downloader_winhttp.h
     thirdparty/StackWalker.cpp
     thirdparty/StackWalker.h
     win32_progress_callback.cpp
@@ -95,6 +99,16 @@ if(WIN32)
   target_link_libraries(common PRIVATE d3dcompiler.lib)
 endif()
 
+if(NOT WIN32 AND NOT ANDROID)
+  target_sources(common PRIVATE
+    http_downloader_curl.cpp
+    http_downloader_curl.h
+  )
+  target_link_libraries(common PRIVATE
+    CURL::libcurl
+  )
+endif()
+
 if(ANDROID)
   target_link_libraries(common PRIVATE log)
 endif()
@@ -242,29 +256,6 @@ if(ENABLE_VULKAN)
   endif()
 endif()
 
-
-
-if(ENABLE_CHEEVOS)
-  target_sources(common PRIVATE
-    http_downloader.cpp
-    http_downloader.h
-  )
-  if(WIN32)
-    target_sources(common PRIVATE
-      http_downloader_winhttp.cpp
-      http_downloader_winhttp.h
-    )
-  elseif(NOT ANDROID)
-    target_sources(common PRIVATE
-      http_downloader_curl.cpp
-      http_downloader_curl.h
-    )
-    target_link_libraries(common PRIVATE
-      CURL::libcurl
-    )
-  endif()
-endif()
-
 if(${CMAKE_SYSTEM_NAME} STREQUAL "Linux")
   # We need -lrt for shm_unlink
   target_link_libraries(common PRIVATE rt)

src/common/http_downloader.cpp

@@ -183,4 +183,74 @@ u32 HTTPDownloader::LockedGetActiveRequestCount()
   return count;
 }
 
-} // namespace FrontendCommon
+std::string HTTPDownloader::URLEncode(const std::string_view& str)
+{
+  std::string ret;
+  ret.reserve(str.length() + ((str.length() + 3) / 4) * 3);
+
+  for (size_t i = 0, l = str.size(); i < l; i++)
+  {
+    const char c = str[i];
+    if ((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '-' || c == '_' ||
+        c == '.' || c == '!' || c == '~' || c == '*' || c == '\'' || c == '(' || c == ')')
+    {
+      ret.push_back(c);
+    }
+    else
+    {
+      ret.push_back('%');
+
+      const unsigned char n1 = static_cast<unsigned char>(c) >> 4;
+      const unsigned char n2 = static_cast<unsigned char>(c) & 0x0F;
+      ret.push_back((n1 >= 10) ? ('a' + (n1 - 10)) : ('0' + n1));
+      ret.push_back((n2 >= 10) ? ('a' + (n2 - 10)) : ('0' + n2));
+    }
+  }
+
+  return ret;
+}
+
+std::string HTTPDownloader::URLDecode(const std::string_view& str)
+{
+  std::string ret;
+  ret.reserve(str.length());
+
+  for (size_t i = 0, l = str.size(); i < l; i++)
+  {
+    const char c = str[i];
+    if (c == '+')
+    {
+      ret.push_back(c);
+    }
+    else if (c == '%')
+    {
+      if ((i + 2) >= str.length())
+        break;
+
+      const char clower = str[i + 1];
+      const char cupper = str[i + 2];
+      const unsigned char lower =
+        (clower >= '0' && clower <= '9') ?
+          static_cast<unsigned char>(clower - '0') :
+          ((clower >= 'a' && clower <= 'f') ?
+             static_cast<unsigned char>(clower - 'a') :
+             ((clower >= 'A' && clower <= 'F') ? static_cast<unsigned char>(clower - 'A') : 0));
+      const unsigned char upper =
+        (cupper >= '0' && cupper <= '9') ?
+          static_cast<unsigned char>(cupper - '0') :
+          ((cupper >= 'a' && cupper <= 'f') ?
+             static_cast<unsigned char>(cupper - 'a') :
+             ((cupper >= 'A' && cupper <= 'F') ? static_cast<unsigned char>(cupper - 'A') : 0));
+      const char dch = static_cast<char>(lower | (upper << 4));
+      ret.push_back(dch);
+    }
+    else
+    {
+      ret.push_back(c);
+    }
+  }
+
+  return std::string(str);
+}
+
+} // namespace Common

src/common/http_downloader.h

@@ -5,6 +5,7 @@
 #include <memory>
 #include <mutex>
 #include <string>
+#include <string_view>
 #include <vector>
 
 namespace Common {
@@ -53,6 +54,8 @@ public:
   virtual ~HTTPDownloader();
 
   static std::unique_ptr<HTTPDownloader> Create(const char* user_agent = DEFAULT_USER_AGENT);
+  static std::string URLEncode(const std::string_view& str);
+  static std::string URLDecode(const std::string_view& str);
 
   void SetTimeout(float timeout);
   void SetMaxActiveRequests(u32 max_active_requests);

src/duckstation-nogui/nogui_host.cpp

@@ -76,7 +76,9 @@ static void CPUThreadMainLoop();
 static std::unique_ptr<NoGUIPlatform> CreatePlatform();
 static std::string GetWindowTitle(const std::string& game_title);
 static void UpdateWindowTitle(const std::string& game_title);
-static void GameListRefreshThreadEntryPoint(bool invalidate_cache);
+static void CancelAsyncOp();
+static void StartAsyncOp(std::function<void(ProgressCallback*)> callback);
+static void AsyncOpThreadEntryPoint(std::function<void(ProgressCallback*)> callback);
 static bool AcquireHostDisplay(RenderAPI api);
 static void ReleaseHostDisplay();
 } // namespace NoGUIHost
@@ -99,9 +101,9 @@ static std::condition_variable s_cpu_thread_event_posted;
 static std::deque<std::pair<std::function<void()>, bool>> s_cpu_thread_events;
 static u32 s_blocking_cpu_events_pending = 0; // TODO: Token system would work better here.
 
-static std::mutex s_game_list_refresh_lock;
+static std::mutex s_async_op_mutex;
-static std::thread s_game_list_refresh_thread;
+static std::thread s_async_op_thread;
-static FullscreenUI::ProgressCallback* s_game_list_refresh_progress = nullptr;
+static FullscreenUI::ProgressCallback* s_async_op_progress = nullptr;
 
 //////////////////////////////////////////////////////////////////////////
 // Initialization/Shutdown
@@ -961,39 +963,66 @@ void Host::RunOnCPUThread(std::function<void()> function, bool block /* = false
     s_cpu_thread_event_done.wait(lock, []() { return s_blocking_cpu_events_pending == 0; });
 }
 
-void NoGUIHost::GameListRefreshThreadEntryPoint(bool invalidate_cache)
+void NoGUIHost::StartAsyncOp(std::function<void(ProgressCallback*)> callback)
 {
-  Threading::SetNameOfCurrentThread("Game List Refresh");
+  CancelAsyncOp();
+  s_async_op_thread = std::thread(AsyncOpThreadEntryPoint, std::move(callback));
+}
 
-  FullscreenUI::ProgressCallback callback("game_list_refresh");
+void NoGUIHost::CancelAsyncOp()
-  std::unique_lock lock(s_game_list_refresh_lock);
+{
-  s_game_list_refresh_progress = &callback;
+  std::unique_lock lock(s_async_op_mutex);
+  if (!s_async_op_thread.joinable())
+    return;
+
+  if (s_async_op_progress)
+    s_async_op_progress->SetCancelled();
 
   lock.unlock();
-  GameList::Refresh(invalidate_cache, false, &callback);
+  s_async_op_thread.join();
+}
+
+void NoGUIHost::AsyncOpThreadEntryPoint(std::function<void(ProgressCallback*)> callback)
+{
+  Threading::SetNameOfCurrentThread("Async Op");
+
+  FullscreenUI::ProgressCallback fs_callback("async_op");
+  std::unique_lock lock(s_async_op_mutex);
+  s_async_op_progress = &fs_callback;
+
+  lock.unlock();
+  callback(&fs_callback);
   lock.lock();
 
-  s_game_list_refresh_progress = nullptr;
+  s_async_op_progress = nullptr;
 }
 
 void Host::RefreshGameListAsync(bool invalidate_cache)
 {
-  CancelGameListRefresh();
+  NoGUIHost::StartAsyncOp(
-
+    [invalidate_cache](ProgressCallback* progress) { GameList::Refresh(invalidate_cache, false, progress); });
-  s_game_list_refresh_thread = std::thread(NoGUIHost::GameListRefreshThreadEntryPoint, invalidate_cache);
 }
 
 void Host::CancelGameListRefresh()
 {
-  std::unique_lock lock(s_game_list_refresh_lock);
+  NoGUIHost::CancelAsyncOp();
-  if (!s_game_list_refresh_thread.joinable())
+}
-    return;
 
-  if (s_game_list_refresh_progress)
+void Host::DownloadCoversAsync(std::vector<std::string> url_templates)
-    s_game_list_refresh_progress->SetCancelled();
+{
+  NoGUIHost::StartAsyncOp([url_templates = std::move(url_templates)](ProgressCallback* progress) {
+    GameList::DownloadCovers(url_templates, progress);
+  });
+}
 
-  lock.unlock();
+void Host::CancelCoversDownload()
-  s_game_list_refresh_thread.join();
+{
+  NoGUIHost::CancelAsyncOp();
+}
+
+void Host::CoversChanged()
+{
+  Host::RunOnCPUThread([]() { FullscreenUI::InvalidateCoverCache(); });
 }
 
 bool Host::IsFullscreen()
@@ -1340,6 +1369,7 @@ int main(int argc, char* argv[])
 
   g_nogui_window->RunMessageLoop();
 
+  NoGUIHost::CancelAsyncOp();
   NoGUIHost::StopCPUThread();
 
   // Ensure log is flushed.

src/duckstation-qt/qthost.cpp

@@ -1078,6 +1078,21 @@ void Host::CancelGameListRefresh()
   QMetaObject::invokeMethod(g_main_window, "cancelGameListRefresh", Qt::BlockingQueuedConnection);
 }
 
+void Host::DownloadCoversAsync(std::vector<std::string> url_templates)
+{
+  //
+}
+
+void Host::CancelCoversDownload()
+{
+  //
+}
+
+void Host::CoversChanged()
+{
+  //
+}
+
 void EmuThread::loadState(const QString& filename)
 {
   if (!isOnThread())

src/frontend-common/fullscreen_ui.cpp

@@ -656,6 +656,14 @@ void FullscreenUI::Render()
   ImGuiFullscreen::ResetCloseMenuIfNeeded();
 }
 
+void FullscreenUI::InvalidateCoverCache()
+{
+  if (!IsInitialized())
+    return;
+
+  Host::RunOnCPUThread([]() { s_cover_image_map.clear(); });
+}
+
 void FullscreenUI::ReturnToMainWindow()
 {
   if (s_pause_menu_was_open)

src/frontend-common/fullscreen_ui.h

@@ -21,6 +21,7 @@ bool OpenLeaderboardsWindow();
 
 void Shutdown();
 void Render();
+void InvalidateCoverCache();
 
 // Returns true if the message has been dismissed.
 bool DrawErrorWindow(const char* message);

src/frontend-common/game_list.cpp

@@ -2,6 +2,7 @@
 #include "common/assert.h"
 #include "common/byte_stream.h"
 #include "common/file_system.h"
+#include "common/http_downloader.h"
 #include "common/log.h"
 #include "common/make_array.h"
 #include "common/path.h"
@@ -18,9 +19,9 @@
 #include <array>
 #include <cctype>
 #include <ctime>
-#include <unordered_map>
 #include <string_view>
 #include <tinyxml2.h>
+#include <unordered_map>
 #include <utility>
 Log_SetChannel(GameList);
 
@@ -688,3 +689,113 @@ size_t GameList::Entry::GetReleaseDateString(char* buffer, size_t buffer_size) c
 
   return std::strftime(buffer, buffer_size, "%d %B %Y", &date_tm);
 }
+
+bool GameList::DownloadCovers(const std::vector<std::string>& url_templates, ProgressCallback* progress /*= nullptr*/)
+{
+  if (!progress)
+    progress = ProgressCallback::NullProgressCallback;
+
+  bool has_title = false;
+  bool has_file_title = false;
+  bool has_serial = false;
+  for (const std::string& url_template : url_templates)
+  {
+    if (!has_title && url_template.find("${title}") != std::string::npos)
+      has_title = true;
+    if (!has_file_title && url_template.find("${filetitle}") != std::string::npos)
+      has_file_title = true;
+    if (!has_serial && url_template.find("${serial}") != std::string::npos)
+      has_serial = true;
+  }
+  if (!has_title && !has_file_title && !has_serial)
+  {
+    progress->DisplayError("URL template must contain at least one of ${title}, ${filetitle}, or ${serial}.");
+    return false;
+  }
+
+  std::vector<std::pair<std::string, std::string>> download_urls;
+  {
+    std::unique_lock lock(s_mutex);
+    for (const GameList::Entry& entry : m_entries)
+    {
+      const std::string existing_path(GetCoverImagePathForEntry(&entry));
+      if (!existing_path.empty())
+        continue;
+
+      for (const std::string& url_template : url_templates)
+      {
+        std::string url(url_template);
+        if (has_title)
+          StringUtil::ReplaceAll(&url, "${title}", Common::HTTPDownloader::URLEncode(entry.title));
+        if (has_file_title)
+        {
+          std::string display_name(FileSystem::GetDisplayNameFromPath(entry.path));
+          StringUtil::ReplaceAll(&url, "${filetitle}",
+                                 Common::HTTPDownloader::URLEncode(Path::GetFileTitle(display_name)));
+        }
+        if (has_serial)
+          StringUtil::ReplaceAll(&url, "${serial}", Common::HTTPDownloader::URLEncode(entry.serial));
+
+        download_urls.emplace_back(entry.path, std::move(url));
+      }
+    }
+  }
+  if (download_urls.empty())
+  {
+    progress->DisplayError("No URLs to download enumerated.");
+    return false;
+  }
+
+  std::unique_ptr<Common::HTTPDownloader> downloader(Common::HTTPDownloader::Create());
+  if (!downloader)
+  {
+    progress->DisplayError("Failed to create HTTP downloader.");
+    return false;
+  }
+
+  progress->SetCancellable(true);
+  progress->SetProgressRange(static_cast<u32>(download_urls.size()));
+
+  for (auto& [entry_path, url] : download_urls)
+  {
+    if (progress->IsCancelled())
+      break;
+
+    // make sure it didn't get done already
+    {
+      std::unique_lock lock(s_mutex);
+      const GameList::Entry* entry = GetEntryForPath(entry_path.c_str());
+      if (!entry || !GetCoverImagePathForEntry(entry).empty())
+      {
+        progress->IncrementProgressValue();
+        continue;
+      }
+
+      progress->SetFormattedStatusText("Downloading cover for %s...", entry->title.c_str());
+    }
+
+    // we could actually do a few in parallel here...
+    std::string filename(Common::HTTPDownloader::URLDecode(url));
+    downloader->CreateRequest(std::move(url), [entry_path = std::move(entry_path), filename = std::move(filename)](
+                                                s32 status_code, Common::HTTPDownloader::Request::Data data) {
+      if (status_code != Common::HTTPDownloader::HTTP_OK || data.empty())
+        return;
+
+      std::unique_lock lock(s_mutex);
+      const GameList::Entry* entry = GetEntryForPath(entry_path.c_str());
+      if (!entry || !GetCoverImagePathForEntry(entry).empty())
+        return;
+
+      std::string write_path(GetNewCoverImagePathForEntry(entry, filename.c_str()));
+      if (write_path.empty())
+        return;
+
+      FileSystem::WriteBinaryFile(write_path.c_str(), data.data(), data.size());
+      Host::CoversChanged();
+    });
+    downloader->WaitForAllRequests();
+    progress->IncrementProgressValue();
+  }
+
+  return true;
+}

src/frontend-common/game_list.h

@@ -75,6 +75,8 @@ void Refresh(bool invalidate_cache, bool only_cache = false, ProgressCallback* p
 std::string GetCoverImagePathForEntry(const Entry* entry);
 std::string GetCoverImagePath(const std::string& path, const std::string& serial, const std::string& title);
 std::string GetNewCoverImagePathForEntry(const Entry* entry, const char* new_filename);
+
+bool DownloadCovers(const std::vector<std::string>& url_templates, ProgressCallback* progress = nullptr);
 }; // namespace GameList
 
 namespace Host {
@@ -83,4 +85,8 @@ void RefreshGameListAsync(bool invalidate_cache);
 
 /// Cancels game list refresh, if there is one in progress.
 void CancelGameListRefresh();
+
+void DownloadCoversAsync(std::vector<std::string> url_templates);
+void CancelCoversDownload();
+void CoversChanged();
 } // namespace Host