diff --git a/CMakeLists.txt b/CMakeLists.txt index f14a4db25..5e42f76aa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -95,6 +95,9 @@ if(NOT ANDROID) if(NOT WIN32 AND USE_SDL2) find_package(SDL2 REQUIRED) endif() + if(NOT WIN32) + find_package(CURL REQUIRED) + endif() if(BUILD_QT_FRONTEND) find_package(Qt6 COMPONENTS Core Gui Widgets Network LinguistTools REQUIRED) endif() @@ -129,9 +132,6 @@ if(USE_EVDEV) endif() if(ENABLE_CHEEVOS) message(STATUS "RetroAchievements support enabled") - if(NOT WIN32 AND NOT ANDROID) - find_package(CURL REQUIRED) - endif() endif() diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index ea086d198..e5ce16a66 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -20,6 +20,8 @@ add_library(common hash_combine.h heap_array.h heterogeneous_containers.h + http_downloader.cpp + http_downloader.h layered_settings_interface.cpp layered_settings_interface.h log.cpp @@ -86,6 +88,8 @@ if(WIN32) d3d11/stream_buffer.h d3d11/texture.cpp d3d11/texture.h + http_downloader_winhttp.cpp + http_downloader_winhttp.h thirdparty/StackWalker.cpp thirdparty/StackWalker.h win32_progress_callback.cpp @@ -95,6 +99,16 @@ if(WIN32) target_link_libraries(common PRIVATE d3dcompiler.lib) endif() +if(NOT WIN32 AND NOT ANDROID) + target_sources(common PRIVATE + http_downloader_curl.cpp + http_downloader_curl.h + ) + target_link_libraries(common PRIVATE + CURL::libcurl + ) +endif() + if(ANDROID) target_link_libraries(common PRIVATE log) endif() @@ -242,29 +256,6 @@ if(ENABLE_VULKAN) endif() endif() - - -if(ENABLE_CHEEVOS) - target_sources(common PRIVATE - http_downloader.cpp - http_downloader.h - ) - if(WIN32) - target_sources(common PRIVATE - http_downloader_winhttp.cpp - http_downloader_winhttp.h - ) - elseif(NOT ANDROID) - target_sources(common PRIVATE - http_downloader_curl.cpp - http_downloader_curl.h - ) - target_link_libraries(common PRIVATE - CURL::libcurl - ) - endif() -endif() - if(${CMAKE_SYSTEM_NAME} STREQUAL "Linux") # We need -lrt for shm_unlink target_link_libraries(common PRIVATE rt) diff --git a/src/common/http_downloader.cpp b/src/common/http_downloader.cpp index ec558595c..b844f0440 100644 --- a/src/common/http_downloader.cpp +++ b/src/common/http_downloader.cpp @@ -183,4 +183,74 @@ u32 HTTPDownloader::LockedGetActiveRequestCount() return count; } -} // namespace FrontendCommon \ No newline at end of file +std::string HTTPDownloader::URLEncode(const std::string_view& str) +{ + std::string ret; + ret.reserve(str.length() + ((str.length() + 3) / 4) * 3); + + for (size_t i = 0, l = str.size(); i < l; i++) + { + const char c = str[i]; + if ((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '-' || c == '_' || + c == '.' || c == '!' || c == '~' || c == '*' || c == '\'' || c == '(' || c == ')') + { + ret.push_back(c); + } + else + { + ret.push_back('%'); + + const unsigned char n1 = static_cast(c) >> 4; + const unsigned char n2 = static_cast(c) & 0x0F; + ret.push_back((n1 >= 10) ? ('a' + (n1 - 10)) : ('0' + n1)); + ret.push_back((n2 >= 10) ? ('a' + (n2 - 10)) : ('0' + n2)); + } + } + + return ret; +} + +std::string HTTPDownloader::URLDecode(const std::string_view& str) +{ + std::string ret; + ret.reserve(str.length()); + + for (size_t i = 0, l = str.size(); i < l; i++) + { + const char c = str[i]; + if (c == '+') + { + ret.push_back(c); + } + else if (c == '%') + { + if ((i + 2) >= str.length()) + break; + + const char clower = str[i + 1]; + const char cupper = str[i + 2]; + const unsigned char lower = + (clower >= '0' && clower <= '9') ? + static_cast(clower - '0') : + ((clower >= 'a' && clower <= 'f') ? + static_cast(clower - 'a') : + ((clower >= 'A' && clower <= 'F') ? static_cast(clower - 'A') : 0)); + const unsigned char upper = + (cupper >= '0' && cupper <= '9') ? + static_cast(cupper - '0') : + ((cupper >= 'a' && cupper <= 'f') ? + static_cast(cupper - 'a') : + ((cupper >= 'A' && cupper <= 'F') ? static_cast(cupper - 'A') : 0)); + const char dch = static_cast(lower | (upper << 4)); + ret.push_back(dch); + } + else + { + ret.push_back(c); + } + } + + return std::string(str); +} + +} // namespace Common \ No newline at end of file diff --git a/src/common/http_downloader.h b/src/common/http_downloader.h index 5a83aafc7..661a32b57 100644 --- a/src/common/http_downloader.h +++ b/src/common/http_downloader.h @@ -5,6 +5,7 @@ #include #include #include +#include #include namespace Common { @@ -53,6 +54,8 @@ public: virtual ~HTTPDownloader(); static std::unique_ptr Create(const char* user_agent = DEFAULT_USER_AGENT); + static std::string URLEncode(const std::string_view& str); + static std::string URLDecode(const std::string_view& str); void SetTimeout(float timeout); void SetMaxActiveRequests(u32 max_active_requests); diff --git a/src/duckstation-nogui/nogui_host.cpp b/src/duckstation-nogui/nogui_host.cpp index 97ba3c9c5..c8acb63de 100644 --- a/src/duckstation-nogui/nogui_host.cpp +++ b/src/duckstation-nogui/nogui_host.cpp @@ -76,7 +76,9 @@ static void CPUThreadMainLoop(); static std::unique_ptr CreatePlatform(); static std::string GetWindowTitle(const std::string& game_title); static void UpdateWindowTitle(const std::string& game_title); -static void GameListRefreshThreadEntryPoint(bool invalidate_cache); +static void CancelAsyncOp(); +static void StartAsyncOp(std::function callback); +static void AsyncOpThreadEntryPoint(std::function callback); static bool AcquireHostDisplay(RenderAPI api); static void ReleaseHostDisplay(); } // namespace NoGUIHost @@ -99,9 +101,9 @@ static std::condition_variable s_cpu_thread_event_posted; static std::deque, bool>> s_cpu_thread_events; static u32 s_blocking_cpu_events_pending = 0; // TODO: Token system would work better here. -static std::mutex s_game_list_refresh_lock; -static std::thread s_game_list_refresh_thread; -static FullscreenUI::ProgressCallback* s_game_list_refresh_progress = nullptr; +static std::mutex s_async_op_mutex; +static std::thread s_async_op_thread; +static FullscreenUI::ProgressCallback* s_async_op_progress = nullptr; ////////////////////////////////////////////////////////////////////////// // Initialization/Shutdown @@ -961,39 +963,66 @@ void Host::RunOnCPUThread(std::function function, bool block /* = false s_cpu_thread_event_done.wait(lock, []() { return s_blocking_cpu_events_pending == 0; }); } -void NoGUIHost::GameListRefreshThreadEntryPoint(bool invalidate_cache) +void NoGUIHost::StartAsyncOp(std::function callback) { - Threading::SetNameOfCurrentThread("Game List Refresh"); + CancelAsyncOp(); + s_async_op_thread = std::thread(AsyncOpThreadEntryPoint, std::move(callback)); +} + +void NoGUIHost::CancelAsyncOp() +{ + std::unique_lock lock(s_async_op_mutex); + if (!s_async_op_thread.joinable()) + return; - FullscreenUI::ProgressCallback callback("game_list_refresh"); - std::unique_lock lock(s_game_list_refresh_lock); - s_game_list_refresh_progress = &callback; + if (s_async_op_progress) + s_async_op_progress->SetCancelled(); lock.unlock(); - GameList::Refresh(invalidate_cache, false, &callback); + s_async_op_thread.join(); +} + +void NoGUIHost::AsyncOpThreadEntryPoint(std::function callback) +{ + Threading::SetNameOfCurrentThread("Async Op"); + + FullscreenUI::ProgressCallback fs_callback("async_op"); + std::unique_lock lock(s_async_op_mutex); + s_async_op_progress = &fs_callback; + + lock.unlock(); + callback(&fs_callback); lock.lock(); - s_game_list_refresh_progress = nullptr; + s_async_op_progress = nullptr; } void Host::RefreshGameListAsync(bool invalidate_cache) { - CancelGameListRefresh(); - - s_game_list_refresh_thread = std::thread(NoGUIHost::GameListRefreshThreadEntryPoint, invalidate_cache); + NoGUIHost::StartAsyncOp( + [invalidate_cache](ProgressCallback* progress) { GameList::Refresh(invalidate_cache, false, progress); }); } void Host::CancelGameListRefresh() { - std::unique_lock lock(s_game_list_refresh_lock); - if (!s_game_list_refresh_thread.joinable()) - return; + NoGUIHost::CancelAsyncOp(); +} + +void Host::DownloadCoversAsync(std::vector url_templates) +{ + NoGUIHost::StartAsyncOp([url_templates = std::move(url_templates)](ProgressCallback* progress) { + GameList::DownloadCovers(url_templates, progress); + }); +} - if (s_game_list_refresh_progress) - s_game_list_refresh_progress->SetCancelled(); +void Host::CancelCoversDownload() +{ + NoGUIHost::CancelAsyncOp(); +} - lock.unlock(); - s_game_list_refresh_thread.join(); +void Host::CoversChanged() +{ + Host::RunOnCPUThread([]() { FullscreenUI::InvalidateCoverCache(); }); } bool Host::IsFullscreen() @@ -1340,6 +1369,7 @@ int main(int argc, char* argv[]) g_nogui_window->RunMessageLoop(); + NoGUIHost::CancelAsyncOp(); NoGUIHost::StopCPUThread(); // Ensure log is flushed. diff --git a/src/duckstation-qt/qthost.cpp b/src/duckstation-qt/qthost.cpp index e83e5bbe6..19a04cfa3 100644 --- a/src/duckstation-qt/qthost.cpp +++ b/src/duckstation-qt/qthost.cpp @@ -1078,6 +1078,21 @@ void Host::CancelGameListRefresh() QMetaObject::invokeMethod(g_main_window, "cancelGameListRefresh", Qt::BlockingQueuedConnection); } +void Host::DownloadCoversAsync(std::vector url_templates) +{ + // +} + +void Host::CancelCoversDownload() +{ + // +} + +void Host::CoversChanged() +{ + // +} + void EmuThread::loadState(const QString& filename) { if (!isOnThread()) diff --git a/src/frontend-common/fullscreen_ui.cpp b/src/frontend-common/fullscreen_ui.cpp index 15e17472c..b66d1ddd2 100644 --- a/src/frontend-common/fullscreen_ui.cpp +++ b/src/frontend-common/fullscreen_ui.cpp @@ -656,6 +656,14 @@ void FullscreenUI::Render() ImGuiFullscreen::ResetCloseMenuIfNeeded(); } +void FullscreenUI::InvalidateCoverCache() +{ + if (!IsInitialized()) + return; + + Host::RunOnCPUThread([]() { s_cover_image_map.clear(); }); +} + void FullscreenUI::ReturnToMainWindow() { if (s_pause_menu_was_open) diff --git a/src/frontend-common/fullscreen_ui.h b/src/frontend-common/fullscreen_ui.h index 9fcbe594f..f052ce439 100644 --- a/src/frontend-common/fullscreen_ui.h +++ b/src/frontend-common/fullscreen_ui.h @@ -21,6 +21,7 @@ bool OpenLeaderboardsWindow(); void Shutdown(); void Render(); +void InvalidateCoverCache(); // Returns true if the message has been dismissed. bool DrawErrorWindow(const char* message); diff --git a/src/frontend-common/game_list.cpp b/src/frontend-common/game_list.cpp index eaec4a1a1..266f7ba90 100644 --- a/src/frontend-common/game_list.cpp +++ b/src/frontend-common/game_list.cpp @@ -2,6 +2,7 @@ #include "common/assert.h" #include "common/byte_stream.h" #include "common/file_system.h" +#include "common/http_downloader.h" #include "common/log.h" #include "common/make_array.h" #include "common/path.h" @@ -18,9 +19,9 @@ #include #include #include -#include #include #include +#include #include Log_SetChannel(GameList); @@ -688,3 +689,113 @@ size_t GameList::Entry::GetReleaseDateString(char* buffer, size_t buffer_size) c return std::strftime(buffer, buffer_size, "%d %B %Y", &date_tm); } + +bool GameList::DownloadCovers(const std::vector& url_templates, ProgressCallback* progress /*= nullptr*/) +{ + if (!progress) + progress = ProgressCallback::NullProgressCallback; + + bool has_title = false; + bool has_file_title = false; + bool has_serial = false; + for (const std::string& url_template : url_templates) + { + if (!has_title && url_template.find("${title}") != std::string::npos) + has_title = true; + if (!has_file_title && url_template.find("${filetitle}") != std::string::npos) + has_file_title = true; + if (!has_serial && url_template.find("${serial}") != std::string::npos) + has_serial = true; + } + if (!has_title && !has_file_title && !has_serial) + { + progress->DisplayError("URL template must contain at least one of ${title}, ${filetitle}, or ${serial}."); + return false; + } + + std::vector> download_urls; + { + std::unique_lock lock(s_mutex); + for (const GameList::Entry& entry : m_entries) + { + const std::string existing_path(GetCoverImagePathForEntry(&entry)); + if (!existing_path.empty()) + continue; + + for (const std::string& url_template : url_templates) + { + std::string url(url_template); + if (has_title) + StringUtil::ReplaceAll(&url, "${title}", Common::HTTPDownloader::URLEncode(entry.title)); + if (has_file_title) + { + std::string display_name(FileSystem::GetDisplayNameFromPath(entry.path)); + StringUtil::ReplaceAll(&url, "${filetitle}", + Common::HTTPDownloader::URLEncode(Path::GetFileTitle(display_name))); + } + if (has_serial) + StringUtil::ReplaceAll(&url, "${serial}", Common::HTTPDownloader::URLEncode(entry.serial)); + + download_urls.emplace_back(entry.path, std::move(url)); + } + } + } + if (download_urls.empty()) + { + progress->DisplayError("No URLs to download enumerated."); + return false; + } + + std::unique_ptr downloader(Common::HTTPDownloader::Create()); + if (!downloader) + { + progress->DisplayError("Failed to create HTTP downloader."); + return false; + } + + progress->SetCancellable(true); + progress->SetProgressRange(static_cast(download_urls.size())); + + for (auto& [entry_path, url] : download_urls) + { + if (progress->IsCancelled()) + break; + + // make sure it didn't get done already + { + std::unique_lock lock(s_mutex); + const GameList::Entry* entry = GetEntryForPath(entry_path.c_str()); + if (!entry || !GetCoverImagePathForEntry(entry).empty()) + { + progress->IncrementProgressValue(); + continue; + } + + progress->SetFormattedStatusText("Downloading cover for %s...", entry->title.c_str()); + } + + // we could actually do a few in parallel here... + std::string filename(Common::HTTPDownloader::URLDecode(url)); + downloader->CreateRequest(std::move(url), [entry_path = std::move(entry_path), filename = std::move(filename)]( + s32 status_code, Common::HTTPDownloader::Request::Data data) { + if (status_code != Common::HTTPDownloader::HTTP_OK || data.empty()) + return; + + std::unique_lock lock(s_mutex); + const GameList::Entry* entry = GetEntryForPath(entry_path.c_str()); + if (!entry || !GetCoverImagePathForEntry(entry).empty()) + return; + + std::string write_path(GetNewCoverImagePathForEntry(entry, filename.c_str())); + if (write_path.empty()) + return; + + FileSystem::WriteBinaryFile(write_path.c_str(), data.data(), data.size()); + Host::CoversChanged(); + }); + downloader->WaitForAllRequests(); + progress->IncrementProgressValue(); + } + + return true; +} diff --git a/src/frontend-common/game_list.h b/src/frontend-common/game_list.h index a6ff70c06..5dca2594e 100644 --- a/src/frontend-common/game_list.h +++ b/src/frontend-common/game_list.h @@ -75,6 +75,8 @@ void Refresh(bool invalidate_cache, bool only_cache = false, ProgressCallback* p std::string GetCoverImagePathForEntry(const Entry* entry); std::string GetCoverImagePath(const std::string& path, const std::string& serial, const std::string& title); std::string GetNewCoverImagePathForEntry(const Entry* entry, const char* new_filename); + +bool DownloadCovers(const std::vector& url_templates, ProgressCallback* progress = nullptr); }; // namespace GameList namespace Host { @@ -83,4 +85,8 @@ void RefreshGameListAsync(bool invalidate_cache); /// Cancels game list refresh, if there is one in progress. void CancelGameListRefresh(); + +void DownloadCoversAsync(std::vector url_templates); +void CancelCoversDownload(); +void CoversChanged(); } // namespace Host