HTTPDownloader: Move to util project

This commit is contained in:
Stenzek 2023-11-06 19:59:02 +10:00
parent 3c6b6c5770
commit 0fe6e9170b
No known key found for this signature in database
16 changed files with 107 additions and 106 deletions

View file

@ -23,8 +23,6 @@ add_library(common
hash_combine.h
heap_array.h
heterogeneous_containers.h
http_downloader.cpp
http_downloader.h
layered_settings_interface.cpp
layered_settings_interface.h
log.cpp
@ -67,13 +65,11 @@ target_link_libraries(common PRIVATE stb zlib minizip Zstd::Zstd "${CMAKE_DL_LIB
if(WIN32)
target_sources(common PRIVATE
http_downloader_winhttp.cpp
http_downloader_winhttp.h
thirdparty/StackWalker.cpp
thirdparty/StackWalker.h
windows_headers.h
)
target_link_libraries(common PRIVATE winhttp.lib OneCore.lib)
target_link_libraries(common PRIVATE OneCore.lib)
endif()
if(MSVC)
@ -98,20 +94,10 @@ if(APPLE)
target_link_libraries(common PRIVATE ${COCOA_LIBRARY})
endif()
if(NOT WIN32 AND NOT ANDROID)
target_sources(common PRIVATE
http_downloader_curl.cpp
http_downloader_curl.h
)
target_link_libraries(common PRIVATE
CURL::libcurl
)
if(LIBBACKTRACE_FOUND)
if(NOT WIN32 AND NOT ANDROID AND LIBBACKTRACE_FOUND)
target_compile_definitions(common PRIVATE "-DENABLE_LIBBACKTRACE=1")
target_link_libraries(common PRIVATE libbacktrace::libbacktrace)
endif()
endif()
if(ANDROID)
target_link_libraries(common PRIVATE log)

View file

@ -10,7 +10,7 @@
<ItemDefinitionGroup>
<Link>
<AdditionalDependencies>%(AdditionalDependencies);winhttp.lib;OneCore.lib</AdditionalDependencies>
<AdditionalDependencies>%(AdditionalDependencies);OneCore.lib</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
</Project>

View file

@ -17,8 +17,6 @@
<ClInclude Include="file_system.h" />
<ClInclude Include="hash_combine.h" />
<ClInclude Include="heap_array.h" />
<ClInclude Include="http_downloader.h" />
<ClInclude Include="http_downloader_winhttp.h" />
<ClInclude Include="image.h" />
<ClInclude Include="intrin.h" />
<ClInclude Include="layered_settings_interface.h" />
@ -51,8 +49,6 @@
<ClCompile Include="error.cpp" />
<ClCompile Include="fastjmp.cpp" />
<ClCompile Include="file_system.cpp" />
<ClCompile Include="http_downloader.cpp" />
<ClCompile Include="http_downloader_winhttp.cpp" />
<ClCompile Include="image.cpp" />
<ClCompile Include="layered_settings_interface.cpp" />
<ClCompile Include="log.cpp" />

View file

@ -28,8 +28,6 @@
<ClInclude Include="lru_cache.h" />
<ClInclude Include="easing.h" />
<ClInclude Include="error.h" />
<ClInclude Include="http_downloader_winhttp.h" />
<ClInclude Include="http_downloader.h" />
<ClInclude Include="path.h" />
<ClInclude Include="windows_headers.h" />
<ClInclude Include="settings_interface.h" />
@ -62,8 +60,6 @@
</ClCompile>
<ClCompile Include="crash_handler.cpp" />
<ClCompile Include="error.cpp" />
<ClCompile Include="http_downloader_winhttp.cpp" />
<ClCompile Include="http_downloader.cpp" />
<ClCompile Include="layered_settings_interface.cpp" />
<ClCompile Include="memory_settings_interface.cpp" />
<ClCompile Include="threading.cpp" />

View file

@ -20,7 +20,6 @@
#include "common/assert.h"
#include "common/error.h"
#include "common/file_system.h"
#include "common/http_downloader.h"
#include "common/log.h"
#include "common/md5_digest.h"
#include "common/path.h"
@ -29,6 +28,7 @@
#include "common/string_util.h"
#include "util/cd_image.h"
#include "util/http_downloader.h"
#include "util/imgui_fullscreen.h"
#include "util/imgui_manager.h"
#include "util/platform_misc.h"
@ -130,8 +130,8 @@ static void BeginLoadGame();
static void UpdateGameSummary();
static void DownloadImage(std::string url, std::string cache_filename);
static bool CreateClient(rc_client_t** client, std::unique_ptr<Common::HTTPDownloader>* http);
static void DestroyClient(rc_client_t** client, std::unique_ptr<Common::HTTPDownloader>* http);
static bool CreateClient(rc_client_t** client, std::unique_ptr<HTTPDownloader>* http);
static void DestroyClient(rc_client_t** client, std::unique_ptr<HTTPDownloader>* http);
static void ClientMessageCallback(const char* message, const rc_client_t* client);
static uint32_t ClientReadMemory(uint32_t address, uint8_t* buffer, uint32_t num_bytes, rc_client_t* client);
static void ClientServerCall(const rc_api_request_t* request, rc_client_server_callback_t callback, void* callback_data,
@ -187,7 +187,7 @@ static bool s_using_raintegration = false;
static std::recursive_mutex s_achievements_mutex;
static rc_client_t* s_client;
static std::string s_image_directory;
static std::unique_ptr<Common::HTTPDownloader> s_http_downloader;
static std::unique_ptr<HTTPDownloader> s_http_downloader;
static std::string s_game_path;
static std::string s_game_hash;
@ -305,9 +305,8 @@ std::string Achievements::GetGameHash(CDImage* image)
void Achievements::DownloadImage(std::string url, std::string cache_filename)
{
auto callback = [cache_filename](s32 status_code, std::string content_type,
Common::HTTPDownloader::Request::Data data) {
if (status_code != Common::HTTPDownloader::HTTP_STATUS_OK)
auto callback = [cache_filename](s32 status_code, std::string content_type, HTTPDownloader::Request::Data data) {
if (status_code != HTTPDownloader::HTTP_STATUS_OK)
return;
if (!FileSystem::WriteBinaryFile(cache_filename.c_str(), data.data(), data.size()))
@ -430,9 +429,9 @@ bool Achievements::Initialize()
return true;
}
bool Achievements::CreateClient(rc_client_t** client, std::unique_ptr<Common::HTTPDownloader>* http)
bool Achievements::CreateClient(rc_client_t** client, std::unique_ptr<HTTPDownloader>* http)
{
*http = Common::HTTPDownloader::Create(GetUserAgent().c_str());
*http = HTTPDownloader::Create(GetUserAgent().c_str());
if (!*http)
{
Host::ReportErrorAsync("Achievements Error", "Failed to create HTTPDownloader, cannot use achievements");
@ -459,7 +458,7 @@ bool Achievements::CreateClient(rc_client_t** client, std::unique_ptr<Common::HT
return true;
}
void Achievements::DestroyClient(rc_client_t** client, std::unique_ptr<Common::HTTPDownloader>* http)
void Achievements::DestroyClient(rc_client_t** client, std::unique_ptr<HTTPDownloader>* http)
{
(*http)->WaitForAllRequests();
@ -613,10 +612,10 @@ uint32_t Achievements::ClientReadMemory(uint32_t address, uint8_t* buffer, uint3
void Achievements::ClientServerCall(const rc_api_request_t* request, rc_client_server_callback_t callback,
void* callback_data, rc_client_t* client)
{
Common::HTTPDownloader::Request::Callback hd_callback =
[callback, callback_data](s32 status_code, std::string content_type, Common::HTTPDownloader::Request::Data data) {
HTTPDownloader::Request::Callback hd_callback = [callback, callback_data](s32 status_code, std::string content_type,
HTTPDownloader::Request::Data data) {
rc_api_server_response_t rr;
rr.http_status_code = (status_code <= 0) ? (status_code == Common::HTTPDownloader::HTTP_STATUS_CANCELLED ?
rr.http_status_code = (status_code <= 0) ? (status_code == HTTPDownloader::HTTP_STATUS_CANCELLED ?
RC_API_SERVER_RESPONSE_CLIENT_ERROR :
RC_API_SERVER_RESPONSE_RETRYABLE_CLIENT_ERROR) :
status_code;
@ -626,7 +625,7 @@ void Achievements::ClientServerCall(const rc_api_request_t* request, rc_client_s
callback(&rr, callback_data);
};
Common::HTTPDownloader* http = static_cast<Common::HTTPDownloader*>(rc_client_get_userdata(client));
HTTPDownloader* http = static_cast<HTTPDownloader*>(rc_client_get_userdata(client));
// TODO: Content-type for post
if (request->post_data)
@ -1598,9 +1597,9 @@ bool Achievements::Login(const char* username, const char* password, Error* erro
// We need to use a temporary client if achievements aren't currently active.
rc_client_t* client = s_client;
Common::HTTPDownloader* http = s_http_downloader.get();
HTTPDownloader* http = s_http_downloader.get();
const bool is_temporary_client = (client == nullptr);
std::unique_ptr<Common::HTTPDownloader> temporary_downloader;
std::unique_ptr<HTTPDownloader> temporary_downloader;
ScopedGuard temporary_client_guard = [&client, is_temporary_client, &temporary_downloader]() {
if (is_temporary_client)
DestroyClient(&client, &temporary_downloader);

View file

@ -9,12 +9,12 @@
#include "system.h"
#include "util/cd_image.h"
#include "util/http_downloader.h"
#include "common/assert.h"
#include "common/byte_stream.h"
#include "common/file_system.h"
#include "common/heterogeneous_containers.h"
#include "common/http_downloader.h"
#include "common/log.h"
#include "common/path.h"
#include "common/progress_callback.h"
@ -1116,15 +1116,14 @@ bool GameList::DownloadCovers(const std::vector<std::string>& url_templates, boo
{
std::string url(url_template);
if (has_title)
StringUtil::ReplaceAll(&url, "${title}", Common::HTTPDownloader::URLEncode(entry.title));
StringUtil::ReplaceAll(&url, "${title}", HTTPDownloader::URLEncode(entry.title));
if (has_file_title)
{
std::string display_name(FileSystem::GetDisplayNameFromPath(entry.path));
StringUtil::ReplaceAll(&url, "${filetitle}",
Common::HTTPDownloader::URLEncode(Path::GetFileTitle(display_name)));
StringUtil::ReplaceAll(&url, "${filetitle}", HTTPDownloader::URLEncode(Path::GetFileTitle(display_name)));
}
if (has_serial)
StringUtil::ReplaceAll(&url, "${serial}", Common::HTTPDownloader::URLEncode(entry.serial));
StringUtil::ReplaceAll(&url, "${serial}", HTTPDownloader::URLEncode(entry.serial));
download_urls.emplace_back(entry.path, std::move(url));
}
@ -1136,7 +1135,7 @@ bool GameList::DownloadCovers(const std::vector<std::string>& url_templates, boo
return false;
}
std::unique_ptr<Common::HTTPDownloader> downloader(Common::HTTPDownloader::Create());
std::unique_ptr<HTTPDownloader> downloader(HTTPDownloader::Create());
if (!downloader)
{
progress->DisplayError("Failed to create HTTP downloader.");
@ -1165,11 +1164,11 @@ bool GameList::DownloadCovers(const std::vector<std::string>& url_templates, boo
}
// we could actually do a few in parallel here...
std::string filename(Common::HTTPDownloader::URLDecode(url));
std::string filename(HTTPDownloader::URLDecode(url));
downloader->CreateRequest(
std::move(url), [use_serial, &save_callback, entry_path = std::move(entry_path), filename = std::move(filename)](
s32 status_code, std::string content_type, Common::HTTPDownloader::Request::Data data) {
if (status_code != Common::HTTPDownloader::HTTP_STATUS_OK || data.empty())
s32 status_code, std::string content_type, HTTPDownloader::Request::Data data) {
if (status_code != HTTPDownloader::HTTP_STATUS_OK || data.empty())
return;
std::unique_lock lock(s_mutex);
@ -1180,7 +1179,7 @@ bool GameList::DownloadCovers(const std::vector<std::string>& url_templates, boo
// prefer the content type from the response for the extension
// otherwise, if it's missing, and the request didn't have an extension.. fall back to jpegs.
std::string template_filename;
std::string content_type_extension(Common::HTTPDownloader::GetExtensionForContentType(content_type));
std::string content_type_extension(HTTPDownloader::GetExtensionForContentType(content_type));
// don't treat the domain name as an extension..
const std::string::size_type last_slash = filename.find('/');

View file

@ -29,6 +29,8 @@ add_library(util
gpu_texture.h
host.cpp
host.h
http_downloader.cpp
http_downloader.h
imgui_fullscreen.cpp
imgui_fullscreen.h
imgui_manager.cpp
@ -234,6 +236,8 @@ if(WIN32)
d3d12_texture.h
dinput_source.cpp
dinput_source.h
http_downloader_winhttp.cpp
http_downloader_winhttp.h
platform_misc_win32.cpp
win32_raw_input_source.cpp
win32_raw_input_source.h
@ -243,7 +247,7 @@ if(WIN32)
xinput_source.h
)
target_link_libraries(util PRIVATE d3d12ma)
target_link_libraries(util PRIVATE d3d11.lib d3d12.lib d3dcompiler.lib dxgi.lib winmm.lib Dwmapi.lib)
target_link_libraries(util PRIVATE d3d11.lib d3d12.lib d3dcompiler.lib dxgi.lib winmm.lib Dwmapi.lib winhttp.lib)
if("${CMAKE_BUILD_TYPE}" STREQUAL "Debug")
target_link_libraries(util PRIVATE WinPixEventRuntime::WinPixEventRuntime)
@ -279,6 +283,16 @@ elseif(NOT ANDROID)
target_link_libraries(util PRIVATE ${DBUS_LINK_LIBRARIES})
endif()
if(NOT WIN32 AND NOT ANDROID)
target_sources(common PRIVATE
http_downloader_curl.cpp
http_downloader_curl.h
)
target_link_libraries(common PRIVATE
CURL::libcurl
)
endif()
function(add_util_resources target)
if(APPLE)
get_property(UTIL_METAL_SOURCES GLOBAL PROPERTY UTIL_METAL_SOURCES)

View file

@ -1,18 +1,18 @@
// SPDX-FileCopyrightText: 2019-2022 Connor McLaughlin <stenzek@gmail.com>
// SPDX-FileCopyrightText: 2019-2023 Connor McLaughlin <stenzek@gmail.com>
// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
#include "http_downloader.h"
#include "assert.h"
#include "log.h"
#include "string_util.h"
#include "timer.h"
#include "common/assert.h"
#include "common/log.h"
#include "common/string_util.h"
#include "common/timer.h"
Log_SetChannel(HTTPDownloader);
static constexpr float DEFAULT_TIMEOUT_IN_SECONDS = 30;
static constexpr u32 DEFAULT_MAX_ACTIVE_REQUESTS = 4;
namespace Common {
const char HTTPDownloader::DEFAULT_USER_AGENT[] =
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:85.0) Gecko/20100101 Firefox/85.0";
@ -355,5 +355,3 @@ std::string HTTPDownloader::GetExtensionForContentType(const std::string& conten
}
return ret;
}
} // namespace Common

View file

@ -1,8 +1,10 @@
// SPDX-FileCopyrightText: 2019-2022 Connor McLaughlin <stenzek@gmail.com>
// SPDX-FileCopyrightText: 2019-2023 Connor McLaughlin <stenzek@gmail.com>
// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
#pragma once
#include "common/types.h"
#include <atomic>
#include <functional>
#include <memory>
@ -11,8 +13,6 @@
#include <string_view>
#include <vector>
namespace Common {
class HTTPDownloader
{
public:
@ -94,5 +94,3 @@ protected:
std::mutex m_pending_http_request_lock;
std::vector<Request*> m_pending_http_requests;
};
} // namespace FrontendCommon

View file

@ -1,17 +1,23 @@
// SPDX-FileCopyrightText: 2019-2023 Connor McLaughlin <stenzek@gmail.com>
// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
#include "http_downloader_curl.h"
#include "common/assert.h"
#include "common/log.h"
#include "common/string_util.h"
#include "common/timer.h"
#include <algorithm>
#include <functional>
#include <pthread.h>
#include <signal.h>
Log_SetChannel(HTTPDownloader);
namespace Common {
HTTPDownloaderCurl::HTTPDownloaderCurl() : HTTPDownloader() {}
HTTPDownloaderCurl::HTTPDownloaderCurl() : HTTPDownloader()
{
}
HTTPDownloaderCurl::~HTTPDownloaderCurl() = default;
@ -163,5 +169,3 @@ void HTTPDownloaderCurl::CloseRequest(HTTPDownloader::Request* request)
else
req->closed.store(true);
}
} // namespace Common

View file

@ -1,12 +1,15 @@
// SPDX-FileCopyrightText: 2019-2023 Connor McLaughlin <stenzek@gmail.com>
// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
#pragma once
#include "http_downloader.h"
#include "common/thirdparty/thread_pool.h"
#include <atomic>
#include <curl/curl.h>
#include <memory>
#include <mutex>
#include <curl/curl.h>
namespace Common {
class HTTPDownloaderCurl final : public HTTPDownloader
{
@ -36,5 +39,3 @@ private:
std::unique_ptr<cb::ThreadPool> m_thread_pool;
std::mutex m_cancel_mutex;
};
} // namespace FrontendCommon

View file

@ -1,18 +1,20 @@
// SPDX-FileCopyrightText: 2019-2022 Connor McLaughlin <stenzek@gmail.com>
// SPDX-FileCopyrightText: 2019-2023 Connor McLaughlin <stenzek@gmail.com>
// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
#include "http_downloader_winhttp.h"
#include "assert.h"
#include "log.h"
#include "string_util.h"
#include "timer.h"
#include <VersionHelpers.h>
#include "common/assert.h"
#include "common/log.h"
#include "common/string_util.h"
#include "common/timer.h"
#include <algorithm>
Log_SetChannel(HTTPDownloader);
namespace Common {
HTTPDownloaderWinHttp::HTTPDownloaderWinHttp() : HTTPDownloader() {}
HTTPDownloaderWinHttp::HTTPDownloaderWinHttp() : HTTPDownloader()
{
}
HTTPDownloaderWinHttp::~HTTPDownloaderWinHttp()
{
@ -34,9 +36,7 @@ std::unique_ptr<HTTPDownloader> HTTPDownloader::Create(const char* user_agent)
bool HTTPDownloaderWinHttp::Initialize(const char* user_agent)
{
// WINHTTP_ACCESS_TYPE_AUTOMATIC_PROXY is not supported before Win8.1.
const DWORD dwAccessType =
IsWindows8Point1OrGreater() ? WINHTTP_ACCESS_TYPE_AUTOMATIC_PROXY : WINHTTP_ACCESS_TYPE_DEFAULT_PROXY;
static constexpr DWORD dwAccessType = WINHTTP_ACCESS_TYPE_AUTOMATIC_PROXY;
m_hSession = WinHttpOpen(StringUtil::UTF8StringToWideString(user_agent).c_str(), dwAccessType, nullptr, nullptr,
WINHTTP_FLAG_ASYNC);
@ -311,5 +311,3 @@ void HTTPDownloaderWinHttp::CloseRequest(HTTPDownloader::Request* request)
delete req;
}
} // namespace Common

View file

@ -1,4 +1,4 @@
// SPDX-FileCopyrightText: 2019-2022 Connor McLaughlin <stenzek@gmail.com>
// SPDX-FileCopyrightText: 2019-2023 Connor McLaughlin <stenzek@gmail.com>
// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
#pragma once
@ -8,8 +8,6 @@
#include <winhttp.h>
namespace Common {
class HTTPDownloaderWinHttp final : public HTTPDownloader
{
public:
@ -38,5 +36,3 @@ private:
HINTERNET m_hSession = NULL;
};
} // namespace FrontendCommon

View file

@ -15,7 +15,7 @@
<ItemDefinitionGroup>
<Link>
<AdditionalDependencies>%(AdditionalDependencies);d3d11.lib;d3d12.lib;d3dcompiler.lib;dxgi.lib;Dwmapi.lib</AdditionalDependencies>
<AdditionalDependencies>%(AdditionalDependencies);d3d11.lib;d3d12.lib;d3dcompiler.lib;dxgi.lib;Dwmapi.lib;winhttp.lib</AdditionalDependencies>
<AdditionalDependencies Condition="'$(Platform)'!='ARM64'">%(AdditionalDependencies);opengl32.lib</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>

View file

@ -29,6 +29,11 @@
<ClInclude Include="gpu_shader_cache.h" />
<ClInclude Include="gpu_texture.h" />
<ClInclude Include="host.h" />
<ClInclude Include="http_downloader.h" />
<ClInclude Include="http_downloader_curl.h">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClInclude>
<ClInclude Include="http_downloader_winhttp.h" />
<ClInclude Include="imgui_fullscreen.h" />
<ClInclude Include="imgui_manager.h" />
<ClInclude Include="ini_settings_interface.h" />
@ -142,6 +147,11 @@
<ClCompile Include="gpu_shader_cache.cpp" />
<ClCompile Include="gpu_texture.cpp" />
<ClCompile Include="host.cpp" />
<ClCompile Include="http_downloader.cpp" />
<ClCompile Include="http_downloader_curl.cpp">
<ExcludedFromBuild>true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="http_downloader_winhttp.cpp" />
<ClCompile Include="imgui_fullscreen.cpp" />
<ClCompile Include="imgui_manager.cpp" />
<ClCompile Include="ini_settings_interface.cpp" />

View file

@ -70,6 +70,9 @@
<ClInclude Include="host.h" />
<ClInclude Include="postprocessing_shader_fx.h" />
<ClInclude Include="pch.h" />
<ClInclude Include="http_downloader_curl.h" />
<ClInclude Include="http_downloader_winhttp.h" />
<ClInclude Include="http_downloader.h" />
</ItemGroup>
<ItemGroup>
<ClCompile Include="jit_code_buffer.cpp" />
@ -146,6 +149,9 @@
<ClCompile Include="host.cpp" />
<ClCompile Include="postprocessing_shader_fx.cpp" />
<ClCompile Include="pch.cpp" />
<ClCompile Include="http_downloader_curl.cpp" />
<ClCompile Include="http_downloader_winhttp.cpp" />
<ClCompile Include="http_downloader.cpp" />
</ItemGroup>
<ItemGroup>
<Filter Include="gl">