diff --git a/CMake/Packages/FindRapidJSON.cmake b/CMake/Packages/FindRapidJSON.cmake new file mode 100644 index 000000000..b5b69e727 --- /dev/null +++ b/CMake/Packages/FindRapidJSON.cmake @@ -0,0 +1,71 @@ +#.rst: +# FindRapidjson +# -------- +# +# Find the native rapidjson includes and library. +# +# IMPORTED Targets +# ^^^^^^^^^^^^^^^^ +# +# +# Result Variables +# ^^^^^^^^^^^^^^^^ +# +# This module defines the following variables: +# +# :: +# +# RAPIDJSON_INCLUDE_DIRS - where to find rapidjson/document.h, etc. +# RAPIDJSON_LIBRARIES - List of libraries when using rapidjson. +# RAPIDJSON_FOUND - True if rapidjson found. +# +# :: +# +# +# Hints +# ^^^^^ +# +# A user may set ``RAPIDJSON_ROOT`` to a rapidjson installation root to tell this +# module where to look. + +#============================================================================= +# Copyright 2018 OWenT. +# +# Distributed under the OSI-approved BSD License (the "License"); +# see accompanying file Copyright.txt for details. +# +# This software is distributed WITHOUT ANY WARRANTY; without even the +# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the License for more information. +#============================================================================= +# (To distribute this file outside of CMake, substitute the full +# License text for the above reference.) + +unset(_RAPIDJSON_SEARCH_ROOT_INC) +unset(_RAPIDJSON_SEARCH_ROOT_LIB) + +# Search RAPIDJSON_ROOT first if it is set. +if (Rapidjson_ROOT) + set(RAPIDJSON_ROOT ${Rapidjson_ROOT}) +endif() + +if(RAPIDJSON_ROOT) + set(_RAPIDJSON_SEARCH_ROOT_INC PATHS ${RAPIDJSON_ROOT} ${RAPIDJSON_ROOT}/include NO_DEFAULT_PATH) +endif() + +# Try each search configuration. +find_path(RAPIDJSON_INCLUDE_DIRS NAMES rapidjson/document.h ${_RAPIDJSON_SEARCH_ROOT_INC}) + +mark_as_advanced(RAPIDJSON_INCLUDE_DIRS) + +# handle the QUIETLY and REQUIRED arguments and set RAPIDJSON_FOUND to TRUE if +# all listed variables are TRUE +include("FindPackageHandleStandardArgs") +FIND_PACKAGE_HANDLE_STANDARD_ARGS(Rapidjson + REQUIRED_VARS RAPIDJSON_INCLUDE_DIRS + FOUND_VAR Rapidjson_FOUND +) + +if(Rapidjson_FOUND) + set(RAPIDJSON_FOUND ${Rapidjson_FOUND}) +endif() diff --git a/CMakeLists.txt b/CMakeLists.txt index 50a31be43..b757e8574 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -57,6 +57,7 @@ find_package(FreeImage REQUIRED) find_package(SDL2 REQUIRED) find_package(CURL REQUIRED) find_package(VLC REQUIRED) +find_package(RapidJSON REQUIRED) find_package(libCEC) #add ALSA for Linux @@ -120,6 +121,7 @@ set(COMMON_INCLUDE_DIRS ${SDL2_INCLUDE_DIR} ${CURL_INCLUDE_DIR} ${VLC_INCLUDE_DIR} + ${RAPIDJSON_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}/external ${CMAKE_CURRENT_SOURCE_DIR}/es-core/src ) diff --git a/es-app/CMakeLists.txt b/es-app/CMakeLists.txt index 0dd0ee72d..4ff4fdcce 100644 --- a/es-app/CMakeLists.txt +++ b/es-app/CMakeLists.txt @@ -39,7 +39,8 @@ set(ES_HEADERS # Scrapers ${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/Scraper.h - ${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/GamesDBScraper.h + ${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/GamesDBJSONScraper.h + ${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/GamesDBJSONScraperResources.h ${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/ScreenScraper.h # Views @@ -96,7 +97,8 @@ set(ES_SOURCES # Scrapers ${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/Scraper.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/GamesDBScraper.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/GamesDBJSONScraper.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/GamesDBJSONScraperResources.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/ScreenScraper.cpp # Views @@ -157,7 +159,7 @@ SET(CPACK_DEBIAN_PACKAGE_MAINTAINER "Alec Lofquist ") SET(CPACK_DEBIAN_PACKAGE_SECTION "misc") SET(CPACK_DEBIAN_PACKAGE_PRIORITY "extra") SET(CPACK_DEBIAN_PACKAGE_DEPENDS "libc6, libsdl2-2.0-0, libfreeimage3, libfreetype6, libcurl3, libasound2") -SET(CPACK_DEBIAN_PACKAGE_BUILDS_DEPENDS "debhelper (>= 8.0.0), cmake, g++ (>= 4.8), libsdl2-dev, libfreeimage-dev, libfreetype6-dev, libcurl4-openssl-dev, libasound2-dev, libgl1-mesa-dev") +SET(CPACK_DEBIAN_PACKAGE_BUILDS_DEPENDS "debhelper (>= 8.0.0), cmake, g++ (>= 4.8), libsdl2-dev, libfreeimage-dev, libfreetype6-dev, libcurl4-openssl-dev, libasound2-dev, libgl1-mesa-dev, rapidjson-dev") SET(CPACK_PACKAGE_VENDOR "emulationstation.org") SET(CPACK_PACKAGE_VERSION "2.0.0~rc1") diff --git a/es-app/src/guis/GuiMenu.cpp b/es-app/src/guis/GuiMenu.cpp index 448304975..2fd4c5f0e 100644 --- a/es-app/src/guis/GuiMenu.cpp +++ b/es-app/src/guis/GuiMenu.cpp @@ -58,7 +58,7 @@ void GuiMenu::openScraperSettings() // Select either the first entry of the one read from the settings, just in case the scraper from settings has vanished. for(auto it = scrapers.cbegin(); it != scrapers.cend(); it++) - scraper_list->add(*it, *it, *it == Settings::getInstance()->getString("Scraper") || it==scrapers.cbegin()); + scraper_list->add(*it, *it, *it == Settings::getInstance()->getString("Scraper")); s->addWithLabel("SCRAPE FROM", scraper_list); s->addSaveFunc([scraper_list] { Settings::getInstance()->setString("Scraper", scraper_list->getSelected()); }); diff --git a/es-app/src/scrapers/GamesDBJSONScraper.cpp b/es-app/src/scrapers/GamesDBJSONScraper.cpp new file mode 100644 index 000000000..a40ee2eca --- /dev/null +++ b/es-app/src/scrapers/GamesDBJSONScraper.cpp @@ -0,0 +1,392 @@ +#include +#include + +#include "scrapers/GamesDBJSONScraper.h" +#include "scrapers/GamesDBJSONScraperResources.h" + +#include "FileData.h" +#include "Log.h" +#include "PlatformId.h" +#include "Settings.h" +#include "SystemData.h" +#include "utils/TimeUtil.h" +#include + +/* When raspbian will get an up to date version of rapidjson we'll be + able to have it throw in case of error with the following: +#ifndef RAPIDJSON_ASSERT +#define RAPIDJSON_ASSERT(x) \ + if (!(x)) { \ + throw std::runtime_error("rapidjson internal assertion failure: " #x); \ + } +#endif // RAPIDJSON_ASSERT +*/ + +#include +#include + +using namespace PlatformIds; +using namespace rapidjson; + +namespace +{ +TheGamesDBJSONRequestResources resources; +} + +const std::map gamesdb_new_platformid_map{ + { THREEDO, "25" }, + { AMIGA, "4911" }, + { AMSTRAD_CPC, "4914" }, + { APPLE_II, "4942" }, + { ARCADE, "23" }, + { ATARI_800, "4943" }, + { ATARI_2600, "22" }, + { ATARI_5200, "26" }, + { ATARI_7800, "27" }, + { ATARI_JAGUAR, "28" }, + { ATARI_JAGUAR_CD, "29" }, + { ATARI_LYNX, "4924" }, + { ATARI_ST, "4937" }, + { ATARI_XE, "30" }, + { COLECOVISION, "31" }, + { COMMODORE_64, "40" }, + { INTELLIVISION, "32" }, + { MAC_OS, "37" }, + { XBOX, "14" }, + { XBOX_360, "15" }, + { MSX, "4929" }, + { NEOGEO, "24" }, + { NEOGEO_POCKET, "4922" }, + { NEOGEO_POCKET_COLOR, "4923" }, + { NINTENDO_3DS, "4912" }, + { NINTENDO_64, "3" }, + { NINTENDO_DS, "8" }, + { FAMICOM_DISK_SYSTEM, "4936" }, + { NINTENDO_ENTERTAINMENT_SYSTEM, "7" }, + { GAME_BOY, "4" }, + { GAME_BOY_ADVANCE, "5" }, + { GAME_BOY_COLOR, "41" }, + { NINTENDO_GAMECUBE, "2" }, + { NINTENDO_WII, "9" }, + { NINTENDO_WII_U, "38" }, + { NINTENDO_VIRTUAL_BOY, "4918" }, + { NINTENDO_GAME_AND_WATCH, "-1" }, + { PC, "1" }, + { SEGA_32X, "33" }, + { SEGA_CD, "21" }, + { SEGA_DREAMCAST, "16" }, + { SEGA_GAME_GEAR, "20" }, + { SEGA_GENESIS, "18" }, + { SEGA_MASTER_SYSTEM, "35" }, + { SEGA_MEGA_DRIVE, "36" }, + { SEGA_SATURN, "17" }, + { SEGA_SG1000, "4949" }, + { PLAYSTATION, "10" }, + { PLAYSTATION_2, "11" }, + { PLAYSTATION_3, "12" }, + { PLAYSTATION_4, "4919" }, + { PLAYSTATION_VITA, "39" }, + { PLAYSTATION_PORTABLE, "13" }, + { SUPER_NINTENDO, "6" }, + { TURBOGRAFX_16, "34" }, // HuCards only + { TURBOGRAFX_CD, "4955" }, // CD-ROMs only + { WONDERSWAN, "4925" }, + { WONDERSWAN_COLOR, "4926" }, + { ZX_SPECTRUM, "4913" }, + { VIDEOPAC_ODYSSEY2, "4927" }, + { VECTREX, "4939" }, + { TRS80_COLOR_COMPUTER, "4941" }, + { TANDY, "4941" }, +}; + +void thegamesdb_generate_json_scraper_requests(const ScraperSearchParams& params, + std::queue>& requests, std::vector& results) +{ + resources.prepare(); + std::string path = "https://api.thegamesdb.net"; + bool usingGameID = false; + const std::string apiKey = std::string("apikey=") + resources.getApiKey(); + std::string cleanName = params.nameOverride; + if (!cleanName.empty() && cleanName.substr(0, 3) == "id:") + { + std::string gameID = cleanName.substr(3); + path += "/Games/ByGameID?" + apiKey + + "&fields=players,publishers,genres,overview,last_updated,rating," + "platform,coop,youtube,os,processor,ram,hdd,video,sound,alternates&" + "include=boxart&id=" + + HttpReq::urlEncode(gameID); + usingGameID = true; + } else + { + if (cleanName.empty()) + cleanName = params.game->getCleanName(); + path += "/Games/ByGameName?" + apiKey + + "&fields=players,publishers,genres,overview,last_updated,rating," + "platform,coop,youtube,os,processor,ram,hdd,video,sound,alternates&" + "include=boxart&name=" + + HttpReq::urlEncode(cleanName); + } + + if (usingGameID) + { + // if we have the ID already, we don't need the GetGameList request + requests.push(std::unique_ptr(new TheGamesDBJSONRequest(results, path))); + } else + { + std::string platformQueryParam; + auto& platforms = params.system->getPlatformIds(); + if (!platforms.empty()) + { + bool first = true; + platformQueryParam += "&filter%5Bplatform%5D="; + for (auto platformIt = platforms.cbegin(); platformIt != platforms.cend(); platformIt++) + { + auto mapIt = gamesdb_new_platformid_map.find(*platformIt); + if (mapIt != gamesdb_new_platformid_map.cend()) + { + if (!first) + { + platformQueryParam += ","; + } + platformQueryParam += HttpReq::urlEncode(mapIt->second); + first = false; + } else + { + LOG(LogWarning) << "TheGamesDB scraper warning - no support for platform " + << getPlatformName(*platformIt); + } + } + path += platformQueryParam; + } + + requests.push(std::unique_ptr(new TheGamesDBJSONRequest(requests, results, path))); + } +} + +namespace +{ + +std::string getStringOrThrow(const Value& v, const std::string& key) +{ + if (!v.HasMember(key.c_str()) || !v[key.c_str()].IsString()) + { + throw std::runtime_error("rapidjson internal assertion failure: missing or non string key:" + key); + } + return v[key.c_str()].GetString(); +} + +int getIntOrThrow(const Value& v, const std::string& key) +{ + if (!v.HasMember(key.c_str()) || !v[key.c_str()].IsInt()) + { + throw std::runtime_error("rapidjson internal assertion failure: missing or non int key:" + key); + } + return v[key.c_str()].GetInt(); +} + +int getIntOrThrow(const Value& v) +{ + if (!v.IsInt()) + { + throw std::runtime_error("rapidjson internal assertion failure: not an int"); + } + return v.GetInt(); +} + +std::string getBoxartImage(const Value& v) +{ + if (!v.IsArray() || v.Size() == 0) + { + return ""; + } + for (int i = 0; i < v.Size(); ++i) + { + auto& im = v[i]; + std::string type = getStringOrThrow(im, "type"); + std::string side = getStringOrThrow(im, "side"); + if (type == "boxart" && side == "front") + { + return getStringOrThrow(im, "filename"); + } + } + return getStringOrThrow(v[0], "filename"); +} + +std::string getDeveloperString(const Value& v) +{ + if (!v.IsArray()) + { + return ""; + } + std::string out = ""; + bool first = true; + for (int i = 0; i < v.Size(); ++i) + { + auto mapIt = resources.gamesdb_new_developers_map.find(getIntOrThrow(v[i])); + if (mapIt == resources.gamesdb_new_developers_map.cend()) + { + continue; + } + if (!first) + { + out += ", "; + } + out += mapIt->second; + first = false; + } + return out; +} + +std::string getPublisherString(const Value& v) +{ + if (!v.IsArray()) + { + return ""; + } + std::string out = ""; + bool first = true; + for (int i = 0; i < v.Size(); ++i) + { + auto mapIt = resources.gamesdb_new_publishers_map.find(getIntOrThrow(v[i])); + if (mapIt == resources.gamesdb_new_publishers_map.cend()) + { + continue; + } + if (!first) + { + out += ", "; + } + out += mapIt->second; + first = false; + } + return out; +} + +std::string getGenreString(const Value& v) +{ + if (!v.IsArray()) + { + return ""; + } + std::string out = ""; + bool first = true; + for (int i = 0; i < v.Size(); ++i) + { + auto mapIt = resources.gamesdb_new_genres_map.find(getIntOrThrow(v[i])); + if (mapIt == resources.gamesdb_new_genres_map.cend()) + { + continue; + } + if (!first) + { + out += ", "; + } + out += mapIt->second; + first = false; + } + return out; +} + +void processGame(const Value& game, const Value& boxart, std::vector& results) +{ + std::string baseImageUrlThumb = getStringOrThrow(boxart["base_url"], "thumb"); + std::string baseImageUrlLarge = getStringOrThrow(boxart["base_url"], "large"); + + ScraperSearchResult result; + + result.mdl.set("name", getStringOrThrow(game, "game_title")); + if (game.HasMember("overview") && game["overview"].IsString()) + { + result.mdl.set("desc", game["overview"].GetString()); + } + if (game.HasMember("release_date") && game["release_date"].IsString()) + { + result.mdl.set( + "releasedate", Utils::Time::DateTime(Utils::Time::stringToTime(game["release_date"].GetString(), "%Y-%m-%d"))); + } + if (game.HasMember("developers") && game["developers"].IsArray()) + { + result.mdl.set("developer", getDeveloperString(game["developers"])); + } + if (game.HasMember("publishers") && game["publishers"].IsArray()) + { + result.mdl.set("publisher", getPublisherString(game["publishers"])); + } + if (game.HasMember("genres") && game["genres"].IsArray()) + { + + result.mdl.set("genre", getGenreString(game["genres"])); + } + if (game.HasMember("players") && game["players"].IsInt()) + { + result.mdl.set("players", std::to_string(game["players"].GetInt())); + } + + std::string id = std::to_string(getIntOrThrow(game, "id")); + if (boxart["data"].HasMember(id.c_str())) + { + std::string image = getBoxartImage(boxart["data"][id.c_str()]); + result.thumbnailUrl = baseImageUrlThumb + "/" + image; + result.imageUrl = baseImageUrlLarge + "/" + image; + } + + results.push_back(result); +} +} // namespace + +void TheGamesDBJSONRequest::process(const std::unique_ptr& req, std::vector& results) +{ + assert(req->status() == HttpReq::REQ_SUCCESS); + + Document doc; + doc.Parse(req->getContent().c_str()); + + if (doc.HasParseError()) + { + std::string err = + std::string("TheGamesDBJSONRequest - Error parsing JSON. \n\t") + GetParseError_En(doc.GetParseError()); + setError(err); + LOG(LogError) << err; + return; + } + + if (!doc.HasMember("data") || !doc["data"].HasMember("games") || !doc["data"]["games"].IsArray()) + { + std::string warn = "TheGamesDBJSONRequest - Response had no game data.\n"; + LOG(LogWarning) << warn; + return; + } + const Value& games = doc["data"]["games"]; + + if (!doc.HasMember("include") || !doc["include"].HasMember("boxart")) + { + std::string warn = "TheGamesDBJSONRequest - Response had no include boxart data.\n"; + LOG(LogWarning) << warn; + return; + } + + const Value& boxart = doc["include"]["boxart"]; + + if (!boxart.HasMember("base_url") || !boxart.HasMember("data") || !boxart.IsObject()) + { + std::string warn = "TheGamesDBJSONRequest - Response include had no usable boxart data.\n"; + LOG(LogWarning) << warn; + return; + } + + resources.ensureResources(); + + + for (int i = 0; i < games.Size(); ++i) + { + auto& v = games[i]; + try + { + processGame(v, boxart, results); + } + catch (std::runtime_error& e) + { + LOG(LogError) << "Error while processing game: " << e.what(); + } + } +} diff --git a/es-app/src/scrapers/GamesDBJSONScraper.h b/es-app/src/scrapers/GamesDBJSONScraper.h new file mode 100644 index 000000000..505dbcf2b --- /dev/null +++ b/es-app/src/scrapers/GamesDBJSONScraper.h @@ -0,0 +1,37 @@ +#pragma once +#ifndef ES_APP_SCRAPERS_GAMES_DB_JSON_SCRAPER_H +#define ES_APP_SCRAPERS_GAMES_DB_JSON_SCRAPER_H + +#include "scrapers/Scraper.h" + +namespace pugi +{ +class xml_document; +} + +void thegamesdb_generate_json_scraper_requests(const ScraperSearchParams& params, + std::queue>& requests, std::vector& results); + +class TheGamesDBJSONRequest : public ScraperHttpRequest +{ + public: + // ctor for a GetGameList request + TheGamesDBJSONRequest(std::queue>& requestsWrite, + std::vector& resultsWrite, const std::string& url) + : ScraperHttpRequest(resultsWrite, url), mRequestQueue(&requestsWrite) + { + } + // ctor for a GetGame request + TheGamesDBJSONRequest(std::vector& resultsWrite, const std::string& url) + : ScraperHttpRequest(resultsWrite, url), mRequestQueue(nullptr) + { + } + + protected: + void process(const std::unique_ptr& req, std::vector& results) override; + bool isGameRequest() { return !mRequestQueue; } + + std::queue>* mRequestQueue; +}; + +#endif // ES_APP_SCRAPERS_GAMES_DB_JSON_SCRAPER_H diff --git a/es-app/src/scrapers/GamesDBJSONScraperResources.cpp b/es-app/src/scrapers/GamesDBJSONScraperResources.cpp new file mode 100644 index 000000000..7d59bef46 --- /dev/null +++ b/es-app/src/scrapers/GamesDBJSONScraperResources.cpp @@ -0,0 +1,207 @@ +#include +#include +#include +#include + +#include "Log.h" + +#include "scrapers/GamesDBJSONScraperResources.h" +#include "utils/FileSystemUtil.h" + + +#include +#include + +using namespace rapidjson; + + +namespace +{ +constexpr char GamesDBAPIKey[] = "445fcbc3f32bb2474bc27016b99eb963d318ee3a608212c543b9a79de1041600"; + + +constexpr int MAX_WAIT_MS = 90000; +constexpr int POLL_TIME_MS = 500; +constexpr int MAX_WAIT_ITER = MAX_WAIT_MS / POLL_TIME_MS; + +constexpr char SCRAPER_RESOURCES_DIR[] = "scrapers"; +constexpr char DEVELOPERS_JSON_FILE[] = "gamesdb_developers.json"; +constexpr char PUBLISHERS_JSON_FILE[] = "gamesdb_publishers.json"; +constexpr char GENRES_JSON_FILE[] = "gamesdb_genres.json"; +constexpr char DEVELOPERS_ENDPOINT[] = "/Developers"; +constexpr char PUBLISHERS_ENDPOINT[] = "/Publishers"; +constexpr char GENRES_ENDPOINT[] = "/Genres"; + +std::string genFilePath(const std::string& file_name) +{ + return Utils::FileSystem::getGenericPath(getScrapersResouceDir() + "/" + file_name); +} + +void ensureScrapersResourcesDir() +{ + std::string path = getScrapersResouceDir(); + if (!Utils::FileSystem::exists(path)) + Utils::FileSystem::createDirectory(path); +} + +} // namespace + + +std::string getScrapersResouceDir() +{ + return Utils::FileSystem::getGenericPath( + Utils::FileSystem::getHomePath() + "/.emulationstation/" + SCRAPER_RESOURCES_DIR); +} + +std::string TheGamesDBJSONRequestResources::getApiKey() const { return GamesDBAPIKey; } + + +void TheGamesDBJSONRequestResources::prepare() +{ + if (checkLoaded()) + { + return; + } + + if (loadResource(gamesdb_new_developers_map, "developers", genFilePath(DEVELOPERS_JSON_FILE)) && + !gamesdb_developers_resource_request) + { + gamesdb_developers_resource_request = fetchResource(DEVELOPERS_ENDPOINT); + } + if (loadResource(gamesdb_new_publishers_map, "publishers", genFilePath(PUBLISHERS_JSON_FILE)) && + !gamesdb_publishers_resource_request) + { + gamesdb_publishers_resource_request = fetchResource(PUBLISHERS_ENDPOINT); + } + if (loadResource(gamesdb_new_genres_map, "genres", genFilePath(GENRES_JSON_FILE)) && !gamesdb_genres_resource_request) + { + gamesdb_genres_resource_request = fetchResource(GENRES_ENDPOINT); + } +} + +void TheGamesDBJSONRequestResources::ensureResources() +{ + + if (checkLoaded()) + { + return; + } + + + for (int i = 0; i < MAX_WAIT_ITER; ++i) + { + if (gamesdb_developers_resource_request && + saveResource(gamesdb_developers_resource_request.get(), gamesdb_new_developers_map, "developers", + genFilePath(DEVELOPERS_JSON_FILE))) + { + + gamesdb_developers_resource_request.reset(nullptr); + } + if (gamesdb_publishers_resource_request && + saveResource(gamesdb_publishers_resource_request.get(), gamesdb_new_publishers_map, "publishers", + genFilePath(PUBLISHERS_JSON_FILE))) + { + gamesdb_publishers_resource_request.reset(nullptr); + } + if (gamesdb_genres_resource_request && saveResource(gamesdb_genres_resource_request.get(), gamesdb_new_genres_map, + "genres", genFilePath(GENRES_JSON_FILE))) + { + gamesdb_genres_resource_request.reset(nullptr); + } + + if (!gamesdb_developers_resource_request && !gamesdb_publishers_resource_request && !gamesdb_genres_resource_request) + { + return; + } + std::this_thread::sleep_for(std::chrono::milliseconds(POLL_TIME_MS)); + } + LOG(LogError) << "Timed out while waiting for resources\n"; +} + +bool TheGamesDBJSONRequestResources::checkLoaded() +{ + return !gamesdb_new_genres_map.empty() && !gamesdb_new_developers_map.empty() && !gamesdb_new_publishers_map.empty(); +} + +bool TheGamesDBJSONRequestResources::saveResource(HttpReq* req, std::unordered_map& resource, + const std::string& resource_name, const std::string& file_name) +{ + + if (req == nullptr) + { + LOG(LogError) << "Http request pointer was null\n"; + return true; + } + if (req->status() == HttpReq::REQ_IN_PROGRESS) + { + return false; // Not ready: wait some more + } + if (req->status() != HttpReq::REQ_SUCCESS) + { + LOG(LogError) << "Resource request for " << file_name << " failed:\n\t" << req->getErrorMsg(); + return true; // Request failed, resetting request. + } + + ensureScrapersResourcesDir(); + + std::ofstream fout(file_name); + fout << req->getContent(); + fout.close(); + loadResource(resource, resource_name, file_name); + return true; +} + +std::unique_ptr TheGamesDBJSONRequestResources::fetchResource(const std::string& endpoint) +{ + std::string path = "https://api.thegamesdb.net"; + path += endpoint; + path += "?apikey=" + getApiKey(); + + return std::unique_ptr(new HttpReq(path)); +} + + +int TheGamesDBJSONRequestResources::loadResource( + std::unordered_map& resource, const std::string& resource_name, const std::string& file_name) +{ + + + std::ifstream fin(file_name); + if (!fin.good()) + { + return 1; + } + std::stringstream buffer; + buffer << fin.rdbuf(); + Document doc; + doc.Parse(buffer.str().c_str()); + + if (doc.HasParseError()) + { + std::string err = std::string("TheGamesDBJSONRequest - Error parsing JSON for resource file ") + file_name + + ":\n\t" + GetParseError_En(doc.GetParseError()); + LOG(LogError) << err; + return 1; + } + + if (!doc.HasMember("data") || !doc["data"].HasMember(resource_name.c_str()) || + !doc["data"][resource_name.c_str()].IsObject()) + { + std::string err = "TheGamesDBJSONRequest - Response had no resource data.\n"; + LOG(LogError) << err; + return 1; + } + auto& data = doc["data"][resource_name.c_str()]; + + for (Value::ConstMemberIterator itr = data.MemberBegin(); itr != data.MemberEnd(); ++itr) + { + auto& entry = itr->value; + if (!entry.IsObject() || !entry.HasMember("id") || !entry["id"].IsInt() || !entry.HasMember("name") || + !entry["name"].IsString()) + { + continue; + } + resource[entry["id"].GetInt()] = entry["name"].GetString(); + } + return resource.empty(); +} diff --git a/es-app/src/scrapers/GamesDBJSONScraperResources.h b/es-app/src/scrapers/GamesDBJSONScraperResources.h new file mode 100644 index 000000000..4cbbcc587 --- /dev/null +++ b/es-app/src/scrapers/GamesDBJSONScraperResources.h @@ -0,0 +1,42 @@ +#pragma once +#ifndef ES_APP_SCRAPERS_GAMES_DB_JSON_SCRAPER_RESOURCES_H +#define ES_APP_SCRAPERS_GAMES_DB_JSON_SCRAPER_RESOURCES_H + +#include +#include +#include +#include + +#include "HttpReq.h" + + +struct TheGamesDBJSONRequestResources +{ + TheGamesDBJSONRequestResources() = default; + + void prepare(); + void ensureResources(); + std::string getApiKey() const; + + std::unordered_map gamesdb_new_developers_map; + std::unordered_map gamesdb_new_publishers_map; + std::unordered_map gamesdb_new_genres_map; + + private: + bool checkLoaded(); + + bool saveResource(HttpReq* req, std::unordered_map& resource, const std::string& resource_name, + const std::string& file_name); + std::unique_ptr fetchResource(const std::string& endpoint); + + int loadResource( + std::unordered_map& resource, const std::string& resource_name, const std::string& file_name); + + std::unique_ptr gamesdb_developers_resource_request; + std::unique_ptr gamesdb_publishers_resource_request; + std::unique_ptr gamesdb_genres_resource_request; +}; + +std::string getScrapersResouceDir(); + +#endif // ES_APP_SCRAPERS_GAMES_DB_JSON_SCRAPER_H diff --git a/es-app/src/scrapers/GamesDBScraper.cpp b/es-app/src/scrapers/GamesDBScraper.cpp deleted file mode 100644 index 157b181ee..000000000 --- a/es-app/src/scrapers/GamesDBScraper.cpp +++ /dev/null @@ -1,211 +0,0 @@ -#include "scrapers/GamesDBScraper.h" - -#include "utils/TimeUtil.h" -#include "FileData.h" -#include "Log.h" -#include "PlatformId.h" -#include "Settings.h" -#include "SystemData.h" -#include - -using namespace PlatformIds; -const std::map gamesdb_platformid_map { - { THREEDO, "3DO" }, - { AMIGA, "Amiga" }, - { AMSTRAD_CPC, "Amstrad CPC" }, - // missing apple2 - { ARCADE, "Arcade" }, - // missing atari 800 - { ATARI_2600, "Atari 2600" }, - { ATARI_5200, "Atari 5200" }, - { ATARI_7800, "Atari 7800" }, - { ATARI_JAGUAR, "Atari Jaguar" }, - { ATARI_JAGUAR_CD, "Atari Jaguar CD" }, - { ATARI_LYNX, "Atari Lynx" }, - // missing atari ST/STE/Falcon - { ATARI_XE, "Atari XE" }, - { COLECOVISION, "Colecovision" }, - { COMMODORE_64, "Commodore 64" }, - { INTELLIVISION, "Intellivision" }, - { MAC_OS, "Mac OS" }, - { XBOX, "Microsoft Xbox" }, - { XBOX_360, "Microsoft Xbox 360" }, - { MSX, "MSX" }, - { NEOGEO, "Neo Geo" }, - { NEOGEO_POCKET, "Neo Geo Pocket" }, - { NEOGEO_POCKET_COLOR, "Neo Geo Pocket Color" }, - { NINTENDO_3DS, "Nintendo 3DS" }, - { NINTENDO_64, "Nintendo 64" }, - { NINTENDO_DS, "Nintendo DS" }, - { FAMICOM_DISK_SYSTEM, "Famicom Disk System" }, - { NINTENDO_ENTERTAINMENT_SYSTEM, "Nintendo Entertainment System (NES)" }, - { GAME_BOY, "Nintendo Game Boy" }, - { GAME_BOY_ADVANCE, "Nintendo Game Boy Advance" }, - { GAME_BOY_COLOR, "Nintendo Game Boy Color" }, - { NINTENDO_GAMECUBE, "Nintendo GameCube" }, - { NINTENDO_WII, "Nintendo Wii" }, - { NINTENDO_WII_U, "Nintendo Wii U" }, - { NINTENDO_VIRTUAL_BOY, "Nintendo Virtual Boy" }, - { NINTENDO_GAME_AND_WATCH, "Game & Watch" }, - { PC, "PC" }, - { SEGA_32X, "Sega 32X" }, - { SEGA_CD, "Sega CD" }, - { SEGA_DREAMCAST, "Sega Dreamcast" }, - { SEGA_GAME_GEAR, "Sega Game Gear" }, - { SEGA_GENESIS, "Sega Genesis" }, - { SEGA_MASTER_SYSTEM, "Sega Master System" }, - { SEGA_MEGA_DRIVE, "Sega Mega Drive" }, - { SEGA_SATURN, "Sega Saturn" }, - { SEGA_SG1000, "SEGA SG-1000" }, - { PLAYSTATION, "Sony Playstation" }, - { PLAYSTATION_2, "Sony Playstation 2" }, - { PLAYSTATION_3, "Sony Playstation 3" }, - { PLAYSTATION_4, "Sony Playstation 4" }, - { PLAYSTATION_VITA, "Sony Playstation Vita" }, - { PLAYSTATION_PORTABLE, "Sony Playstation Portable" }, - { SUPER_NINTENDO, "Super Nintendo (SNES)" }, - { TURBOGRAFX_16, "TurboGrafx 16" }, // HuCards only - { TURBOGRAFX_CD, "TurboGrafx CD" }, // CD-ROMs only - { WONDERSWAN, "WonderSwan" }, - { WONDERSWAN_COLOR, "WonderSwan Color" }, - { ZX_SPECTRUM, "Sinclair ZX Spectrum" }, - { VIDEOPAC_ODYSSEY2, "Magnavox Odyssey 2" }, - { VECTREX, "Vectrex" }, - { TRS80_COLOR_COMPUTER, "TRS-80 Color Computer" }, - { TANDY, "TRS-80 Color Computer" } -}; - -void thegamesdb_generate_scraper_requests(const ScraperSearchParams& params, std::queue< std::unique_ptr >& requests, - std::vector& results) -{ - std::string path; - bool usingGameID = false; - - std::string cleanName = params.nameOverride; - if (!cleanName.empty() && cleanName.substr(0,3) == "id:") - { - std::string gameID = cleanName.substr(3); - path = "legacy.thegamesdb.net/api/GetGame.php?id=" + HttpReq::urlEncode(gameID); - usingGameID = true; - }else{ - if (cleanName.empty()) - cleanName = params.game->getCleanName(); - path += "legacy.thegamesdb.net/api/GetGamesList.php?name=" + HttpReq::urlEncode(cleanName); - } - - if(usingGameID) - { - // if we have the ID already, we don't need the GetGameList request - requests.push(std::unique_ptr(new TheGamesDBRequest(results, path))); - }else if(params.system->getPlatformIds().empty()){ - // no platform specified, we're done - requests.push(std::unique_ptr(new TheGamesDBRequest(requests, results, path))); - }else{ - // go through the list, we need to split this into multiple requests - // because TheGamesDB API either sucks or I don't know how to use it properly... - std::string urlBase = path; - auto& platforms = params.system->getPlatformIds(); - for(auto platformIt = platforms.cbegin(); platformIt != platforms.cend(); platformIt++) - { - path = urlBase; - auto mapIt = gamesdb_platformid_map.find(*platformIt); - if(mapIt != gamesdb_platformid_map.cend()) - { - path += "&platform="; - path += HttpReq::urlEncode(mapIt->second); - }else{ - LOG(LogWarning) << "TheGamesDB scraper warning - no support for platform " << getPlatformName(*platformIt); - } - - requests.push(std::unique_ptr(new TheGamesDBRequest(requests, results, path))); - } - } -} - -void TheGamesDBRequest::process(const std::unique_ptr& req, std::vector& results) -{ - assert(req->status() == HttpReq::REQ_SUCCESS); - - pugi::xml_document doc; - pugi::xml_parse_result parseResult = doc.load(req->getContent().c_str()); - if(!parseResult) - { - std::stringstream ss; - ss << "TheGamesDBRequest - Error parsing XML. \n\t" << parseResult.description() << ""; - std::string err = ss.str(); - setError(err); - LOG(LogError) << err; - return; - } - - if (isGameRequest()) - processGame(doc, results); - else - processList(doc, results); -} - -void TheGamesDBRequest::processGame(const pugi::xml_document& xmldoc, std::vector& results) -{ - pugi::xml_node data = xmldoc.child("Data"); - - std::string baseImageUrl = data.child("baseImgUrl").text().get(); - - pugi::xml_node game = data.child("Game"); - if(game) - { - ScraperSearchResult result; - - result.mdl.set("name", game.child("GameTitle").text().get()); - result.mdl.set("desc", game.child("Overview").text().get()); - result.mdl.set("releasedate", Utils::Time::DateTime(Utils::Time::stringToTime(game.child("ReleaseDate").text().get(), "%m/%d/%Y"))); - result.mdl.set("developer", game.child("Developer").text().get()); - result.mdl.set("publisher", game.child("Publisher").text().get()); - result.mdl.set("genre", game.child("Genres").first_child().text().get()); - result.mdl.set("players", game.child("Players").text().get()); - - if(Settings::getInstance()->getBool("ScrapeRatings") && game.child("Rating")) - { - float ratingVal = (game.child("Rating").text().as_int() / 10.0f); - std::stringstream ss; - ss << ratingVal; - result.mdl.set("rating", ss.str()); - } - - pugi::xml_node images = game.child("Images"); - - if(images) - { - pugi::xml_node art = images.find_child_by_attribute("boxart", "side", "front"); - - if(art) - { - result.thumbnailUrl = baseImageUrl + art.attribute("thumb").as_string(); - result.imageUrl = baseImageUrl + art.text().get(); - } - } - - results.push_back(result); - } -} - -void TheGamesDBRequest::processList(const pugi::xml_document& xmldoc, std::vector& results) -{ - assert(mRequestQueue != nullptr); - - pugi::xml_node data = xmldoc.child("Data"); - pugi::xml_node game = data.child("Game"); - - // limit the number of results per platform, not in total. - // otherwise if the first platform returns >= 7 games - // but the second platform contains the relevant game, - // the relevant result would not be shown. - for(int i = 0; game && i < MAX_SCRAPER_RESULTS; i++) - { - std::string id = game.child("id").text().get(); - std::string path = "legacy.thegamesdb.net/api/GetGame.php?id=" + id; - - mRequestQueue->push(std::unique_ptr(new TheGamesDBRequest(results, path))); - - game = game.next_sibling("Game"); - } -} diff --git a/es-app/src/scrapers/GamesDBScraper.h b/es-app/src/scrapers/GamesDBScraper.h deleted file mode 100644 index 092366bd1..000000000 --- a/es-app/src/scrapers/GamesDBScraper.h +++ /dev/null @@ -1,29 +0,0 @@ -#pragma once -#ifndef ES_APP_SCRAPERS_GAMES_DB_SCRAPER_H -#define ES_APP_SCRAPERS_GAMES_DB_SCRAPER_H - -#include "scrapers/Scraper.h" - -namespace pugi { class xml_document; } - -void thegamesdb_generate_scraper_requests(const ScraperSearchParams& params, std::queue< std::unique_ptr >& requests, - std::vector& results); - -class TheGamesDBRequest : public ScraperHttpRequest -{ -public: - // ctor for a GetGameList request - TheGamesDBRequest(std::queue< std::unique_ptr >& requestsWrite, std::vector& resultsWrite, const std::string& url) : ScraperHttpRequest(resultsWrite, url), mRequestQueue(&requestsWrite) {} - // ctor for a GetGame request - TheGamesDBRequest(std::vector& resultsWrite, const std::string& url) : ScraperHttpRequest(resultsWrite, url), mRequestQueue(nullptr) {} - -protected: - void process(const std::unique_ptr& req, std::vector& results) override; - void processList(const pugi::xml_document& xmldoc, std::vector& results); - void processGame(const pugi::xml_document& xmldoc, std::vector& results); - bool isGameRequest() { return !mRequestQueue; } - - std::queue< std::unique_ptr >* mRequestQueue; -}; - -#endif // ES_APP_SCRAPERS_GAMES_DB_SCRAPER_H diff --git a/es-app/src/scrapers/Scraper.cpp b/es-app/src/scrapers/Scraper.cpp index eead8a812..76d3fc963 100644 --- a/es-app/src/scrapers/Scraper.cpp +++ b/es-app/src/scrapers/Scraper.cpp @@ -1,7 +1,7 @@ #include "scrapers/Scraper.h" #include "FileData.h" -#include "GamesDBScraper.h" +#include "GamesDBJSONScraper.h" #include "ScreenScraper.h" #include "Log.h" #include "Settings.h" @@ -10,7 +10,7 @@ #include const std::map scraper_request_funcs { -// { "TheGamesDB", &thegamesdb_generate_scraper_requests }, + { "TheGamesDB", &thegamesdb_generate_json_scraper_requests }, { "ScreenScraper", &screenscraper_generate_scraper_requests } };