mirror of
https://github.com/RetroDECK/ES-DE.git
synced 2024-11-22 06:05:38 +00:00
Adding new scraper GamesDBJSONScraper. The scraper uses the new GamesDb JSON API.
On the first run it downloads a few resource files needed to intepret the otput of game search api calls these resources go into ~/.emulatiostation/scrapers The resource files include the list of developers and the list of publishers. To update the lists simply delete the files and relaunch emulationstation. Searching a game by id (GamesDB id) is done as before by manually edit the search query and query for "id:<gameId>".
This commit is contained in:
acrummyidea
parent
12464024f0
commit
44395f5f45
71
CMake/Packages/FindRapidJSON.cmake
Normal file
71
CMake/Packages/FindRapidJSON.cmake
Normal file
|
|
@ -0,0 +1,71 @@
|
||||||
|
#.rst:
|
||||||
|
# FindRapidjson
|
||||||
|
# --------
|
||||||
|
#
|
||||||
|
# Find the native rapidjson includes and library.
|
||||||
|
#
|
||||||
|
# IMPORTED Targets
|
||||||
|
# ^^^^^^^^^^^^^^^^
|
||||||
|
#
|
||||||
|
#
|
||||||
|
# Result Variables
|
||||||
|
# ^^^^^^^^^^^^^^^^
|
||||||
|
#
|
||||||
|
# This module defines the following variables:
|
||||||
|
#
|
||||||
|
# ::
|
||||||
|
#
|
||||||
|
# RAPIDJSON_INCLUDE_DIRS - where to find rapidjson/document.h, etc.
|
||||||
|
# RAPIDJSON_LIBRARIES - List of libraries when using rapidjson.
|
||||||
|
# RAPIDJSON_FOUND - True if rapidjson found.
|
||||||
|
#
|
||||||
|
# ::
|
||||||
|
#
|
||||||
|
#
|
||||||
|
# Hints
|
||||||
|
# ^^^^^
|
||||||
|
#
|
||||||
|
# A user may set ``RAPIDJSON_ROOT`` to a rapidjson installation root to tell this
|
||||||
|
# module where to look.
|
||||||
|
|
||||||
|
#=============================================================================
|
||||||
|
# Copyright 2018 OWenT.
|
||||||
|
#
|
||||||
|
# Distributed under the OSI-approved BSD License (the "License");
|
||||||
|
# see accompanying file Copyright.txt for details.
|
||||||
|
#
|
||||||
|
# This software is distributed WITHOUT ANY WARRANTY; without even the
|
||||||
|
# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
|
||||||
|
# See the License for more information.
|
||||||
|
#=============================================================================
|
||||||
|
# (To distribute this file outside of CMake, substitute the full
|
||||||
|
# License text for the above reference.)
|
||||||
|
|
||||||
|
unset(_RAPIDJSON_SEARCH_ROOT_INC)
|
||||||
|
unset(_RAPIDJSON_SEARCH_ROOT_LIB)
|
||||||
|
|
||||||
|
# Search RAPIDJSON_ROOT first if it is set.
|
||||||
|
if (Rapidjson_ROOT)
|
||||||
|
set(RAPIDJSON_ROOT ${Rapidjson_ROOT})
|
||||||
|
endif()
|
||||||
|
|
||||||
|
if(RAPIDJSON_ROOT)
|
||||||
|
set(_RAPIDJSON_SEARCH_ROOT_INC PATHS ${RAPIDJSON_ROOT} ${RAPIDJSON_ROOT}/include NO_DEFAULT_PATH)
|
||||||
|
endif()
|
||||||
|
|
||||||
|
# Try each search configuration.
|
||||||
|
find_path(RAPIDJSON_INCLUDE_DIRS NAMES rapidjson/document.h ${_RAPIDJSON_SEARCH_ROOT_INC})
|
||||||
|
|
||||||
|
mark_as_advanced(RAPIDJSON_INCLUDE_DIRS)
|
||||||
|
|
||||||
|
# handle the QUIETLY and REQUIRED arguments and set RAPIDJSON_FOUND to TRUE if
|
||||||
|
# all listed variables are TRUE
|
||||||
|
include("FindPackageHandleStandardArgs")
|
||||||
|
FIND_PACKAGE_HANDLE_STANDARD_ARGS(Rapidjson
|
||||||
|
REQUIRED_VARS RAPIDJSON_INCLUDE_DIRS
|
||||||
|
FOUND_VAR Rapidjson_FOUND
|
||||||
|
)
|
||||||
|
|
||||||
|
if(Rapidjson_FOUND)
|
||||||
|
set(RAPIDJSON_FOUND ${Rapidjson_FOUND})
|
||||||
|
endif()
|
||||||
|
|
@ -57,6 +57,7 @@ find_package(FreeImage REQUIRED)
|
||||||
find_package(SDL2 REQUIRED)
|
find_package(SDL2 REQUIRED)
|
||||||
find_package(CURL REQUIRED)
|
find_package(CURL REQUIRED)
|
||||||
find_package(VLC REQUIRED)
|
find_package(VLC REQUIRED)
|
||||||
|
find_package(RapidJSON REQUIRED)
|
||||||
find_package(libCEC)
|
find_package(libCEC)
|
||||||
|
|
||||||
#add ALSA for Linux
|
#add ALSA for Linux
|
||||||
|
|
@ -120,6 +121,7 @@ set(COMMON_INCLUDE_DIRS
|
||||||
${SDL2_INCLUDE_DIR}
|
${SDL2_INCLUDE_DIR}
|
||||||
${CURL_INCLUDE_DIR}
|
${CURL_INCLUDE_DIR}
|
||||||
${VLC_INCLUDE_DIR}
|
${VLC_INCLUDE_DIR}
|
||||||
|
${RAPIDJSON_INCLUDE_DIRS}
|
||||||
${CMAKE_CURRENT_SOURCE_DIR}/external
|
${CMAKE_CURRENT_SOURCE_DIR}/external
|
||||||
${CMAKE_CURRENT_SOURCE_DIR}/es-core/src
|
${CMAKE_CURRENT_SOURCE_DIR}/es-core/src
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -39,7 +39,8 @@ set(ES_HEADERS
|
||||||
|
|
||||||
# Scrapers
|
# Scrapers
|
||||||
${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/Scraper.h
|
${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/Scraper.h
|
||||||
${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/GamesDBScraper.h
|
${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/GamesDBJSONScraper.h
|
||||||
|
${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/GamesDBJSONScraperResources.h
|
||||||
${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/ScreenScraper.h
|
${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/ScreenScraper.h
|
||||||
|
|
||||||
# Views
|
# Views
|
||||||
|
|
@ -96,7 +97,8 @@ set(ES_SOURCES
|
||||||
|
|
||||||
# Scrapers
|
# Scrapers
|
||||||
${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/Scraper.cpp
|
${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/Scraper.cpp
|
||||||
${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/GamesDBScraper.cpp
|
${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/GamesDBJSONScraper.cpp
|
||||||
|
${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/GamesDBJSONScraperResources.cpp
|
||||||
${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/ScreenScraper.cpp
|
${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/ScreenScraper.cpp
|
||||||
|
|
||||||
# Views
|
# Views
|
||||||
|
|
@ -157,7 +159,7 @@ SET(CPACK_DEBIAN_PACKAGE_MAINTAINER "Alec Lofquist <allofquist@yahoo.com>")
|
||||||
SET(CPACK_DEBIAN_PACKAGE_SECTION "misc")
|
SET(CPACK_DEBIAN_PACKAGE_SECTION "misc")
|
||||||
SET(CPACK_DEBIAN_PACKAGE_PRIORITY "extra")
|
SET(CPACK_DEBIAN_PACKAGE_PRIORITY "extra")
|
||||||
SET(CPACK_DEBIAN_PACKAGE_DEPENDS "libc6, libsdl2-2.0-0, libfreeimage3, libfreetype6, libcurl3, libasound2")
|
SET(CPACK_DEBIAN_PACKAGE_DEPENDS "libc6, libsdl2-2.0-0, libfreeimage3, libfreetype6, libcurl3, libasound2")
|
||||||
SET(CPACK_DEBIAN_PACKAGE_BUILDS_DEPENDS "debhelper (>= 8.0.0), cmake, g++ (>= 4.8), libsdl2-dev, libfreeimage-dev, libfreetype6-dev, libcurl4-openssl-dev, libasound2-dev, libgl1-mesa-dev")
|
SET(CPACK_DEBIAN_PACKAGE_BUILDS_DEPENDS "debhelper (>= 8.0.0), cmake, g++ (>= 4.8), libsdl2-dev, libfreeimage-dev, libfreetype6-dev, libcurl4-openssl-dev, libasound2-dev, libgl1-mesa-dev, rapidjson-dev")
|
||||||
|
|
||||||
SET(CPACK_PACKAGE_VENDOR "emulationstation.org")
|
SET(CPACK_PACKAGE_VENDOR "emulationstation.org")
|
||||||
SET(CPACK_PACKAGE_VERSION "2.0.0~rc1")
|
SET(CPACK_PACKAGE_VERSION "2.0.0~rc1")
|
||||||
|
|
|
||||||
|
|
@ -58,7 +58,7 @@ void GuiMenu::openScraperSettings()
|
||||||
|
|
||||||
// Select either the first entry of the one read from the settings, just in case the scraper from settings has vanished.
|
// Select either the first entry of the one read from the settings, just in case the scraper from settings has vanished.
|
||||||
for(auto it = scrapers.cbegin(); it != scrapers.cend(); it++)
|
for(auto it = scrapers.cbegin(); it != scrapers.cend(); it++)
|
||||||
scraper_list->add(*it, *it, *it == Settings::getInstance()->getString("Scraper") || it==scrapers.cbegin());
|
scraper_list->add(*it, *it, *it == Settings::getInstance()->getString("Scraper"));
|
||||||
|
|
||||||
s->addWithLabel("SCRAPE FROM", scraper_list);
|
s->addWithLabel("SCRAPE FROM", scraper_list);
|
||||||
s->addSaveFunc([scraper_list] { Settings::getInstance()->setString("Scraper", scraper_list->getSelected()); });
|
s->addSaveFunc([scraper_list] { Settings::getInstance()->setString("Scraper", scraper_list->getSelected()); });
|
||||||
|
|
|
||||||
392
es-app/src/scrapers/GamesDBJSONScraper.cpp
Normal file
392
es-app/src/scrapers/GamesDBJSONScraper.cpp
Normal file
|
|
@ -0,0 +1,392 @@
|
||||||
|
#include <exception>
|
||||||
|
#include <map>
|
||||||
|
|
||||||
|
#include "scrapers/GamesDBJSONScraper.h"
|
||||||
|
#include "scrapers/GamesDBJSONScraperResources.h"
|
||||||
|
|
||||||
|
#include "FileData.h"
|
||||||
|
#include "Log.h"
|
||||||
|
#include "PlatformId.h"
|
||||||
|
#include "Settings.h"
|
||||||
|
#include "SystemData.h"
|
||||||
|
#include "utils/TimeUtil.h"
|
||||||
|
#include <pugixml/src/pugixml.hpp>
|
||||||
|
|
||||||
|
/* When raspbian will get an up to date version of rapidjson we'll be
|
||||||
|
able to have it throw in case of error with the following:
|
||||||
|
#ifndef RAPIDJSON_ASSERT
|
||||||
|
#define RAPIDJSON_ASSERT(x) \
|
||||||
|
if (!(x)) { \
|
||||||
|
throw std::runtime_error("rapidjson internal assertion failure: " #x); \
|
||||||
|
}
|
||||||
|
#endif // RAPIDJSON_ASSERT
|
||||||
|
*/
|
||||||
|
|
||||||
|
#include <rapidjson/document.h>
|
||||||
|
#include <rapidjson/error/en.h>
|
||||||
|
|
||||||
|
using namespace PlatformIds;
|
||||||
|
using namespace rapidjson;
|
||||||
|
|
||||||
|
namespace
|
||||||
|
{
|
||||||
|
TheGamesDBJSONRequestResources resources;
|
||||||
|
}
|
||||||
|
|
||||||
|
const std::map<PlatformId, std::string> gamesdb_new_platformid_map{
|
||||||
|
{ THREEDO, "25" },
|
||||||
|
{ AMIGA, "4911" },
|
||||||
|
{ AMSTRAD_CPC, "4914" },
|
||||||
|
{ APPLE_II, "4942" },
|
||||||
|
{ ARCADE, "23" },
|
||||||
|
{ ATARI_800, "4943" },
|
||||||
|
{ ATARI_2600, "22" },
|
||||||
|
{ ATARI_5200, "26" },
|
||||||
|
{ ATARI_7800, "27" },
|
||||||
|
{ ATARI_JAGUAR, "28" },
|
||||||
|
{ ATARI_JAGUAR_CD, "29" },
|
||||||
|
{ ATARI_LYNX, "4924" },
|
||||||
|
{ ATARI_ST, "4937" },
|
||||||
|
{ ATARI_XE, "30" },
|
||||||
|
{ COLECOVISION, "31" },
|
||||||
|
{ COMMODORE_64, "40" },
|
||||||
|
{ INTELLIVISION, "32" },
|
||||||
|
{ MAC_OS, "37" },
|
||||||
|
{ XBOX, "14" },
|
||||||
|
{ XBOX_360, "15" },
|
||||||
|
{ MSX, "4929" },
|
||||||
|
{ NEOGEO, "24" },
|
||||||
|
{ NEOGEO_POCKET, "4922" },
|
||||||
|
{ NEOGEO_POCKET_COLOR, "4923" },
|
||||||
|
{ NINTENDO_3DS, "4912" },
|
||||||
|
{ NINTENDO_64, "3" },
|
||||||
|
{ NINTENDO_DS, "8" },
|
||||||
|
{ FAMICOM_DISK_SYSTEM, "4936" },
|
||||||
|
{ NINTENDO_ENTERTAINMENT_SYSTEM, "7" },
|
||||||
|
{ GAME_BOY, "4" },
|
||||||
|
{ GAME_BOY_ADVANCE, "5" },
|
||||||
|
{ GAME_BOY_COLOR, "41" },
|
||||||
|
{ NINTENDO_GAMECUBE, "2" },
|
||||||
|
{ NINTENDO_WII, "9" },
|
||||||
|
{ NINTENDO_WII_U, "38" },
|
||||||
|
{ NINTENDO_VIRTUAL_BOY, "4918" },
|
||||||
|
{ NINTENDO_GAME_AND_WATCH, "-1" },
|
||||||
|
{ PC, "1" },
|
||||||
|
{ SEGA_32X, "33" },
|
||||||
|
{ SEGA_CD, "21" },
|
||||||
|
{ SEGA_DREAMCAST, "16" },
|
||||||
|
{ SEGA_GAME_GEAR, "20" },
|
||||||
|
{ SEGA_GENESIS, "18" },
|
||||||
|
{ SEGA_MASTER_SYSTEM, "35" },
|
||||||
|
{ SEGA_MEGA_DRIVE, "36" },
|
||||||
|
{ SEGA_SATURN, "17" },
|
||||||
|
{ SEGA_SG1000, "4949" },
|
||||||
|
{ PLAYSTATION, "10" },
|
||||||
|
{ PLAYSTATION_2, "11" },
|
||||||
|
{ PLAYSTATION_3, "12" },
|
||||||
|
{ PLAYSTATION_4, "4919" },
|
||||||
|
{ PLAYSTATION_VITA, "39" },
|
||||||
|
{ PLAYSTATION_PORTABLE, "13" },
|
||||||
|
{ SUPER_NINTENDO, "6" },
|
||||||
|
{ TURBOGRAFX_16, "34" }, // HuCards only
|
||||||
|
{ TURBOGRAFX_CD, "4955" }, // CD-ROMs only
|
||||||
|
{ WONDERSWAN, "4925" },
|
||||||
|
{ WONDERSWAN_COLOR, "4926" },
|
||||||
|
{ ZX_SPECTRUM, "4913" },
|
||||||
|
{ VIDEOPAC_ODYSSEY2, "4927" },
|
||||||
|
{ VECTREX, "4939" },
|
||||||
|
{ TRS80_COLOR_COMPUTER, "4941" },
|
||||||
|
{ TANDY, "4941" },
|
||||||
|
};
|
||||||
|
|
||||||
|
void thegamesdb_generate_json_scraper_requests(const ScraperSearchParams& params,
|
||||||
|
std::queue<std::unique_ptr<ScraperRequest>>& requests, std::vector<ScraperSearchResult>& results)
|
||||||
|
{
|
||||||
|
resources.prepare();
|
||||||
|
std::string path = "https://api.thegamesdb.net";
|
||||||
|
bool usingGameID = false;
|
||||||
|
const std::string apiKey = std::string("apikey=") + resources.getApiKey();
|
||||||
|
std::string cleanName = params.nameOverride;
|
||||||
|
if (!cleanName.empty() && cleanName.substr(0, 3) == "id:")
|
||||||
|
{
|
||||||
|
std::string gameID = cleanName.substr(3);
|
||||||
|
path += "/Games/ByGameID?" + apiKey +
|
||||||
|
"&fields=players,publishers,genres,overview,last_updated,rating,"
|
||||||
|
"platform,coop,youtube,os,processor,ram,hdd,video,sound,alternates&"
|
||||||
|
"include=boxart&id=" +
|
||||||
|
HttpReq::urlEncode(gameID);
|
||||||
|
usingGameID = true;
|
||||||
|
} else
|
||||||
|
{
|
||||||
|
if (cleanName.empty())
|
||||||
|
cleanName = params.game->getCleanName();
|
||||||
|
path += "/Games/ByGameName?" + apiKey +
|
||||||
|
"&fields=players,publishers,genres,overview,last_updated,rating,"
|
||||||
|
"platform,coop,youtube,os,processor,ram,hdd,video,sound,alternates&"
|
||||||
|
"include=boxart&name=" +
|
||||||
|
HttpReq::urlEncode(cleanName);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (usingGameID)
|
||||||
|
{
|
||||||
|
// if we have the ID already, we don't need the GetGameList request
|
||||||
|
requests.push(std::unique_ptr<ScraperRequest>(new TheGamesDBJSONRequest(results, path)));
|
||||||
|
} else
|
||||||
|
{
|
||||||
|
std::string platformQueryParam;
|
||||||
|
auto& platforms = params.system->getPlatformIds();
|
||||||
|
if (!platforms.empty())
|
||||||
|
{
|
||||||
|
bool first = true;
|
||||||
|
platformQueryParam += "&filter%5Bplatform%5D=";
|
||||||
|
for (auto platformIt = platforms.cbegin(); platformIt != platforms.cend(); platformIt++)
|
||||||
|
{
|
||||||
|
auto mapIt = gamesdb_new_platformid_map.find(*platformIt);
|
||||||
|
if (mapIt != gamesdb_new_platformid_map.cend())
|
||||||
|
{
|
||||||
|
if (!first)
|
||||||
|
{
|
||||||
|
platformQueryParam += ",";
|
||||||
|
}
|
||||||
|
platformQueryParam += HttpReq::urlEncode(mapIt->second);
|
||||||
|
first = false;
|
||||||
|
} else
|
||||||
|
{
|
||||||
|
LOG(LogWarning) << "TheGamesDB scraper warning - no support for platform "
|
||||||
|
<< getPlatformName(*platformIt);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
path += platformQueryParam;
|
||||||
|
}
|
||||||
|
|
||||||
|
requests.push(std::unique_ptr<ScraperRequest>(new TheGamesDBJSONRequest(requests, results, path)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
namespace
|
||||||
|
{
|
||||||
|
|
||||||
|
std::string getStringOrThrow(const Value& v, const std::string& key)
|
||||||
|
{
|
||||||
|
if (!v.HasMember(key.c_str()) || !v[key.c_str()].IsString())
|
||||||
|
{
|
||||||
|
throw std::runtime_error("rapidjson internal assertion failure: missing or non string key:" + key);
|
||||||
|
}
|
||||||
|
return v[key.c_str()].GetString();
|
||||||
|
}
|
||||||
|
|
||||||
|
int getIntOrThrow(const Value& v, const std::string& key)
|
||||||
|
{
|
||||||
|
if (!v.HasMember(key.c_str()) || !v[key.c_str()].IsInt())
|
||||||
|
{
|
||||||
|
throw std::runtime_error("rapidjson internal assertion failure: missing or non int key:" + key);
|
||||||
|
}
|
||||||
|
return v[key.c_str()].GetInt();
|
||||||
|
}
|
||||||
|
|
||||||
|
int getIntOrThrow(const Value& v)
|
||||||
|
{
|
||||||
|
if (!v.IsInt())
|
||||||
|
{
|
||||||
|
throw std::runtime_error("rapidjson internal assertion failure: not an int");
|
||||||
|
}
|
||||||
|
return v.GetInt();
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string getBoxartImage(const Value& v)
|
||||||
|
{
|
||||||
|
if (!v.IsArray() || v.Size() == 0)
|
||||||
|
{
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
for (int i = 0; i < v.Size(); ++i)
|
||||||
|
{
|
||||||
|
auto& im = v[i];
|
||||||
|
std::string type = getStringOrThrow(im, "type");
|
||||||
|
std::string side = getStringOrThrow(im, "side");
|
||||||
|
if (type == "boxart" && side == "front")
|
||||||
|
{
|
||||||
|
return getStringOrThrow(im, "filename");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return getStringOrThrow(v[0], "filename");
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string getDeveloperString(const Value& v)
|
||||||
|
{
|
||||||
|
if (!v.IsArray())
|
||||||
|
{
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
std::string out = "";
|
||||||
|
bool first = true;
|
||||||
|
for (int i = 0; i < v.Size(); ++i)
|
||||||
|
{
|
||||||
|
auto mapIt = resources.gamesdb_new_developers_map.find(getIntOrThrow(v[i]));
|
||||||
|
if (mapIt == resources.gamesdb_new_developers_map.cend())
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!first)
|
||||||
|
{
|
||||||
|
out += ", ";
|
||||||
|
}
|
||||||
|
out += mapIt->second;
|
||||||
|
first = false;
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string getPublisherString(const Value& v)
|
||||||
|
{
|
||||||
|
if (!v.IsArray())
|
||||||
|
{
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
std::string out = "";
|
||||||
|
bool first = true;
|
||||||
|
for (int i = 0; i < v.Size(); ++i)
|
||||||
|
{
|
||||||
|
auto mapIt = resources.gamesdb_new_publishers_map.find(getIntOrThrow(v[i]));
|
||||||
|
if (mapIt == resources.gamesdb_new_publishers_map.cend())
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!first)
|
||||||
|
{
|
||||||
|
out += ", ";
|
||||||
|
}
|
||||||
|
out += mapIt->second;
|
||||||
|
first = false;
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string getGenreString(const Value& v)
|
||||||
|
{
|
||||||
|
if (!v.IsArray())
|
||||||
|
{
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
std::string out = "";
|
||||||
|
bool first = true;
|
||||||
|
for (int i = 0; i < v.Size(); ++i)
|
||||||
|
{
|
||||||
|
auto mapIt = resources.gamesdb_new_genres_map.find(getIntOrThrow(v[i]));
|
||||||
|
if (mapIt == resources.gamesdb_new_genres_map.cend())
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!first)
|
||||||
|
{
|
||||||
|
out += ", ";
|
||||||
|
}
|
||||||
|
out += mapIt->second;
|
||||||
|
first = false;
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
void processGame(const Value& game, const Value& boxart, std::vector<ScraperSearchResult>& results)
|
||||||
|
{
|
||||||
|
std::string baseImageUrlThumb = getStringOrThrow(boxart["base_url"], "thumb");
|
||||||
|
std::string baseImageUrlLarge = getStringOrThrow(boxart["base_url"], "large");
|
||||||
|
|
||||||
|
ScraperSearchResult result;
|
||||||
|
|
||||||
|
result.mdl.set("name", getStringOrThrow(game, "game_title"));
|
||||||
|
if (game.HasMember("overview") && game["overview"].IsString())
|
||||||
|
{
|
||||||
|
result.mdl.set("desc", game["overview"].GetString());
|
||||||
|
}
|
||||||
|
if (game.HasMember("release_date") && game["release_date"].IsString())
|
||||||
|
{
|
||||||
|
result.mdl.set(
|
||||||
|
"releasedate", Utils::Time::DateTime(Utils::Time::stringToTime(game["release_date"].GetString(), "%Y-%m-%d")));
|
||||||
|
}
|
||||||
|
if (game.HasMember("developers") && game["developers"].IsArray())
|
||||||
|
{
|
||||||
|
result.mdl.set("developer", getDeveloperString(game["developers"]));
|
||||||
|
}
|
||||||
|
if (game.HasMember("publishers") && game["publishers"].IsArray())
|
||||||
|
{
|
||||||
|
result.mdl.set("publisher", getPublisherString(game["publishers"]));
|
||||||
|
}
|
||||||
|
if (game.HasMember("genres") && game["genres"].IsArray())
|
||||||
|
{
|
||||||
|
|
||||||
|
result.mdl.set("genre", getGenreString(game["genres"]));
|
||||||
|
}
|
||||||
|
if (game.HasMember("players") && game["players"].IsInt())
|
||||||
|
{
|
||||||
|
result.mdl.set("players", std::to_string(game["players"].GetInt()));
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string id = std::to_string(getIntOrThrow(game, "id"));
|
||||||
|
if (boxart["data"].HasMember(id.c_str()))
|
||||||
|
{
|
||||||
|
std::string image = getBoxartImage(boxart["data"][id.c_str()]);
|
||||||
|
result.thumbnailUrl = baseImageUrlThumb + "/" + image;
|
||||||
|
result.imageUrl = baseImageUrlLarge + "/" + image;
|
||||||
|
}
|
||||||
|
|
||||||
|
results.push_back(result);
|
||||||
|
}
|
||||||
|
} // namespace
|
||||||
|
|
||||||
|
void TheGamesDBJSONRequest::process(const std::unique_ptr<HttpReq>& req, std::vector<ScraperSearchResult>& results)
|
||||||
|
{
|
||||||
|
assert(req->status() == HttpReq::REQ_SUCCESS);
|
||||||
|
|
||||||
|
Document doc;
|
||||||
|
doc.Parse(req->getContent().c_str());
|
||||||
|
|
||||||
|
if (doc.HasParseError())
|
||||||
|
{
|
||||||
|
std::string err =
|
||||||
|
std::string("TheGamesDBJSONRequest - Error parsing JSON. \n\t") + GetParseError_En(doc.GetParseError());
|
||||||
|
setError(err);
|
||||||
|
LOG(LogError) << err;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!doc.HasMember("data") || !doc["data"].HasMember("games") || !doc["data"]["games"].IsArray())
|
||||||
|
{
|
||||||
|
std::string warn = "TheGamesDBJSONRequest - Response had no game data.\n";
|
||||||
|
LOG(LogWarning) << warn;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const Value& games = doc["data"]["games"];
|
||||||
|
|
||||||
|
if (!doc.HasMember("include") || !doc["include"].HasMember("boxart"))
|
||||||
|
{
|
||||||
|
std::string warn = "TheGamesDBJSONRequest - Response had no include boxart data.\n";
|
||||||
|
LOG(LogWarning) << warn;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const Value& boxart = doc["include"]["boxart"];
|
||||||
|
|
||||||
|
if (!boxart.HasMember("base_url") || !boxart.HasMember("data") || !boxart.IsObject())
|
||||||
|
{
|
||||||
|
std::string warn = "TheGamesDBJSONRequest - Response include had no usable boxart data.\n";
|
||||||
|
LOG(LogWarning) << warn;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
resources.ensureResources();
|
||||||
|
|
||||||
|
|
||||||
|
for (int i = 0; i < games.Size(); ++i)
|
||||||
|
{
|
||||||
|
auto& v = games[i];
|
||||||
|
try
|
||||||
|
{
|
||||||
|
processGame(v, boxart, results);
|
||||||
|
}
|
||||||
|
catch (std::runtime_error& e)
|
||||||
|
{
|
||||||
|
LOG(LogError) << "Error while processing game: " << e.what();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
37
es-app/src/scrapers/GamesDBJSONScraper.h
Normal file
37
es-app/src/scrapers/GamesDBJSONScraper.h
Normal file
|
|
@ -0,0 +1,37 @@
|
||||||
|
#pragma once
|
||||||
|
#ifndef ES_APP_SCRAPERS_GAMES_DB_JSON_SCRAPER_H
|
||||||
|
#define ES_APP_SCRAPERS_GAMES_DB_JSON_SCRAPER_H
|
||||||
|
|
||||||
|
#include "scrapers/Scraper.h"
|
||||||
|
|
||||||
|
namespace pugi
|
||||||
|
{
|
||||||
|
class xml_document;
|
||||||
|
}
|
||||||
|
|
||||||
|
void thegamesdb_generate_json_scraper_requests(const ScraperSearchParams& params,
|
||||||
|
std::queue<std::unique_ptr<ScraperRequest>>& requests, std::vector<ScraperSearchResult>& results);
|
||||||
|
|
||||||
|
class TheGamesDBJSONRequest : public ScraperHttpRequest
|
||||||
|
{
|
||||||
|
public:
|
||||||
|
// ctor for a GetGameList request
|
||||||
|
TheGamesDBJSONRequest(std::queue<std::unique_ptr<ScraperRequest>>& requestsWrite,
|
||||||
|
std::vector<ScraperSearchResult>& resultsWrite, const std::string& url)
|
||||||
|
: ScraperHttpRequest(resultsWrite, url), mRequestQueue(&requestsWrite)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
// ctor for a GetGame request
|
||||||
|
TheGamesDBJSONRequest(std::vector<ScraperSearchResult>& resultsWrite, const std::string& url)
|
||||||
|
: ScraperHttpRequest(resultsWrite, url), mRequestQueue(nullptr)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
protected:
|
||||||
|
void process(const std::unique_ptr<HttpReq>& req, std::vector<ScraperSearchResult>& results) override;
|
||||||
|
bool isGameRequest() { return !mRequestQueue; }
|
||||||
|
|
||||||
|
std::queue<std::unique_ptr<ScraperRequest>>* mRequestQueue;
|
||||||
|
};
|
||||||
|
|
||||||
|
#endif // ES_APP_SCRAPERS_GAMES_DB_JSON_SCRAPER_H
|
||||||
207
es-app/src/scrapers/GamesDBJSONScraperResources.cpp
Normal file
207
es-app/src/scrapers/GamesDBJSONScraperResources.cpp
Normal file
|
|
@ -0,0 +1,207 @@
|
||||||
|
#include <chrono>
|
||||||
|
#include <fstream>
|
||||||
|
#include <memory>
|
||||||
|
#include <thread>
|
||||||
|
|
||||||
|
#include "Log.h"
|
||||||
|
|
||||||
|
#include "scrapers/GamesDBJSONScraperResources.h"
|
||||||
|
#include "utils/FileSystemUtil.h"
|
||||||
|
|
||||||
|
|
||||||
|
#include <rapidjson/document.h>
|
||||||
|
#include <rapidjson/error/en.h>
|
||||||
|
|
||||||
|
using namespace rapidjson;
|
||||||
|
|
||||||
|
|
||||||
|
namespace
|
||||||
|
{
|
||||||
|
constexpr char GamesDBAPIKey[] = "445fcbc3f32bb2474bc27016b99eb963d318ee3a608212c543b9a79de1041600";
|
||||||
|
|
||||||
|
|
||||||
|
constexpr int MAX_WAIT_MS = 90000;
|
||||||
|
constexpr int POLL_TIME_MS = 500;
|
||||||
|
constexpr int MAX_WAIT_ITER = MAX_WAIT_MS / POLL_TIME_MS;
|
||||||
|
|
||||||
|
constexpr char SCRAPER_RESOURCES_DIR[] = "scrapers";
|
||||||
|
constexpr char DEVELOPERS_JSON_FILE[] = "gamesdb_developers.json";
|
||||||
|
constexpr char PUBLISHERS_JSON_FILE[] = "gamesdb_publishers.json";
|
||||||
|
constexpr char GENRES_JSON_FILE[] = "gamesdb_genres.json";
|
||||||
|
constexpr char DEVELOPERS_ENDPOINT[] = "/Developers";
|
||||||
|
constexpr char PUBLISHERS_ENDPOINT[] = "/Publishers";
|
||||||
|
constexpr char GENRES_ENDPOINT[] = "/Genres";
|
||||||
|
|
||||||
|
std::string genFilePath(const std::string& file_name)
|
||||||
|
{
|
||||||
|
return Utils::FileSystem::getGenericPath(getScrapersResouceDir() + "/" + file_name);
|
||||||
|
}
|
||||||
|
|
||||||
|
void ensureScrapersResourcesDir()
|
||||||
|
{
|
||||||
|
std::string path = getScrapersResouceDir();
|
||||||
|
if (!Utils::FileSystem::exists(path))
|
||||||
|
Utils::FileSystem::createDirectory(path);
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace
|
||||||
|
|
||||||
|
|
||||||
|
std::string getScrapersResouceDir()
|
||||||
|
{
|
||||||
|
return Utils::FileSystem::getGenericPath(
|
||||||
|
Utils::FileSystem::getHomePath() + "/.emulationstation/" + SCRAPER_RESOURCES_DIR);
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string TheGamesDBJSONRequestResources::getApiKey() const { return GamesDBAPIKey; }
|
||||||
|
|
||||||
|
|
||||||
|
void TheGamesDBJSONRequestResources::prepare()
|
||||||
|
{
|
||||||
|
if (checkLoaded())
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (loadResource(gamesdb_new_developers_map, "developers", genFilePath(DEVELOPERS_JSON_FILE)) &&
|
||||||
|
!gamesdb_developers_resource_request)
|
||||||
|
{
|
||||||
|
gamesdb_developers_resource_request = fetchResource(DEVELOPERS_ENDPOINT);
|
||||||
|
}
|
||||||
|
if (loadResource(gamesdb_new_publishers_map, "publishers", genFilePath(PUBLISHERS_JSON_FILE)) &&
|
||||||
|
!gamesdb_publishers_resource_request)
|
||||||
|
{
|
||||||
|
gamesdb_publishers_resource_request = fetchResource(PUBLISHERS_ENDPOINT);
|
||||||
|
}
|
||||||
|
if (loadResource(gamesdb_new_genres_map, "genres", genFilePath(GENRES_JSON_FILE)) && !gamesdb_genres_resource_request)
|
||||||
|
{
|
||||||
|
gamesdb_genres_resource_request = fetchResource(GENRES_ENDPOINT);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void TheGamesDBJSONRequestResources::ensureResources()
|
||||||
|
{
|
||||||
|
|
||||||
|
if (checkLoaded())
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
for (int i = 0; i < MAX_WAIT_ITER; ++i)
|
||||||
|
{
|
||||||
|
if (gamesdb_developers_resource_request &&
|
||||||
|
saveResource(gamesdb_developers_resource_request.get(), gamesdb_new_developers_map, "developers",
|
||||||
|
genFilePath(DEVELOPERS_JSON_FILE)))
|
||||||
|
{
|
||||||
|
|
||||||
|
gamesdb_developers_resource_request.reset(nullptr);
|
||||||
|
}
|
||||||
|
if (gamesdb_publishers_resource_request &&
|
||||||
|
saveResource(gamesdb_publishers_resource_request.get(), gamesdb_new_publishers_map, "publishers",
|
||||||
|
genFilePath(PUBLISHERS_JSON_FILE)))
|
||||||
|
{
|
||||||
|
gamesdb_publishers_resource_request.reset(nullptr);
|
||||||
|
}
|
||||||
|
if (gamesdb_genres_resource_request && saveResource(gamesdb_genres_resource_request.get(), gamesdb_new_genres_map,
|
||||||
|
"genres", genFilePath(GENRES_JSON_FILE)))
|
||||||
|
{
|
||||||
|
gamesdb_genres_resource_request.reset(nullptr);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!gamesdb_developers_resource_request && !gamesdb_publishers_resource_request && !gamesdb_genres_resource_request)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
std::this_thread::sleep_for(std::chrono::milliseconds(POLL_TIME_MS));
|
||||||
|
}
|
||||||
|
LOG(LogError) << "Timed out while waiting for resources\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
bool TheGamesDBJSONRequestResources::checkLoaded()
|
||||||
|
{
|
||||||
|
return !gamesdb_new_genres_map.empty() && !gamesdb_new_developers_map.empty() && !gamesdb_new_publishers_map.empty();
|
||||||
|
}
|
||||||
|
|
||||||
|
bool TheGamesDBJSONRequestResources::saveResource(HttpReq* req, std::unordered_map<int, std::string>& resource,
|
||||||
|
const std::string& resource_name, const std::string& file_name)
|
||||||
|
{
|
||||||
|
|
||||||
|
if (req == nullptr)
|
||||||
|
{
|
||||||
|
LOG(LogError) << "Http request pointer was null\n";
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (req->status() == HttpReq::REQ_IN_PROGRESS)
|
||||||
|
{
|
||||||
|
return false; // Not ready: wait some more
|
||||||
|
}
|
||||||
|
if (req->status() != HttpReq::REQ_SUCCESS)
|
||||||
|
{
|
||||||
|
LOG(LogError) << "Resource request for " << file_name << " failed:\n\t" << req->getErrorMsg();
|
||||||
|
return true; // Request failed, resetting request.
|
||||||
|
}
|
||||||
|
|
||||||
|
ensureScrapersResourcesDir();
|
||||||
|
|
||||||
|
std::ofstream fout(file_name);
|
||||||
|
fout << req->getContent();
|
||||||
|
fout.close();
|
||||||
|
loadResource(resource, resource_name, file_name);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::unique_ptr<HttpReq> TheGamesDBJSONRequestResources::fetchResource(const std::string& endpoint)
|
||||||
|
{
|
||||||
|
std::string path = "https://api.thegamesdb.net";
|
||||||
|
path += endpoint;
|
||||||
|
path += "?apikey=" + getApiKey();
|
||||||
|
|
||||||
|
return std::unique_ptr<HttpReq>(new HttpReq(path));
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
int TheGamesDBJSONRequestResources::loadResource(
|
||||||
|
std::unordered_map<int, std::string>& resource, const std::string& resource_name, const std::string& file_name)
|
||||||
|
{
|
||||||
|
|
||||||
|
|
||||||
|
std::ifstream fin(file_name);
|
||||||
|
if (!fin.good())
|
||||||
|
{
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
std::stringstream buffer;
|
||||||
|
buffer << fin.rdbuf();
|
||||||
|
Document doc;
|
||||||
|
doc.Parse(buffer.str().c_str());
|
||||||
|
|
||||||
|
if (doc.HasParseError())
|
||||||
|
{
|
||||||
|
std::string err = std::string("TheGamesDBJSONRequest - Error parsing JSON for resource file ") + file_name +
|
||||||
|
":\n\t" + GetParseError_En(doc.GetParseError());
|
||||||
|
LOG(LogError) << err;
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!doc.HasMember("data") || !doc["data"].HasMember(resource_name.c_str()) ||
|
||||||
|
!doc["data"][resource_name.c_str()].IsObject())
|
||||||
|
{
|
||||||
|
std::string err = "TheGamesDBJSONRequest - Response had no resource data.\n";
|
||||||
|
LOG(LogError) << err;
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
auto& data = doc["data"][resource_name.c_str()];
|
||||||
|
|
||||||
|
for (Value::ConstMemberIterator itr = data.MemberBegin(); itr != data.MemberEnd(); ++itr)
|
||||||
|
{
|
||||||
|
auto& entry = itr->value;
|
||||||
|
if (!entry.IsObject() || !entry.HasMember("id") || !entry["id"].IsInt() || !entry.HasMember("name") ||
|
||||||
|
!entry["name"].IsString())
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
resource[entry["id"].GetInt()] = entry["name"].GetString();
|
||||||
|
}
|
||||||
|
return resource.empty();
|
||||||
|
}
|
||||||
42
es-app/src/scrapers/GamesDBJSONScraperResources.h
Normal file
42
es-app/src/scrapers/GamesDBJSONScraperResources.h
Normal file
|
|
@ -0,0 +1,42 @@
|
||||||
|
#pragma once
|
||||||
|
#ifndef ES_APP_SCRAPERS_GAMES_DB_JSON_SCRAPER_RESOURCES_H
|
||||||
|
#define ES_APP_SCRAPERS_GAMES_DB_JSON_SCRAPER_RESOURCES_H
|
||||||
|
|
||||||
|
#include <fstream>
|
||||||
|
#include <memory>
|
||||||
|
#include <string>
|
||||||
|
#include <unordered_map>
|
||||||
|
|
||||||
|
#include "HttpReq.h"
|
||||||
|
|
||||||
|
|
||||||
|
struct TheGamesDBJSONRequestResources
|
||||||
|
{
|
||||||
|
TheGamesDBJSONRequestResources() = default;
|
||||||
|
|
||||||
|
void prepare();
|
||||||
|
void ensureResources();
|
||||||
|
std::string getApiKey() const;
|
||||||
|
|
||||||
|
std::unordered_map<int, std::string> gamesdb_new_developers_map;
|
||||||
|
std::unordered_map<int, std::string> gamesdb_new_publishers_map;
|
||||||
|
std::unordered_map<int, std::string> gamesdb_new_genres_map;
|
||||||
|
|
||||||
|
private:
|
||||||
|
bool checkLoaded();
|
||||||
|
|
||||||
|
bool saveResource(HttpReq* req, std::unordered_map<int, std::string>& resource, const std::string& resource_name,
|
||||||
|
const std::string& file_name);
|
||||||
|
std::unique_ptr<HttpReq> fetchResource(const std::string& endpoint);
|
||||||
|
|
||||||
|
int loadResource(
|
||||||
|
std::unordered_map<int, std::string>& resource, const std::string& resource_name, const std::string& file_name);
|
||||||
|
|
||||||
|
std::unique_ptr<HttpReq> gamesdb_developers_resource_request;
|
||||||
|
std::unique_ptr<HttpReq> gamesdb_publishers_resource_request;
|
||||||
|
std::unique_ptr<HttpReq> gamesdb_genres_resource_request;
|
||||||
|
};
|
||||||
|
|
||||||
|
std::string getScrapersResouceDir();
|
||||||
|
|
||||||
|
#endif // ES_APP_SCRAPERS_GAMES_DB_JSON_SCRAPER_H
|
||||||
|
|
@ -1,211 +0,0 @@
|
||||||
#include "scrapers/GamesDBScraper.h"
|
|
||||||
|
|
||||||
#include "utils/TimeUtil.h"
|
|
||||||
#include "FileData.h"
|
|
||||||
#include "Log.h"
|
|
||||||
#include "PlatformId.h"
|
|
||||||
#include "Settings.h"
|
|
||||||
#include "SystemData.h"
|
|
||||||
#include <pugixml/src/pugixml.hpp>
|
|
||||||
|
|
||||||
using namespace PlatformIds;
|
|
||||||
const std::map<PlatformId, const char*> gamesdb_platformid_map {
|
|
||||||
{ THREEDO, "3DO" },
|
|
||||||
{ AMIGA, "Amiga" },
|
|
||||||
{ AMSTRAD_CPC, "Amstrad CPC" },
|
|
||||||
// missing apple2
|
|
||||||
{ ARCADE, "Arcade" },
|
|
||||||
// missing atari 800
|
|
||||||
{ ATARI_2600, "Atari 2600" },
|
|
||||||
{ ATARI_5200, "Atari 5200" },
|
|
||||||
{ ATARI_7800, "Atari 7800" },
|
|
||||||
{ ATARI_JAGUAR, "Atari Jaguar" },
|
|
||||||
{ ATARI_JAGUAR_CD, "Atari Jaguar CD" },
|
|
||||||
{ ATARI_LYNX, "Atari Lynx" },
|
|
||||||
// missing atari ST/STE/Falcon
|
|
||||||
{ ATARI_XE, "Atari XE" },
|
|
||||||
{ COLECOVISION, "Colecovision" },
|
|
||||||
{ COMMODORE_64, "Commodore 64" },
|
|
||||||
{ INTELLIVISION, "Intellivision" },
|
|
||||||
{ MAC_OS, "Mac OS" },
|
|
||||||
{ XBOX, "Microsoft Xbox" },
|
|
||||||
{ XBOX_360, "Microsoft Xbox 360" },
|
|
||||||
{ MSX, "MSX" },
|
|
||||||
{ NEOGEO, "Neo Geo" },
|
|
||||||
{ NEOGEO_POCKET, "Neo Geo Pocket" },
|
|
||||||
{ NEOGEO_POCKET_COLOR, "Neo Geo Pocket Color" },
|
|
||||||
{ NINTENDO_3DS, "Nintendo 3DS" },
|
|
||||||
{ NINTENDO_64, "Nintendo 64" },
|
|
||||||
{ NINTENDO_DS, "Nintendo DS" },
|
|
||||||
{ FAMICOM_DISK_SYSTEM, "Famicom Disk System" },
|
|
||||||
{ NINTENDO_ENTERTAINMENT_SYSTEM, "Nintendo Entertainment System (NES)" },
|
|
||||||
{ GAME_BOY, "Nintendo Game Boy" },
|
|
||||||
{ GAME_BOY_ADVANCE, "Nintendo Game Boy Advance" },
|
|
||||||
{ GAME_BOY_COLOR, "Nintendo Game Boy Color" },
|
|
||||||
{ NINTENDO_GAMECUBE, "Nintendo GameCube" },
|
|
||||||
{ NINTENDO_WII, "Nintendo Wii" },
|
|
||||||
{ NINTENDO_WII_U, "Nintendo Wii U" },
|
|
||||||
{ NINTENDO_VIRTUAL_BOY, "Nintendo Virtual Boy" },
|
|
||||||
{ NINTENDO_GAME_AND_WATCH, "Game & Watch" },
|
|
||||||
{ PC, "PC" },
|
|
||||||
{ SEGA_32X, "Sega 32X" },
|
|
||||||
{ SEGA_CD, "Sega CD" },
|
|
||||||
{ SEGA_DREAMCAST, "Sega Dreamcast" },
|
|
||||||
{ SEGA_GAME_GEAR, "Sega Game Gear" },
|
|
||||||
{ SEGA_GENESIS, "Sega Genesis" },
|
|
||||||
{ SEGA_MASTER_SYSTEM, "Sega Master System" },
|
|
||||||
{ SEGA_MEGA_DRIVE, "Sega Mega Drive" },
|
|
||||||
{ SEGA_SATURN, "Sega Saturn" },
|
|
||||||
{ SEGA_SG1000, "SEGA SG-1000" },
|
|
||||||
{ PLAYSTATION, "Sony Playstation" },
|
|
||||||
{ PLAYSTATION_2, "Sony Playstation 2" },
|
|
||||||
{ PLAYSTATION_3, "Sony Playstation 3" },
|
|
||||||
{ PLAYSTATION_4, "Sony Playstation 4" },
|
|
||||||
{ PLAYSTATION_VITA, "Sony Playstation Vita" },
|
|
||||||
{ PLAYSTATION_PORTABLE, "Sony Playstation Portable" },
|
|
||||||
{ SUPER_NINTENDO, "Super Nintendo (SNES)" },
|
|
||||||
{ TURBOGRAFX_16, "TurboGrafx 16" }, // HuCards only
|
|
||||||
{ TURBOGRAFX_CD, "TurboGrafx CD" }, // CD-ROMs only
|
|
||||||
{ WONDERSWAN, "WonderSwan" },
|
|
||||||
{ WONDERSWAN_COLOR, "WonderSwan Color" },
|
|
||||||
{ ZX_SPECTRUM, "Sinclair ZX Spectrum" },
|
|
||||||
{ VIDEOPAC_ODYSSEY2, "Magnavox Odyssey 2" },
|
|
||||||
{ VECTREX, "Vectrex" },
|
|
||||||
{ TRS80_COLOR_COMPUTER, "TRS-80 Color Computer" },
|
|
||||||
{ TANDY, "TRS-80 Color Computer" }
|
|
||||||
};
|
|
||||||
|
|
||||||
void thegamesdb_generate_scraper_requests(const ScraperSearchParams& params, std::queue< std::unique_ptr<ScraperRequest> >& requests,
|
|
||||||
std::vector<ScraperSearchResult>& results)
|
|
||||||
{
|
|
||||||
std::string path;
|
|
||||||
bool usingGameID = false;
|
|
||||||
|
|
||||||
std::string cleanName = params.nameOverride;
|
|
||||||
if (!cleanName.empty() && cleanName.substr(0,3) == "id:")
|
|
||||||
{
|
|
||||||
std::string gameID = cleanName.substr(3);
|
|
||||||
path = "legacy.thegamesdb.net/api/GetGame.php?id=" + HttpReq::urlEncode(gameID);
|
|
||||||
usingGameID = true;
|
|
||||||
}else{
|
|
||||||
if (cleanName.empty())
|
|
||||||
cleanName = params.game->getCleanName();
|
|
||||||
path += "legacy.thegamesdb.net/api/GetGamesList.php?name=" + HttpReq::urlEncode(cleanName);
|
|
||||||
}
|
|
||||||
|
|
||||||
if(usingGameID)
|
|
||||||
{
|
|
||||||
// if we have the ID already, we don't need the GetGameList request
|
|
||||||
requests.push(std::unique_ptr<ScraperRequest>(new TheGamesDBRequest(results, path)));
|
|
||||||
}else if(params.system->getPlatformIds().empty()){
|
|
||||||
// no platform specified, we're done
|
|
||||||
requests.push(std::unique_ptr<ScraperRequest>(new TheGamesDBRequest(requests, results, path)));
|
|
||||||
}else{
|
|
||||||
// go through the list, we need to split this into multiple requests
|
|
||||||
// because TheGamesDB API either sucks or I don't know how to use it properly...
|
|
||||||
std::string urlBase = path;
|
|
||||||
auto& platforms = params.system->getPlatformIds();
|
|
||||||
for(auto platformIt = platforms.cbegin(); platformIt != platforms.cend(); platformIt++)
|
|
||||||
{
|
|
||||||
path = urlBase;
|
|
||||||
auto mapIt = gamesdb_platformid_map.find(*platformIt);
|
|
||||||
if(mapIt != gamesdb_platformid_map.cend())
|
|
||||||
{
|
|
||||||
path += "&platform=";
|
|
||||||
path += HttpReq::urlEncode(mapIt->second);
|
|
||||||
}else{
|
|
||||||
LOG(LogWarning) << "TheGamesDB scraper warning - no support for platform " << getPlatformName(*platformIt);
|
|
||||||
}
|
|
||||||
|
|
||||||
requests.push(std::unique_ptr<ScraperRequest>(new TheGamesDBRequest(requests, results, path)));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
void TheGamesDBRequest::process(const std::unique_ptr<HttpReq>& req, std::vector<ScraperSearchResult>& results)
|
|
||||||
{
|
|
||||||
assert(req->status() == HttpReq::REQ_SUCCESS);
|
|
||||||
|
|
||||||
pugi::xml_document doc;
|
|
||||||
pugi::xml_parse_result parseResult = doc.load(req->getContent().c_str());
|
|
||||||
if(!parseResult)
|
|
||||||
{
|
|
||||||
std::stringstream ss;
|
|
||||||
ss << "TheGamesDBRequest - Error parsing XML. \n\t" << parseResult.description() << "";
|
|
||||||
std::string err = ss.str();
|
|
||||||
setError(err);
|
|
||||||
LOG(LogError) << err;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (isGameRequest())
|
|
||||||
processGame(doc, results);
|
|
||||||
else
|
|
||||||
processList(doc, results);
|
|
||||||
}
|
|
||||||
|
|
||||||
void TheGamesDBRequest::processGame(const pugi::xml_document& xmldoc, std::vector<ScraperSearchResult>& results)
|
|
||||||
{
|
|
||||||
pugi::xml_node data = xmldoc.child("Data");
|
|
||||||
|
|
||||||
std::string baseImageUrl = data.child("baseImgUrl").text().get();
|
|
||||||
|
|
||||||
pugi::xml_node game = data.child("Game");
|
|
||||||
if(game)
|
|
||||||
{
|
|
||||||
ScraperSearchResult result;
|
|
||||||
|
|
||||||
result.mdl.set("name", game.child("GameTitle").text().get());
|
|
||||||
result.mdl.set("desc", game.child("Overview").text().get());
|
|
||||||
result.mdl.set("releasedate", Utils::Time::DateTime(Utils::Time::stringToTime(game.child("ReleaseDate").text().get(), "%m/%d/%Y")));
|
|
||||||
result.mdl.set("developer", game.child("Developer").text().get());
|
|
||||||
result.mdl.set("publisher", game.child("Publisher").text().get());
|
|
||||||
result.mdl.set("genre", game.child("Genres").first_child().text().get());
|
|
||||||
result.mdl.set("players", game.child("Players").text().get());
|
|
||||||
|
|
||||||
if(Settings::getInstance()->getBool("ScrapeRatings") && game.child("Rating"))
|
|
||||||
{
|
|
||||||
float ratingVal = (game.child("Rating").text().as_int() / 10.0f);
|
|
||||||
std::stringstream ss;
|
|
||||||
ss << ratingVal;
|
|
||||||
result.mdl.set("rating", ss.str());
|
|
||||||
}
|
|
||||||
|
|
||||||
pugi::xml_node images = game.child("Images");
|
|
||||||
|
|
||||||
if(images)
|
|
||||||
{
|
|
||||||
pugi::xml_node art = images.find_child_by_attribute("boxart", "side", "front");
|
|
||||||
|
|
||||||
if(art)
|
|
||||||
{
|
|
||||||
result.thumbnailUrl = baseImageUrl + art.attribute("thumb").as_string();
|
|
||||||
result.imageUrl = baseImageUrl + art.text().get();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
results.push_back(result);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
void TheGamesDBRequest::processList(const pugi::xml_document& xmldoc, std::vector<ScraperSearchResult>& results)
|
|
||||||
{
|
|
||||||
assert(mRequestQueue != nullptr);
|
|
||||||
|
|
||||||
pugi::xml_node data = xmldoc.child("Data");
|
|
||||||
pugi::xml_node game = data.child("Game");
|
|
||||||
|
|
||||||
// limit the number of results per platform, not in total.
|
|
||||||
// otherwise if the first platform returns >= 7 games
|
|
||||||
// but the second platform contains the relevant game,
|
|
||||||
// the relevant result would not be shown.
|
|
||||||
for(int i = 0; game && i < MAX_SCRAPER_RESULTS; i++)
|
|
||||||
{
|
|
||||||
std::string id = game.child("id").text().get();
|
|
||||||
std::string path = "legacy.thegamesdb.net/api/GetGame.php?id=" + id;
|
|
||||||
|
|
||||||
mRequestQueue->push(std::unique_ptr<ScraperRequest>(new TheGamesDBRequest(results, path)));
|
|
||||||
|
|
||||||
game = game.next_sibling("Game");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,29 +0,0 @@
|
||||||
#pragma once
|
|
||||||
#ifndef ES_APP_SCRAPERS_GAMES_DB_SCRAPER_H
|
|
||||||
#define ES_APP_SCRAPERS_GAMES_DB_SCRAPER_H
|
|
||||||
|
|
||||||
#include "scrapers/Scraper.h"
|
|
||||||
|
|
||||||
namespace pugi { class xml_document; }
|
|
||||||
|
|
||||||
void thegamesdb_generate_scraper_requests(const ScraperSearchParams& params, std::queue< std::unique_ptr<ScraperRequest> >& requests,
|
|
||||||
std::vector<ScraperSearchResult>& results);
|
|
||||||
|
|
||||||
class TheGamesDBRequest : public ScraperHttpRequest
|
|
||||||
{
|
|
||||||
public:
|
|
||||||
// ctor for a GetGameList request
|
|
||||||
TheGamesDBRequest(std::queue< std::unique_ptr<ScraperRequest> >& requestsWrite, std::vector<ScraperSearchResult>& resultsWrite, const std::string& url) : ScraperHttpRequest(resultsWrite, url), mRequestQueue(&requestsWrite) {}
|
|
||||||
// ctor for a GetGame request
|
|
||||||
TheGamesDBRequest(std::vector<ScraperSearchResult>& resultsWrite, const std::string& url) : ScraperHttpRequest(resultsWrite, url), mRequestQueue(nullptr) {}
|
|
||||||
|
|
||||||
protected:
|
|
||||||
void process(const std::unique_ptr<HttpReq>& req, std::vector<ScraperSearchResult>& results) override;
|
|
||||||
void processList(const pugi::xml_document& xmldoc, std::vector<ScraperSearchResult>& results);
|
|
||||||
void processGame(const pugi::xml_document& xmldoc, std::vector<ScraperSearchResult>& results);
|
|
||||||
bool isGameRequest() { return !mRequestQueue; }
|
|
||||||
|
|
||||||
std::queue< std::unique_ptr<ScraperRequest> >* mRequestQueue;
|
|
||||||
};
|
|
||||||
|
|
||||||
#endif // ES_APP_SCRAPERS_GAMES_DB_SCRAPER_H
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
#include "scrapers/Scraper.h"
|
#include "scrapers/Scraper.h"
|
||||||
|
|
||||||
#include "FileData.h"
|
#include "FileData.h"
|
||||||
#include "GamesDBScraper.h"
|
#include "GamesDBJSONScraper.h"
|
||||||
#include "ScreenScraper.h"
|
#include "ScreenScraper.h"
|
||||||
#include "Log.h"
|
#include "Log.h"
|
||||||
#include "Settings.h"
|
#include "Settings.h"
|
||||||
|
|
@ -10,7 +10,7 @@
|
||||||
#include <fstream>
|
#include <fstream>
|
||||||
|
|
||||||
const std::map<std::string, generate_scraper_requests_func> scraper_request_funcs {
|
const std::map<std::string, generate_scraper_requests_func> scraper_request_funcs {
|
||||||
// { "TheGamesDB", &thegamesdb_generate_scraper_requests },
|
{ "TheGamesDB", &thegamesdb_generate_json_scraper_requests },
|
||||||
{ "ScreenScraper", &screenscraper_generate_scraper_requests }
|
{ "ScreenScraper", &screenscraper_generate_scraper_requests }
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue