// // GamesDBJSONScraper.cpp // // Functions specifically for scraping from thegamesdb.net // Called from Scraper. // #include #include #include "scrapers/GamesDBJSONScraper.h" #include "scrapers/GamesDBJSONScraperResources.h" #include "FileData.h" #include "Log.h" #include "PlatformId.h" #include "Settings.h" #include "SystemData.h" #include "utils/TimeUtil.h" #include // When raspbian will get an up to date version of rapidjson we'll be // able to have it throw in case of error with the following: //ifndef RAPIDJSON_ASSERT //#define RAPIDJSON_ASSERT(x) \ // if (!(x)) { \ // throw std::runtime_error("rapidjson internal assertion failure: " #x); \ // } //#endif // RAPIDJSON_ASSERT #include #include using namespace PlatformIds; using namespace rapidjson; namespace { TheGamesDBJSONRequestResources resources; } const std::map gamesdb_new_platformid_map { { THREEDO, "25" }, { AMIGA, "4911" }, { AMSTRAD_CPC, "4914" }, { APPLE_II, "4942" }, { ARCADE, "23" }, { ATARI_800, "4943" }, { ATARI_2600, "22" }, { ATARI_5200, "26" }, { ATARI_7800, "27" }, { ATARI_JAGUAR, "28" }, { ATARI_JAGUAR_CD, "29" }, { ATARI_LYNX, "4924" }, { ATARI_ST, "4937" }, { ATARI_XE, "30" }, { COLECOVISION, "31" }, { COMMODORE_64, "40" }, { INTELLIVISION, "32" }, { MAC_OS, "37" }, { XBOX, "14" }, { XBOX_360, "15" }, { MSX, "4929" }, { NEOGEO, "24" }, { NEOGEO_POCKET, "4922" }, { NEOGEO_POCKET_COLOR, "4923" }, { NINTENDO_3DS, "4912" }, { NINTENDO_64, "3" }, { NINTENDO_DS, "8" }, { FAMICOM_DISK_SYSTEM, "4936" }, { NINTENDO_ENTERTAINMENT_SYSTEM, "7" }, { GAME_BOY, "4" }, { GAME_BOY_ADVANCE, "5" }, { GAME_BOY_COLOR, "41" }, { NINTENDO_GAMECUBE, "2" }, { NINTENDO_WII, "9" }, { NINTENDO_WII_U, "38" }, { NINTENDO_VIRTUAL_BOY, "4918" }, { NINTENDO_GAME_AND_WATCH, "-1" }, { PC, "1" }, { SEGA_32X, "33" }, { SEGA_CD, "21" }, { SEGA_DREAMCAST, "16" }, { SEGA_GAME_GEAR, "20" }, { SEGA_GENESIS, "18" }, { SEGA_MASTER_SYSTEM, "35" }, { SEGA_MEGA_DRIVE, "36" }, { SEGA_SATURN, "17" }, { SEGA_SG1000, "4949" }, { PLAYSTATION, "10" }, { PLAYSTATION_2, "11" }, { PLAYSTATION_3, "12" }, { PLAYSTATION_4, "4919" }, { PLAYSTATION_VITA, "39" }, { PLAYSTATION_PORTABLE, "13" }, { SUPER_NINTENDO, "6" }, { TURBOGRAFX_16, "34" }, // HuCards only. { TURBOGRAFX_CD, "4955" }, // CD-ROMs only. { WONDERSWAN, "4925" }, { WONDERSWAN_COLOR, "4926" }, { ZX_SPECTRUM, "4913" }, { VIDEOPAC_ODYSSEY2, "4927" }, { VECTREX, "4939" }, { TRS80_COLOR_COMPUTER, "4941" }, { TANDY, "4941" }, }; void thegamesdb_generate_json_scraper_requests( const ScraperSearchParams& params, std::queue>& requests, std::vector& results) { resources.prepare(); std::string path = "https://api.thegamesdb.net/v1"; bool usingGameID = false; const std::string apiKey = std::string("apikey=") + resources.getApiKey(); std::string cleanName = params.nameOverride; if (!cleanName.empty() && cleanName.substr(0, 3) == "id:") { std::string gameID = cleanName.substr(3); path += "/Games/ByGameID?" + apiKey + "&fields=players,publishers,genres,overview,last_updated,rating," "platform,coop,youtube,os,processor,ram,hdd,video,sound,alternates&" "include=boxart&id=" + HttpReq::urlEncode(gameID); usingGameID = true; } else { if (cleanName.empty()) cleanName = params.game->getCleanName(); path += "/Games/ByGameName?" + apiKey + "&fields=players,publishers,genres,overview,last_updated,rating," "platform,coop,youtube,os,processor,ram,hdd,video,sound,alternates&" "include=boxart&name=" + HttpReq::urlEncode(cleanName); } if (usingGameID) { // Ff we have the ID already, we don't need the GetGameList request. requests.push(std::unique_ptr(new TheGamesDBJSONRequest(results, path))); } else { std::string platformQueryParam; auto& platforms = params.system->getPlatformIds(); if (!platforms.empty()) { bool first = true; platformQueryParam += "&filter%5Bplatform%5D="; for (auto platformIt = platforms.cbegin(); platformIt != platforms.cend(); platformIt++) { auto mapIt = gamesdb_new_platformid_map.find(*platformIt); if (mapIt != gamesdb_new_platformid_map.cend()) { if (!first) platformQueryParam += ","; platformQueryParam += HttpReq::urlEncode(mapIt->second); first = false; } else { LOG(LogWarning) << "TheGamesDB scraper warning - no support for platform " << getPlatformName(*platformIt); } } path += platformQueryParam; } requests.push(std::unique_ptr (new TheGamesDBJSONRequest(requests, results, path))); } } namespace { std::string getStringOrThrow(const Value& v, const std::string& key) { if (!v.HasMember(key.c_str()) || !v[key.c_str()].IsString()) { throw std::runtime_error( "rapidjson internal assertion failure: missing or non string key:" + key); } return v[key.c_str()].GetString(); } int getIntOrThrow(const Value& v, const std::string& key) { if (!v.HasMember(key.c_str()) || !v[key.c_str()].IsInt()) { throw std::runtime_error( "rapidjson internal assertion failure: missing or non int key:" + key); } return v[key.c_str()].GetInt(); } int getIntOrThrow(const Value& v) { if (!v.IsInt()) { throw std::runtime_error("rapidjson internal assertion failure: not an int"); } return v.GetInt(); } std::string getBoxartImage(const Value& v) { if (!v.IsArray() || v.Size() == 0) return ""; for (int i = 0; i < (int)v.Size(); ++i) { auto& im = v[i]; std::string type = getStringOrThrow(im, "type"); std::string side = getStringOrThrow(im, "side"); if (type == "boxart" && side == "front") return getStringOrThrow(im, "filename"); } return getStringOrThrow(v[0], "filename"); } std::string getDeveloperString(const Value& v) { if (!v.IsArray()) return ""; std::string out = ""; bool first = true; for (int i = 0; i < (int)v.Size(); ++i) { auto mapIt = resources.gamesdb_new_developers_map.find(getIntOrThrow(v[i])); if (mapIt == resources.gamesdb_new_developers_map.cend()) continue; if (!first) out += ", "; out += mapIt->second; first = false; } return out; } std::string getPublisherString(const Value& v) { if (!v.IsArray()) return ""; std::string out = ""; bool first = true; for (int i = 0; i < (int)v.Size(); ++i) { auto mapIt = resources.gamesdb_new_publishers_map.find(getIntOrThrow(v[i])); if (mapIt == resources.gamesdb_new_publishers_map.cend()) continue; if (!first) out += ", "; out += mapIt->second; first = false; } return out; } std::string getGenreString(const Value& v) { if (!v.IsArray()) return ""; std::string out = ""; bool first = true; for (int i = 0; i < (int)v.Size(); ++i) { auto mapIt = resources.gamesdb_new_genres_map.find(getIntOrThrow(v[i])); if (mapIt == resources.gamesdb_new_genres_map.cend()) continue; if (!first) out += ", "; out += mapIt->second; first = false; } return out; } void processGame(const Value& game, const Value& boxart, std::vector& results) { std::string baseImageUrlThumb = getStringOrThrow(boxart["base_url"], "thumb"); std::string baseImageUrlLarge = getStringOrThrow(boxart["base_url"], "large"); ScraperSearchResult result; result.mdl.set("name", getStringOrThrow(game, "game_title")); if (game.HasMember("overview") && game["overview"].IsString()) result.mdl.set("desc", game["overview"].GetString()); if (game.HasMember("release_date") && game["release_date"].IsString()) result.mdl.set("releasedate", Utils::Time::DateTime(Utils::Time::stringToTime( game["release_date"].GetString(), "%Y-%m-%d"))); if (game.HasMember("developers") && game["developers"].IsArray()) result.mdl.set("developer", getDeveloperString(game["developers"])); if (game.HasMember("publishers") && game["publishers"].IsArray()) result.mdl.set("publisher", getPublisherString(game["publishers"])); if (game.HasMember("genres") && game["genres"].IsArray()) result.mdl.set("genre", getGenreString(game["genres"])); if (game.HasMember("players") && game["players"].IsInt()) result.mdl.set("players", std::to_string(game["players"].GetInt())); if (boxart.HasMember("data") && boxart["data"].IsObject()) { std::string id = std::to_string(getIntOrThrow(game, "id")); if (boxart["data"].HasMember(id.c_str())) { std::string image = getBoxartImage(boxart["data"][id.c_str()]); result.thumbnailUrl = baseImageUrlThumb + "/" + image; result.imageUrl = baseImageUrlLarge + "/" + image; } } results.push_back(result); } } // namespace void TheGamesDBJSONRequest::process(const std::unique_ptr& req, std::vector& results) { assert(req->status() == HttpReq::REQ_SUCCESS); Document doc; doc.Parse(req->getContent().c_str()); if (doc.HasParseError()) { std::string err = std::string("TheGamesDBJSONRequest - Error parsing JSON. \n\t") + GetParseError_En(doc.GetParseError()); setError(err); LOG(LogError) << err; return; } if (!doc.HasMember("data") || !doc["data"].HasMember("games") || !doc["data"]["games"].IsArray()) { std::string warn = "TheGamesDBJSONRequest - Response had no game data.\n"; LOG(LogWarning) << warn; return; } const Value& games = doc["data"]["games"]; if (!doc.HasMember("include") || !doc["include"].HasMember("boxart")) { std::string warn = "TheGamesDBJSONRequest - Response had no include boxart data.\n"; LOG(LogWarning) << warn; return; } const Value& boxart = doc["include"]["boxart"]; if (!boxart.HasMember("base_url") || !boxart.HasMember("data") || !boxart.IsObject()) { std::string warn = "TheGamesDBJSONRequest - Response include had no usable boxart data.\n"; LOG(LogWarning) << warn; return; } resources.ensureResources(); for (int i = 0; i < (int)games.Size(); ++i) { auto& v = games[i]; try { processGame(v, boxart, results); } catch (std::runtime_error& e) { LOG(LogError) << "Error while processing game: " << e.what(); } } }