ES-DE/es-app/src/scrapers/ScreenScraper.cpp

//
//	ScreenScraper.cpp
//
//	Functions specifically for scraping from screenscraper.fr
//	Called from Scraper.
//

#include "scrapers/ScreenScraper.h"

#include "utils/TimeUtil.h"
#include "utils/StringUtil.h"
#include "FileData.h"
#include "Log.h"
#include "PlatformId.h"
#include "Settings.h"
#include "SystemData.h"
#include "math/Misc.h"
#include <pugixml/src/pugixml.hpp>
#include <cstring>

using namespace PlatformIds;

// List of systems and their IDs from:
// https://www.screenscraper.fr/api/systemesListe.php?devid=xxx&devpassword=yyy&softname=zzz&output=XML
const std::map<PlatformId, unsigned short> screenscraper_platformid_map {
	{ THREEDO, 29 },
	{ AMIGA, 64 },
	{ AMSTRAD_CPC, 65 },
	{ APPLE_II, 86 },
	{ ARCADE, 75 },
	{ ATARI_800, 26 }, // Use ATARI_2600 as an alias for atari 800.
	{ ATARI_2600, 26 },
	{ ATARI_5200, 40 },
	{ ATARI_7800, 41 },
	{ ATARI_JAGUAR, 27 },
	{ ATARI_JAGUAR_CD, 171 },
	{ ATARI_LYNX, 28 },
	{ ATARI_ST, 42},
	// Missing Atari XE ?
	{ COLECOVISION, 48 },
	{ COMMODORE_64, 66 },
	{ INTELLIVISION, 115 },
	{ MAC_OS, 146 },
	{ XBOX, 32 },
	{ XBOX_360, 33 },
	{ MSX, 113 },
	{ NEOGEO, 142 },
	{ NEOGEO_POCKET, 25},
	{ NEOGEO_POCKET_COLOR, 82 },
	{ NINTENDO_3DS, 17 },
	{ NINTENDO_64, 14 },
	{ NINTENDO_DS, 15 },
	{ FAMICOM_DISK_SYSTEM, 106 },
	{ NINTENDO_ENTERTAINMENT_SYSTEM, 3 },
	{ FAIRCHILD_CHANNELF, 80 },
	{ GAME_BOY, 9 },
	{ GAME_BOY_ADVANCE, 12 },
	{ GAME_BOY_COLOR, 10 },
	{ NINTENDO_GAMECUBE, 13 },
	{ NINTENDO_WII, 16 },
	{ NINTENDO_WII_U, 18 },
	{ NINTENDO_VIRTUAL_BOY, 11 },
	{ NINTENDO_GAME_AND_WATCH, 52 },
	{ PC, 135 },
	{ OPENBOR, 214 },
	{ SCUMMVM, 123},
	{ SEGA_32X, 19 },
	{ SEGA_CD, 20 },
	{ SEGA_DREAMCAST, 23 },
	{ SEGA_GAME_GEAR, 21 },
	{ SEGA_GENESIS, 1 },
	{ SEGA_MASTER_SYSTEM, 2 },
	{ SEGA_MEGA_DRIVE, 1 },
	{ SEGA_SATURN, 22 },
	{ SEGA_SG1000, 109 },
	{ SHARP_X6800, 79},
	{ SOLARUS, 223 },
	{ PLAYSTATION, 57 },
	{ PLAYSTATION_2, 58 },
	{ PLAYSTATION_3, 59 },
	// Missing Sony Playstation 4 ?
	{ PLAYSTATION_VITA, 62 },
	{ PLAYSTATION_PORTABLE, 61 },
	{ SUPER_NINTENDO, 4 },
	{ TURBOGRAFX_16, 31 },
	{ TURBOGRAFX_CD, 114 },
	{ WONDERSWAN, 45 },
	{ WONDERSWAN_COLOR, 46 },
	{ ZX_SPECTRUM, 76 },
	{ ZX81_SINCLAR, 77 },
	{ VIDEOPAC_ODYSSEY2, 104 },
	{ VECTREX, 102 },
	{ TRS80_COLOR_COMPUTER, 144 },
	{ TANDY, 144 }
};

// Helper XML parsing method, finding a node-by-name recursively.
pugi::xml_node find_node_by_name_re(const pugi::xml_node& node,
		const std::vector<std::string> node_names) {

	for (const std::string& _val : node_names) {
		pugi::xpath_query query_node_name((static_cast<std::string>("//") + _val).c_str());
		pugi::xpath_node_set results = node.select_nodes(query_node_name);

		if (results.size() > 0)
			return results.first().node();
	}

	return pugi::xml_node();
}

// Help XML parsing method, finding an direct child XML node starting from the parent and
// filtering by an attribute value list.
pugi::xml_node find_child_by_attribute_list(const pugi::xml_node& node_parent,
		const std::string& node_name, const std::string& attribute_name,
		const std::vector<std::string> attribute_values)
{
	for (auto _val : attribute_values) {
		for (pugi::xml_node node : node_parent.children(node_name.c_str())) {
			if (strcmp(node.attribute(attribute_name.c_str()).value(), _val.c_str()) == 0)
				return node;
		}
	}

	return pugi::xml_node(NULL);
}

void screenscraper_generate_scraper_requests(const ScraperSearchParams& params,
		std::queue< std::unique_ptr<ScraperRequest> >& requests,
		std::vector<ScraperSearchResult>& results)
{
	std::string path;

	ScreenScraperRequest::ScreenScraperConfig ssConfig;

	if (params.nameOverride == "")
		path = ssConfig.getGameSearchUrl(params.game->getCleanName());
	else
		path = ssConfig.getGameSearchUrl(params.nameOverride);

	auto& platforms = params.system->getPlatformIds();
	std::vector<unsigned short> p_ids;

	// Get the IDs of each platform from the ScreenScraper list.
	for (auto platformIt = platforms.cbegin(); platformIt != platforms.cend(); platformIt++) {
		auto mapIt = screenscraper_platformid_map.find(*platformIt);

		if (mapIt != screenscraper_platformid_map.cend()) {
			p_ids.push_back(mapIt->second);
		}
		else {
			LOG(LogWarning) << "ScreenScraper: no support for platform " <<
					getPlatformName(*platformIt);
			// Add the scrape request without a platform/system ID.
			requests.push(std::unique_ptr<ScraperRequest>
					(new ScreenScraperRequest(requests, results, path)));
		}
	}

	// Sort the platform IDs and remove duplicates.
	std::sort(p_ids.begin(), p_ids.end());
	auto last = std::unique(p_ids.begin(), p_ids.end());
	p_ids.erase(last, p_ids.end());

	for (auto platform = p_ids.cbegin(); platform != p_ids.cend(); platform++)
	{
		path += "&systemeid=";
		path += HttpReq::urlEncode(std::to_string(*platform));
		requests.push(std::unique_ptr<ScraperRequest>
				(new ScreenScraperRequest(requests, results, path)));
	}
}

void ScreenScraperRequest::process(const std::unique_ptr<HttpReq>& req,
		std::vector<ScraperSearchResult>& results)
{
	assert(req->status() == HttpReq::REQ_SUCCESS);

	pugi::xml_document doc;
	pugi::xml_parse_result parseResult = doc.load_string(req->getContent().c_str());

	if (!parseResult) {
		std::stringstream ss;
		ss << "ScreenScraperRequest - Error parsing XML." << std::endl <<
				parseResult.description() << "";

		std::string err = ss.str();
		setError(err);
		LOG(LogError) << err;

		return;
	}

	processGame(doc, results);
}

void ScreenScraperRequest::processGame(const pugi::xml_document& xmldoc,
		std::vector<ScraperSearchResult>& out_results)
{
	pugi::xml_node data = xmldoc.child("Data");
	pugi::xml_node game = data.child("jeu");

	if (game) {
		ScraperSearchResult result;
		ScreenScraperRequest::ScreenScraperConfig ssConfig;

		result.gameID = game.attribute("id").as_string();

		// Find how many more requests we can make before the scraper request
		// allowance counter is reset.
		unsigned requestsToday =
				data.child("ssuser").child("requeststoday").text().as_uint();
		unsigned maxRequestsPerDay =
				data.child("ssuser").child("maxrequestsperday").text().as_uint();
		result.scraperRequestAllowance = maxRequestsPerDay - requestsToday;

		std::string region =
				Utils::String::toLower(Settings::getInstance()->getString("ScraperRegion"));
		std::string language =
				Utils::String::toLower(Settings::getInstance()->getString("ScraperLanguage"));

		// Name fallback: US, WOR(LD). ( Xpath: Data/jeu[0]/noms/nom[*] ).
		result.mdl.set("name", find_child_by_attribute_list(game.child("noms"),
				"nom", "region", { region, "wor", "us" , "ss", "eu", "jp" }).text().get());

		// Description fallback language: EN, WOR(LD).
		std::string description = find_child_by_attribute_list(game.child("synopsis"),
				"synopsis", "langue", { language, "en", "wor" }).text().get();

		if (!description.empty())
			result.mdl.set("desc", Utils::String::replace(description, "&nbsp;", " "));

		// Genre fallback language: EN. ( Xpath: Data/jeu[0]/genres/genre[*] ).
		result.mdl.set("genre", find_child_by_attribute_list(game.child("genres"),
				"genre", "langue", { language, "en" }).text().get());
		LOG(LogDebug) << "Genre: " << result.mdl.get("genre");

		// Get the date proper. The API returns multiple 'date' children nodes to the 'dates'
		// main child of 'jeu'.
		// Date fallback: WOR(LD), US, SS, JP, EU.
		std::string _date = find_child_by_attribute_list(game.child("dates"), "date", "region",
				{ region, "wor", "us", "ss", "jp", "eu" }).text().get();
		LOG(LogDebug) << "Release Date (unparsed): " << _date;

		// Date can be YYYY-MM-DD or just YYYY.
		if (_date.length() > 4) {
			result.mdl.set("releasedate", Utils::Time::DateTime(
					Utils::Time::stringToTime(_date, "%Y-%m-%d")));
		}
		else if (_date.length() > 0) {
			result.mdl.set("releasedate", Utils::Time::DateTime(
					Utils::Time::stringToTime(_date, "%Y")));
		}

		LOG(LogDebug) << "Release Date (parsed): " << result.mdl.get("releasedate");

		/// Developer for the game( Xpath: Data/jeu[0]/developpeur ).
		std::string developer = game.child("developpeur").text().get();
		if (!developer.empty())
			result.mdl.set("developer", Utils::String::replace(developer, "&nbsp;", " "));

		// Publisher for the game ( Xpath: Data/jeu[0]/editeur ).
		std::string publisher = game.child("editeur").text().get();
		if (!publisher.empty())
			result.mdl.set("publisher", Utils::String::replace(publisher, "&nbsp;", " "));

		// Players.
		result.mdl.set("players", game.child("joueurs").text().get());

		// Validate rating.
		if (Settings::getInstance()->getBool("ScrapeRatings") && game.child("note")) {
			float ratingVal = (game.child("note").text().as_int() / 20.0f);
			// Round up to the closest .1 value, i.e. to the closest half-star.
			ratingVal = Math::ceilf(ratingVal / 0.1) / 10;
			std::stringstream ss;
			ss << ratingVal;
			result.mdl.set("rating", ss.str());
		}

		// Media super-node.
		pugi::xml_node media_list = game.child("medias");

		if (media_list) {
			// 3D box
			processMedia(result, media_list, ssConfig.media_3dbox,
					result.box3dUrl, result.box3dFormat, region);
			// Cover
			processMedia(result, media_list, ssConfig.media_cover,
					result.coverUrl, result.coverFormat, region);
			// Marquee (wheel)
			processMedia(result, media_list, ssConfig.media_marquee,
					result.marqueeUrl, result.marqueeFormat, region);
			// Screenshot
			processMedia(result, media_list, ssConfig.media_screenshot,
					result.screenshotUrl, result.screenshotFormat, region);
		}
		result.mediaURLFetch = COMPLETED;
		out_results.push_back(result);
	} // Game.
}

void ScreenScraperRequest::processMedia(
		ScraperSearchResult& result,
		const pugi::xml_node& media_list,
		std::string mediaType,
		std::string& fileURL,
		std::string& fileFormat,
		std::string region)
{
	pugi::xml_node art = pugi::xml_node(NULL);

	// Do an XPath query for media[type='$media_type'], then filter by region.
	// We need to do this because any child of 'medias' has the form
	// <media type="..." region="..." format="...">
	// and we need to find the right media for the region.
	pugi::xpath_node_set results = media_list.select_nodes((static_cast<std::string>
		("media[@type='") + mediaType + "']").c_str());

		if (results.size()) {
			// Region fallback: WOR(LD), US, CUS(TOM?), JP, EU.
			for (auto _region : std::vector<std::string>{
					region, "wor", "us", "cus", "jp", "eu" }) {
				if (art)
					break;

				for (auto node : results) {
					if (node.node().attribute("region").value() == _region) {
						art = node.node();
						break;
					}
				}
			}
		}

		if (art) {
			// Sending a 'softname' containing space will make the image URLs returned
			// by the API also contain the space. Escape any spaces in the URL here.
			fileURL = Utils::String::replace(art.text().get(), " ", "%20");

			// Get the media type returned by ScreenScraper.
			std::string media_type = art.attribute("format").value();
			if (!media_type.empty())
				fileFormat = "." + media_type;
		}
		else {
			LOG(LogDebug) << "Failed to find media XML node with name=" << mediaType;
		}
}

// Currently not used in this module.
void ScreenScraperRequest::processList(const pugi::xml_document& xmldoc,
		std::vector<ScraperSearchResult>& results)
{
	assert(mRequestQueue != nullptr);

	LOG(LogDebug) << "Processing a list of results";

	pugi::xml_node data = xmldoc.child("Data");
	pugi::xml_node game = data.child("jeu");

	if (!game)
		LOG(LogDebug) << "Found nothing";

	ScreenScraperRequest::ScreenScraperConfig ssConfig;

	// Limit the number of results per platform, not in total.
	// Otherwise if the first platform returns >= 7 games
	// but the second platform contains the relevant game,
	// the relevant result would not be shown.
	for (int i = 0; game && i < MAX_SCRAPER_RESULTS; i++) {
		std::string id = game.child("id").text().get();
		std::string name = game.child("nom").text().get();
		std::string platformId = game.child("systemeid").text().get();
		std::string path = ssConfig.getGameSearchUrl(name) + "&systemeid=" +
				platformId + "&gameid=" + id;

		mRequestQueue->push(std::unique_ptr<ScraperRequest>
				(new ScreenScraperRequest(results, path)));

		game = game.next_sibling("jeu");
	}
}

std::string ScreenScraperRequest::ScreenScraperConfig::getGameSearchUrl(
		const std::string gameName) const
{
	return API_URL_BASE
		+ "/jeuInfos.php?devid=" + Utils::String::scramble(API_DEV_U, API_DEV_KEY)
		+ "&devpassword=" + Utils::String::scramble(API_DEV_P, API_DEV_KEY)
		+ "&softname=" + HttpReq::urlEncode(API_SOFT_NAME)
		+ "&output=xml"
		+ "&romnom=" + HttpReq::urlEncode(gameName);
}