Xargon/ES-DE
1
0
Fork 0
mirror of https://github.com/RetroDECK/ES-DE.git synced 2024-11-22 14:15:38 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #238 from mooware/scraper-use-getgameslist

Browse source 
TheGamesDB scrapper should use GetGameList.php
This commit is contained in:
Jools Wills 2017-10-13 20:15:08 +01:00 committed by GitHub
parent 2ee3d6313a d75510bde1
commit 3c60235363
4 changed files with 66 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
es-app/src/components/ScraperSearchComponent.cpp
View file

@ -231,11 +231,9 @@ void ScraperSearchComponent::onSearchDone(const std::vector<ScraperSearchResult>
mScraperResults = results;
const int end = results.size() > MAX_SCRAPER_RESULTS ? MAX_SCRAPER_RESULTS : results.size(); // at max display 5
auto font = Font::get(FONT_SIZE_MEDIUM);
unsigned int color = 0x777777FF;
if(end == 0)
if(results.empty())
{
ComponentListRow row;
row.addElement(std::make_shared<TextComponent>(mWindow, "NO GAMES FOUND - SKIP", font, color), true);
@ -247,7 +245,7 @@ void ScraperSearchComponent::onSearchDone(const std::vector<ScraperSearchResult>
mGrid.resetCursor();
}else{
ComponentListRow row;
for(int i = 0; i < end; i++)
for(size_t i = 0; i < results.size(); i++)
{
row.elements.clear();
row.addElement(std::make_shared<TextComponent>(mWindow, strToUpper(results.at(i).mdl.get("name")), font, color), true);

68
es-app/src/scrapers/GamesDBScraper.cpp
View file

@ -74,31 +74,28 @@ const std::map<PlatformId, const char*> gamesdb_platformid_map = boost::assign::
void thegamesdb_generate_scraper_requests(const ScraperSearchParams& params, std::queue< std::unique_ptr<ScraperRequest> >& requests,
std::vector<ScraperSearchResult>& results)
{
std::string path = "thegamesdb.net/api/GetGame.php?";
std::string path;
bool usingGameID = false;
std::string cleanName = params.nameOverride;
if (cleanName.empty())
if (!cleanName.empty() && cleanName.substr(0,3) == "id:")
{
cleanName = params.game->getCleanName();
path += "name=" + HttpReq::urlEncode(cleanName);
}
else
{
if (cleanName.substr(0,3) == "id:") {
std::string gameID = cleanName.substr(3,-1);
path += "id=" + HttpReq::urlEncode(gameID);
usingGameID = true;
}
else {
path += "exactname=" + HttpReq::urlEncode(cleanName);
}
std::string gameID = cleanName.substr(3);
path = "thegamesdb.net/api/GetGame.php?id=" + HttpReq::urlEncode(gameID);
usingGameID = true;
}else{
if (cleanName.empty())
cleanName = params.game->getCleanName();
path += "thegamesdb.net/api/GetGamesList.php?name=" + HttpReq::urlEncode(cleanName);
}
if(params.system->getPlatformIds().empty() || usingGameID)
if(usingGameID)
{
// no platform specified, we're done
// if we have the ID already, we don't need the GetGameList request
requests.push(std::unique_ptr<ScraperRequest>(new TheGamesDBRequest(results, path)));
}else if(params.system->getPlatformIds().empty()){
// no platform specified, we're done
requests.push(std::unique_ptr<ScraperRequest>(new TheGamesDBRequest(requests, results, path)));
}else{
// go through the list, we need to split this into multiple requests
// because TheGamesDB API either sucks or I don't know how to use it properly...
@ -116,7 +113,7 @@ void thegamesdb_generate_scraper_requests(const ScraperSearchParams& params, std
LOG(LogWarning) << "TheGamesDB scraper warning - no support for platform " << getPlatformName(*platformIt);
}
requests.push(std::unique_ptr<ScraperRequest>(new TheGamesDBRequest(results, path)));
requests.push(std::unique_ptr<ScraperRequest>(new TheGamesDBRequest(requests, results, path)));
}
}
}
@ -130,19 +127,27 @@ void TheGamesDBRequest::process(const std::unique_ptr<HttpReq>& req, std::vector
if(!parseResult)
{
std::stringstream ss;
ss << "GamesDBRequest - Error parsing XML. \n\t" << parseResult.description() << "";
ss << "TheGamesDBRequest - Error parsing XML. \n\t" << parseResult.description() << "";
std::string err = ss.str();
setError(err);
LOG(LogError) << err;
return;
}
pugi::xml_node data = doc.child("Data");
if (isGameRequest())
processGame(doc, results);
else
processList(doc, results);
}
void TheGamesDBRequest::processGame(const pugi::xml_document& xmldoc, std::vector<ScraperSearchResult>& results)
{
pugi::xml_node data = xmldoc.child("Data");
std::string baseImageUrl = data.child("baseImgUrl").text().get();
pugi::xml_node game = data.child("Game");
while(game && results.size() < MAX_SCRAPER_RESULTS)
if(game)
{
ScraperSearchResult result;
@ -179,6 +184,27 @@ void TheGamesDBRequest::process(const std::unique_ptr<HttpReq>& req, std::vector
}
results.push_back(result);
}
}
void TheGamesDBRequest::processList(const pugi::xml_document& xmldoc, std::vector<ScraperSearchResult>& results)
{
assert(mRequestQueue != nullptr);
pugi::xml_node data = xmldoc.child("Data");
pugi::xml_node game = data.child("Game");
// limit the number of results per platform, not in total.
// otherwise if the first platform returns >= 7 games
// but the second platform contains the relevant game,
// the relevant result would not be shown.
for(int i = 0; game && i < MAX_SCRAPER_RESULTS; i++)
{
std::string id = game.child("id").text().get();
std::string path = "thegamesdb.net/api/GetGame.php?id=" + id;
mRequestQueue->push(std::unique_ptr<ScraperRequest>(new TheGamesDBRequest(results, path)));
game = game.next_sibling("Game");
}
}

13
es-app/src/scrapers/GamesDBScraper.h
View file

@ -2,13 +2,24 @@
#include "scrapers/Scraper.h"
namespace pugi { class xml_document; }
void thegamesdb_generate_scraper_requests(const ScraperSearchParams& params, std::queue< std::unique_ptr<ScraperRequest> >& requests,
std::vector<ScraperSearchResult>& results);
class TheGamesDBRequest : public ScraperHttpRequest
{
public:
TheGamesDBRequest(std::vector<ScraperSearchResult>& resultsWrite, const std::string& url) : ScraperHttpRequest(resultsWrite, url) {}
// ctor for a GetGameList request
TheGamesDBRequest(std::queue< std::unique_ptr<ScraperRequest> >& requestsWrite, std::vector<ScraperSearchResult>& resultsWrite, const std::string& url) : ScraperHttpRequest(resultsWrite, url), mRequestQueue(&requestsWrite) {}
// ctor for a GetGame request
TheGamesDBRequest(std::vector<ScraperSearchResult>& resultsWrite, const std::string& url) : ScraperHttpRequest(resultsWrite, url), mRequestQueue(nullptr) {}
protected:
void process(const std::unique_ptr<HttpReq>& req, std::vector<ScraperSearchResult>& results) override;
void processList(const pugi::xml_document& xmldoc, std::vector<ScraperSearchResult>& results);
void processGame(const pugi::xml_document& xmldoc, std::vector<ScraperSearchResult>& results);
bool isGameRequest() { return !mRequestQueue; }
std::queue< std::unique_ptr<ScraperRequest> >* mRequestQueue;
};

8
es-app/src/scrapers/Scraper.cpp
View file

@ -43,13 +43,15 @@ void ScraperSearchHandle::update()
if(!mRequestQueue.empty())
{
auto& req = mRequestQueue.front();
AsyncHandleStatus status = req->status();
// a request can add more requests to the queue while running,
// so be careful with references into the queue
auto& req = *(mRequestQueue.front());
AsyncHandleStatus status = req.status();
if(status == ASYNC_ERROR)
{
// propegate error
setError(req->getStatusString());
setError(req.getStatusString());
// empty our queue
while(!mRequestQueue.empty())