Xargon/ES-DE
1
0
Fork 0
mirror of https://github.com/RetroDECK/ES-DE.git synced 2025-01-17 22:55:38 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #436 from RetroPie/scraper-fix

Browse source 
set curl to follow redirects to fix scraper error
This commit is contained in:
John Rassa 2018-06-06 14:47:41 -07:00 committed by GitHub
parent e64277923a 86a2cb7840
commit c6a8a4b20c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 30 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
es-app/src/scrapers/GamesDBScraper.cpp
View file

@ -84,12 +84,12 @@ void thegamesdb_generate_scraper_requests(const ScraperSearchParams& params, std
if (!cleanName.empty() && cleanName.substr(0,3) == "id:") if (!cleanName.empty() && cleanName.substr(0,3) == "id:")
{ {
std::string gameID = cleanName.substr(3); std::string gameID = cleanName.substr(3);
path = "thegamesdb.net/api/GetGame.php?id=" + HttpReq::urlEncode(gameID); path = "legacy.thegamesdb.net/api/GetGame.php?id=" + HttpReq::urlEncode(gameID);
usingGameID = true; usingGameID = true;
}else{ }else{
if (cleanName.empty()) if (cleanName.empty())
cleanName = params.game->getCleanName(); cleanName = params.game->getCleanName();
path += "thegamesdb.net/api/GetGamesList.php?name=" + HttpReq::urlEncode(cleanName); path += "legacy.thegamesdb.net/api/GetGamesList.php?name=" + HttpReq::urlEncode(cleanName);
} }
if(usingGameID) if(usingGameID)
@ -201,7 +201,7 @@ void TheGamesDBRequest::processList(const pugi::xml_document& xmldoc, std::vecto
for(int i = 0; game && i < MAX_SCRAPER_RESULTS; i++) for(int i = 0; game && i < MAX_SCRAPER_RESULTS; i++)
{ {
std::string id = game.child("id").text().get(); std::string id = game.child("id").text().get();
std::string path = "thegamesdb.net/api/GetGame.php?id=" + id; std::string path = "legacy.thegamesdb.net/api/GetGame.php?id=" + id;
mRequestQueue->push(std::unique_ptr<ScraperRequest>(new TheGamesDBRequest(results, path))); mRequestQueue->push(std::unique_ptr<ScraperRequest>(new TheGamesDBRequest(results, path)));

27
es-core/src/HttpReq.cpp
View file

@ -58,6 +58,33 @@ HttpReq::HttpReq(const std::string& url)
return; return;
} }
//set curl to handle redirects
err = curl_easy_setopt(mHandle, CURLOPT_FOLLOWLOCATION, 1L);
if(err != CURLE_OK)
{
mStatus = REQ_IO_ERROR;
onError(curl_easy_strerror(err));
return;
}
//set curl max redirects
err = curl_easy_setopt(mHandle, CURLOPT_MAXREDIRS, 2L);
if(err != CURLE_OK)
{
mStatus = REQ_IO_ERROR;
onError(curl_easy_strerror(err));
return;
}
//set curl restrict redirect protocols
err = curl_easy_setopt(mHandle, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
if(err != CURLE_OK)
{
mStatus = REQ_IO_ERROR;
onError(curl_easy_strerror(err));
return;
}
//tell curl how to write the data //tell curl how to write the data
err = curl_easy_setopt(mHandle, CURLOPT_WRITEFUNCTION, &HttpReq::write_content); err = curl_easy_setopt(mHandle, CURLOPT_WRITEFUNCTION, &HttpReq::write_content);
if(err != CURLE_OK) if(err != CURLE_OK)