Xargon/ES-DE
1
0
Fork 0
mirror of https://github.com/RetroDECK/ES-DE.git synced 2025-02-16 12:05:38 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Added automatic retries to the scraper for ScreenScraper TLS errors.

Browse source
This commit is contained in:
Leon Styhre 2021-05-24 18:51:16 +02:00
parent 9b395d6526
commit 0d8f50f632
7 changed files with 75 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
es-app/src/guis/GuiScraperMenu.cpp
View file

@ -456,7 +456,7 @@ void GuiScraperMenu::openOtherSettings()
// Respect the per-file multi-scraper exclusion flag.
auto scraper_respect_exclusions = std::make_shared<SwitchComponent>(mWindow);
scraper_respect_exclusions->setState(
Settings::getInstance()->getBool("ScraperRespectExclusions"));
Settings::getInstance()->getBool("ScraperRespectExclusions"));
s->addWithLabel("RESPECT PER-FILE SCRAPER EXCLUSIONS", scraper_respect_exclusions);
s->addSaveFunc([scraper_respect_exclusions, s] {
if (scraper_respect_exclusions->getState() !=
@ -470,7 +470,7 @@ void GuiScraperMenu::openOtherSettings()
// Exclude files recursively for excluded folders.
auto scraper_exclude_recursively = std::make_shared<SwitchComponent>(mWindow);
scraper_exclude_recursively->setState(
Settings::getInstance()->getBool("ScraperExcludeRecursively"));
Settings::getInstance()->getBool("ScraperExcludeRecursively"));
s->addWithLabel("EXCLUDE FOLDERS RECURSIVELY", scraper_exclude_recursively);
s->addSaveFunc([scraper_exclude_recursively, s] {
if (scraper_exclude_recursively->getState() !=
@ -492,7 +492,7 @@ void GuiScraperMenu::openOtherSettings()
// Include actual folders when scraping.
auto scraper_include_folders = std::make_shared<SwitchComponent>(mWindow);
scraper_include_folders->setState(
Settings::getInstance()->getBool("ScraperIncludeFolders"));
Settings::getInstance()->getBool("ScraperIncludeFolders"));
s->addWithLabel("SCRAPE ACTUAL FOLDERS", scraper_include_folders);
s->addSaveFunc([scraper_include_folders, s] {
if (scraper_include_folders->getState() !=
@ -503,6 +503,20 @@ void GuiScraperMenu::openOtherSettings()
}
});
// Retry search on peer verification errors (TLS/certificate issues).
auto retry_peer_verification = std::make_shared<SwitchComponent>(mWindow);
retry_peer_verification->setState(
Settings::getInstance()->getBool("ScraperRetryPeerVerification"));
s->addWithLabel("RETRY ON PEER VERIFICATION ERRORS", retry_peer_verification);
s->addSaveFunc([retry_peer_verification, s] {
if (retry_peer_verification->getState() !=
Settings::getInstance()->getBool("ScraperRetryPeerVerification")) {
Settings::getInstance()->setBool("ScraperRetryPeerVerification",
retry_peer_verification->getState());
s->setNeedsSaving();
}
});
// Switch callbacks.
auto interactiveToggleFunc = [scraper_semiautomatic]() {
if (scraper_semiautomatic->getEnabled()) {

46
es-app/src/guis/GuiScraperSearch.cpp
View file

@ -34,6 +34,8 @@
#include "SystemData.h"
#include "Window.h"
#define FAILED_VERIFICATION_RETRIES 5
GuiScraperSearch::GuiScraperSearch(
Window* window,
SearchType type,
@ -50,6 +52,8 @@ GuiScraperSearch::GuiScraperSearch(
addChild(&mGrid);
mBlockAccept = false;
mRetrySearch = false;
mRetryCount = 0;
// Left spacer (empty component, needed for borders).
mGrid.setEntry(std::make_shared<GuiComponent>(mWindow), Vector2i(0, 0),
@ -387,19 +391,38 @@ void GuiScraperSearch::onSearchDone(const std::vector<ScraperSearchResult>& resu
}
}
void GuiScraperSearch::onSearchError(const std::string& error)
void GuiScraperSearch::onSearchError(const std::string& error, HttpReq::Status status)
{
// This is a workaround for a somehow frequently recurring issue with screenscraper.fr
// where requests to download the thumbnails are randomly met with TLS verification errors.
// It's unclear why it only happens to the thumbnail requests, but it usually goes away
// after a few days or so. If this issue occurs and the corresponding setting has been
// enabled, we'll retry the search automatically up to FAILED_VERIFICATION_RETRIES number
// of times. Usually a few retries is enough to get the thumbnail to download. If not,
// the error dialog will be presented to the user, and if the "Retry" button is pressed,
// a new round of retries will take place.
if (status == HttpReq::REQ_FAILED_VERIFICATION && mRetryCount < FAILED_VERIFICATION_RETRIES &&
Settings::getInstance()->getBool("ScraperRetryPeerVerification")) {
LOG(LogError) << "GuiScraperSearch: " << Utils::String::replace(error, "\n", "");
mRetrySearch = true;
mRetryCount++;
LOG(LogError) << "GuiScraperSearch: Attempting automatic retry " << mRetryCount <<
" of " << FAILED_VERIFICATION_RETRIES;
return;
}
else {
mRetryCount = 0;
}
if (mScrapeCount > 1) {
LOG(LogError) << "GuiScraperSearch search error: " <<
Utils::String::replace(error, "\n", "");
LOG(LogError) << "GuiScraperSearch: " << Utils::String::replace(error, "\n", "");
mWindow->pushGui(new GuiMsgBox(mWindow, getHelpStyle(), Utils::String::toUpper(error),
"RETRY", std::bind(&GuiScraperSearch::search, this, mLastSearch),
"SKIP", mSkipCallback,
"CANCEL", mCancelCallback, true));
}
else {
LOG(LogError) << "GuiScraperSearch search error: " <<
Utils::String::replace(error, "\n", "");
LOG(LogError) << "GuiScraperSearch: " << Utils::String::replace(error, "\n", "");
mWindow->pushGui(new GuiMsgBox(mWindow, getHelpStyle(), Utils::String::toUpper(error),
"RETRY", std::bind(&GuiScraperSearch::search, this, mLastSearch),
"CANCEL", mCancelCallback, "", nullptr, true));
@ -532,12 +555,21 @@ void GuiScraperSearch::returnResult(ScraperSearchResult result)
mScrapeCount -= 1;
mAcceptCallback(result);
mRefinedSearch = false;
mRetryCount = 0;
}
void GuiScraperSearch::update(int deltaTime)
{
GuiComponent::update(deltaTime);
// There was a failure and we're attempting an automatic retry.
if (mRetrySearch) {
mRetrySearch = false;
stop();
search(mLastSearch);
return;
}
if (mBlockAccept)
mBusyAnim.update(deltaTime);
@ -647,7 +679,8 @@ void GuiScraperSearch::updateThumbnail()
}
else {
mResultThumbnail->setImage("");
onSearchError("Error downloading thumbnail:\n " + it->second->getErrorMsg());
onSearchError("Error downloading thumbnail:\n " + it->second->getErrorMsg(),
it->second->status());
}
mThumbnailReqMap.erase(it);
@ -676,6 +709,7 @@ void GuiScraperSearch::openInputScreen(ScraperSearchParams& params)
};
stop();
mRetryCount = 0;
std::string searchString;

6
es-app/src/guis/GuiScraperSearch.h
View file

@ -76,7 +76,8 @@ private:
void resizeMetadata();
void onSearchError(const std::string& error);
void onSearchError(const std::string& error, HttpReq::Status status =
HttpReq::REQ_UNDEFINED_ERROR);
void onSearchDone(const std::vector<ScraperSearchResult>& results);
int getSelectedIndex();
@ -129,6 +130,9 @@ private:
bool mFoundGame;
bool mScrapeRatings;
bool mRetrySearch;
unsigned int mRetryCount;
std::unique_ptr<ScraperSearchHandle> mSearchHandle;
std::unique_ptr<ScraperSearchHandle> mMDRetrieveURLsHandle;
std::unique_ptr<MDResolveHandle> mMDResolveHandle;

4
es-core/src/HttpReq.cpp
View file

@ -173,6 +173,10 @@ HttpReq::Status HttpReq::status()
if (msg->data.result == CURLE_OK) {
req->mStatus = REQ_SUCCESS;
}
else if (msg->data.result == CURLE_PEER_FAILED_VERIFICATION) {
req->mStatus = REQ_FAILED_VERIFICATION;
req->onError(curl_easy_strerror(msg->data.result));
}
else {
req->mStatus = REQ_IO_ERROR;
req->onError(curl_easy_strerror(msg->data.result));

12
es-core/src/HttpReq.h
View file

@ -43,11 +43,13 @@ public:
~HttpReq();
enum Status {
REQ_IN_PROGRESS, // Request is in progress.
REQ_SUCCESS, // Request completed successfully, get it with getContent().
REQ_IO_ERROR, // Some error happened, get it with getErrorMsg().
REQ_BAD_STATUS_CODE, // Some invalid HTTP response status code happened (non-200).
REQ_INVALID_RESPONSE // The HTTP response was invalid.
REQ_IN_PROGRESS, // Request is in progress.
REQ_SUCCESS, // Request completed successfully, get it with getContent().
REQ_IO_ERROR, // Some error happened, get it with getErrorMsg().
REQ_FAILED_VERIFICATION, // Peer's certificate or fingerprint wasn't verified correctly.
REQ_BAD_STATUS_CODE, // Some invalid HTTP response status code happened (non-200).
REQ_INVALID_RESPONSE, // The HTTP response was invalid.
REQ_UNDEFINED_ERROR
};
Status status(); // Process any received data and return the status afterwards.

1
es-core/src/Settings.cpp
View file

@ -116,6 +116,7 @@ void Settings::setDefaults()
mBoolMap["ScraperRespectExclusions"] = { true, true };
mBoolMap["ScraperExcludeRecursively"] = { true, true };
mBoolMap["ScraperIncludeFolders"] = { false, false };
mBoolMap["ScraperRetryPeerVerification"] = { false, false };
// UI settings.
mStringMap["StartupSystem"] = { "", "" };

3
es-core/src/components/ComponentGrid.cpp
View file

@ -355,8 +355,7 @@ void ComponentGrid::update(int deltaTime)
{
// Update everything.
const GridEntry* cursorEntry = getCellAt(mCursor);
for (auto it = mCells.cbegin(); it != mCells.cend(); it++)
{
for (auto it = mCells.cbegin(); it != mCells.cend(); it++) {
if (it->updateType == UPDATE_ALWAYS ||
(it->updateType == UPDATE_WHEN_SELECTED && cursorEntry == &(*it)))
it->component->update(deltaTime);