From 0535f0d3338e7d3d590f8439b0e309836cfb5fc7 Mon Sep 17 00:00:00 2001 From: Leon Styhre Date: Thu, 11 Jan 2024 01:02:35 +0100 Subject: [PATCH] Removed the ScraperHaltOnInvalidMedia option and corresponding menu entry --- es-app/src/guis/GuiScraperMenu.cpp | 14 -------- es-app/src/scrapers/Scraper.cpp | 57 ------------------------------ es-core/src/Settings.cpp | 1 - 3 files changed, 72 deletions(-) diff --git a/es-app/src/guis/GuiScraperMenu.cpp b/es-app/src/guis/GuiScraperMenu.cpp index 14d120e4f..5d3974bf4 100644 --- a/es-app/src/guis/GuiScraperMenu.cpp +++ b/es-app/src/guis/GuiScraperMenu.cpp @@ -950,20 +950,6 @@ void GuiScraperMenu::openOtherOptions() } }); - // Halt scraping on invalid media files. - auto scraperHaltOnInvalidMedia = std::make_shared(); - scraperHaltOnInvalidMedia->setState( - Settings::getInstance()->getBool("ScraperHaltOnInvalidMedia")); - s->addWithLabel("HALT ON INVALID MEDIA FILES", scraperHaltOnInvalidMedia); - s->addSaveFunc([scraperHaltOnInvalidMedia, s] { - if (scraperHaltOnInvalidMedia->getState() != - Settings::getInstance()->getBool("ScraperHaltOnInvalidMedia")) { - Settings::getInstance()->setBool("ScraperHaltOnInvalidMedia", - scraperHaltOnInvalidMedia->getState()); - s->setNeedsSaving(); - } - }); - // Search using file hashes for non-interactive mode. auto scraperSearchFileHash = std::make_shared(); scraperSearchFileHash->setState(Settings::getInstance()->getBool("ScraperSearchFileHash")); diff --git a/es-app/src/scrapers/Scraper.cpp b/es-app/src/scrapers/Scraper.cpp index ad5d590de..ce9367b20 100644 --- a/es-app/src/scrapers/Scraper.cpp +++ b/es-app/src/scrapers/Scraper.cpp @@ -315,34 +315,6 @@ MDResolveHandle::MDResolveHandle(const ScraperSearchResult& result, // If the image is cached already as the thumbnail, then we don't need // to download it again, in this case just save it to disk and resize it. if (mResult.thumbnailImageUrl == it->fileURL && mResult.thumbnailImageData.size() > 0) { - // This is just a temporary workaround to avoid saving media files to disk that - // are actually just containing error messages from the scraper service. The - // proper solution is to implement file checksum checks to determine if the - // server response contains valid media. As for the current approach, if the - // file is less than 350 bytes, we check if FreeImage can actually detect a - // valid format, and if not, we present an error message. Black/empty images - // are sometimes returned from the scraper service and these can actually be - // less than 350 bytes in size. - if (Settings::getInstance()->getBool("ScraperHaltOnInvalidMedia") && - mResult.thumbnailImageData.size() < 350) { - - FIMEMORY* memoryStream { - FreeImage_OpenMemory(reinterpret_cast(&mResult.thumbnailImageData.at(0)), - static_cast(mResult.thumbnailImageData.size()))}; - - FREE_IMAGE_FORMAT imageFormat {FreeImage_GetFileTypeFromMemory(memoryStream, 0)}; - FreeImage_CloseMemory(memoryStream); - - if (imageFormat == FIF_UNKNOWN) { - setError( - "The file \"" + Utils::FileSystem::getFileName(filePath) + - "\" returned by the scraper seems to be invalid as it's less than " + - "350 bytes in size", - true); - return; - } - } - // Remove any existing media file before attempting to write a new one. // This avoids the problem where there's already a file for this media type // with a different format/extension (e.g. game.jpg and we're going to write @@ -542,35 +514,6 @@ void MediaDownloadHandle::update() } } - // This is just a temporary workaround to avoid saving media files to disk that are - // actually just containing error messages from the scraper service. The proper solution - // is to implement file checksum checks to determine if the server response contains valid - // media. As for the current approach, if the file is less than 350 bytes, we check if - // FreeImage can actually detect a valid format, and if not, we present an error message. - // Black/empty images are sometimes returned from the scraper service and these can actually - // be less than 350 bytes in size. - if (Settings::getInstance()->getBool("ScraperHaltOnInvalidMedia") && - mReq->getContent().size() < 350) { - - FREE_IMAGE_FORMAT imageFormat {FIF_UNKNOWN}; - - if (mMediaType != "videos") { - std::string imageData {mReq->getContent()}; - FIMEMORY* memoryStream {FreeImage_OpenMemory(reinterpret_cast(&imageData.at(0)), - static_cast(imageData.size()))}; - imageFormat = FreeImage_GetFileTypeFromMemory(memoryStream, 0); - FreeImage_CloseMemory(memoryStream); - } - - if (imageFormat == FIF_UNKNOWN) { - setError("The " + mMediaType + " file \"" + Utils::FileSystem::getFileName(mSavePath) + - "\" returned by the scraper seems to be invalid as it's less than " + - "350 bytes in size", - true); - return; - } - } - // Remove any existing media file before attempting to write a new one. // This avoids the problem where there's already a file for this media type // with a different format/extension (e.g. game.jpg and we're going to write diff --git a/es-core/src/Settings.cpp b/es-core/src/Settings.cpp index f2edc2493..9165b975f 100644 --- a/es-core/src/Settings.cpp +++ b/es-core/src/Settings.cpp @@ -149,7 +149,6 @@ void Settings::setDefaults() mIntMap["ScraperRetryOnErrorTimer"] = {3, 3}; mIntMap["ScraperSearchFileHashMaxSize"] = {384, 384}; mBoolMap["ScraperOverwriteData"] = {true, true}; - mBoolMap["ScraperHaltOnInvalidMedia"] = {true, true}; mBoolMap["ScraperIgnoreHTTP404Errors"] = {true, true}; mBoolMap["ScraperSearchFileHash"] = {true, true}; mBoolMap["ScraperSearchMetadataName"] = {true, true};