From d83374b38f9ae218213894aaad7043b36b0646b5 Mon Sep 17 00:00:00 2001 From: Leon Styhre Date: Mon, 8 May 2023 17:14:52 +0200 Subject: [PATCH] Added an option to scrape game manuals using ScreenScraper Also changed the scraper auto-retry functionality to not run on non-recoverable errors or duing manual scraping --- es-app/src/FileData.cpp | 23 ++++ es-app/src/FileData.h | 1 + es-app/src/guis/GuiScraperMenu.cpp | 26 +++++ es-app/src/guis/GuiScraperSearch.cpp | 30 +++--- es-app/src/guis/GuiScraperSearch.h | 1 + es-app/src/scrapers/GamesDBJSONScraper.cpp | 29 ++--- es-app/src/scrapers/Scraper.cpp | 119 ++++++++++++--------- es-app/src/scrapers/Scraper.h | 2 + es-app/src/scrapers/ScreenScraper.cpp | 5 +- es-app/src/scrapers/ScreenScraper.h | 1 + es-app/src/views/GamelistBase.cpp | 8 ++ es-core/src/AsyncHandle.h | 7 +- es-core/src/Settings.cpp | 1 + 13 files changed, 177 insertions(+), 76 deletions(-) diff --git a/es-app/src/FileData.cpp b/es-app/src/FileData.cpp index 5f581325a..5780cce92 100644 --- a/es-app/src/FileData.cpp +++ b/es-app/src/FileData.cpp @@ -322,6 +322,29 @@ const std::string FileData::getVideoPath() const return ""; } +const std::string FileData::getManualPath() const +{ + const std::vector extList {".pdf"}; + std::string subFolders; + + // Extract possible subfolders from the path. + if (mEnvData->mStartPath != "") + subFolders = + Utils::String::replace(Utils::FileSystem::getParent(mPath), mEnvData->mStartPath, ""); + + const std::string tempPath {getMediaDirectory() + mSystemName + "/manuals" + subFolders + "/" + + getDisplayName()}; + + // Look for manuals in the media directory. + for (size_t i {0}; i < extList.size(); ++i) { + std::string mediaPath {tempPath + extList[i]}; + if (Utils::FileSystem::exists(mediaPath)) + return mediaPath; + } + + return ""; +} + const std::vector& FileData::getChildrenListToDisplay() { FileFilterIndex* idx {mSystem->getIndex()}; diff --git a/es-app/src/FileData.h b/es-app/src/FileData.h index f3fc3d15a..c5849c008 100644 --- a/es-app/src/FileData.h +++ b/es-app/src/FileData.h @@ -96,6 +96,7 @@ public: const std::string getScreenshotPath() const; const std::string getTitleScreenPath() const; const std::string getVideoPath() const; + const std::string getManualPath() const; const bool getDeletionFlag() const { return mDeletionFlag; } void setDeletionFlag(bool setting) { mDeletionFlag = setting; } diff --git a/es-app/src/guis/GuiScraperMenu.cpp b/es-app/src/guis/GuiScraperMenu.cpp index ff708631d..8a6d97787 100644 --- a/es-app/src/guis/GuiScraperMenu.cpp +++ b/es-app/src/guis/GuiScraperMenu.cpp @@ -419,6 +419,27 @@ void GuiScraperMenu::openContentOptions() } }); + // Scrape game manuals. + auto scrapeManuals = std::make_shared(); + scrapeManuals->setState(Settings::getInstance()->getBool("ScrapeManuals")); + s->addWithLabel("GAME MANUALS", scrapeManuals); + s->addSaveFunc([scrapeManuals, s] { + if (scrapeManuals->getState() != Settings::getInstance()->getBool("ScrapeManuals")) { + Settings::getInstance()->setBool("ScrapeManuals", scrapeManuals->getState()); + s->setNeedsSaving(); + } + }); + + // Game manuals are not supported by TheGamesDB, so gray out the option if this scraper + // is selected. + if (Settings::getInstance()->getString("Scraper") == "thegamesdb") { + scrapeManuals->setEnabled(false); + scrapeManuals->setOpacity(DISABLED_OPACITY); + scrapeManuals->getParent() + ->getChild(scrapeManuals->getChildIndex() - 1) + ->setOpacity(DISABLED_OPACITY); + } + mWindow->pushGui(s); } @@ -1126,6 +1147,11 @@ void GuiScraperMenu::start() contentToScrape = true; break; } + if (scraperService == "screenscraper" && + Settings::getInstance()->getBool("ScrapeManuals")) { + contentToScrape = true; + break; + } if (Settings::getInstance()->getBool("ScrapeMarquees")) { contentToScrape = true; break; diff --git a/es-app/src/guis/GuiScraperSearch.cpp b/es-app/src/guis/GuiScraperSearch.cpp index 736ba889b..5cfc1d94d 100644 --- a/es-app/src/guis/GuiScraperSearch.cpp +++ b/es-app/src/guis/GuiScraperSearch.cpp @@ -255,7 +255,7 @@ void GuiScraperSearch::resizeMetadata() mRenderer->getScreenWidthModifier())); } - for (unsigned int i = 0; i < mMD_Pairs.size(); ++i) + for (unsigned int i {0}; i < mMD_Pairs.size(); ++i) mMD_Grid->setRowHeightPerc( i * 2, (fontLbl->getLetterHeight() + (2.0f * (mRenderer->getIsVerticalOrientation() ? @@ -406,7 +406,7 @@ void GuiScraperSearch::onSearchDone(std::vector& results) mFoundGame = true; ComponentListRow row; - for (size_t i = 0; i < results.size(); ++i) { + for (size_t i {0}; i < results.size(); ++i) { // If the platform IDs returned by the scraper do not match the platform IDs of the // scraped game, then add the additional platform information to the end of the game // name (within square brackets). @@ -425,7 +425,7 @@ void GuiScraperSearch::onSearchDone(std::vector& results) } } - bool hasOtherPlatforms = false; + bool hasOtherPlatforms {false}; for (auto& platformID : mLastSearch.system->getSystemEnvData()->mPlatformIds) { if (!results.at(i).platformIDs.empty() && @@ -495,11 +495,13 @@ void GuiScraperSearch::onSearchDone(std::vector& results) } } -void GuiScraperSearch::onSearchError(const std::string& error, HttpReq::Status status) +void GuiScraperSearch::onSearchError(const std::string& error, + const bool retry, + HttpReq::Status status) { const int retries { glm::clamp(Settings::getInstance()->getInt("ScraperRetryOnErrorCount"), 0, 10)}; - if (retries > 0 && mRetryCount < retries) { + if (retry && mSearchType != NEVER_AUTO_ACCEPT && retries > 0 && mRetryCount < retries) { LOG(LogError) << "GuiScraperSearch: " << Utils::String::replace(error, "\n", ""); mRetrySearch = true; ++mRetryCount; @@ -535,7 +537,7 @@ int GuiScraperSearch::getSelectedIndex() void GuiScraperSearch::updateInfoPane() { - int i = getSelectedIndex(); + int i {getSelectedIndex()}; if (mSearchType == ALWAYS_ACCEPT_FIRST_RESULT && mScraperResults.size()) i = 0; @@ -670,6 +672,8 @@ void GuiScraperSearch::returnResult(ScraperSearchResult result) // Resolve metadata image before returning. if (result.mediaFilesDownloadStatus != COMPLETED) { result.mediaFilesDownloadStatus = IN_PROGRESS; + LOG(LogDebug) << "GuiScraperSearch::returnResult(): Selected game \"" + << result.mdl.get("name") << "\""; mMDResolveHandle = resolveMetaDataAssets(result, mLastSearch); return; } @@ -710,7 +714,8 @@ void GuiScraperSearch::update(int deltaTime) if (mSearchHandle && mSearchHandle->status() != ASYNC_IN_PROGRESS) { auto status = mSearchHandle->status(); mScraperResults = mSearchHandle->getResults(); - auto statusString = mSearchHandle->getStatusString(); + const std::string statusString {mSearchHandle->getStatusString()}; + const bool retryFlag {mSearchHandle->getRetry()}; // We reset here because onSearchDone in auto mode can call mSkipCallback() which // can call another search() which will set our mSearchHandle to something important. @@ -734,7 +739,7 @@ void GuiScraperSearch::update(int deltaTime) } } else if (status == ASYNC_ERROR) { - onSearchError(statusString); + onSearchError(statusString, retryFlag); } } @@ -767,7 +772,8 @@ void GuiScraperSearch::update(int deltaTime) onSearchDone(results_scrape); } else if (mMDRetrieveURLsHandle->status() == ASYNC_ERROR) { - onSearchError(mMDRetrieveURLsHandle->getStatusString()); + onSearchError(mMDRetrieveURLsHandle->getStatusString(), + mMDRetrieveURLsHandle->getRetry()); mMDRetrieveURLsHandle.reset(); } } @@ -823,7 +829,7 @@ void GuiScraperSearch::update(int deltaTime) } } else if (mMDResolveHandle->status() == ASYNC_ERROR) { - onSearchError(mMDResolveHandle->getStatusString()); + onSearchError(mMDResolveHandle->getStatusString(), mMDResolveHandle->getRetry()); mMDResolveHandle.reset(); } } @@ -850,7 +856,7 @@ void GuiScraperSearch::updateThumbnail() } else { mResultThumbnail->setImage(""); - onSearchError("Error downloading thumbnail:\n " + it->second->getErrorMsg(), + onSearchError("Error downloading thumbnail:\n " + it->second->getErrorMsg(), true, it->second->status()); } @@ -954,7 +960,7 @@ bool GuiScraperSearch::saveMetadata(const ScraperSearchResult& result, if (defaultName == metadata.get("name")) hasDefaultName = true; - for (unsigned int i = 0; i < mMetaDataDecl.size(); ++i) { + for (unsigned int i {0}; i < mMetaDataDecl.size(); ++i) { // Skip elements that are tagged not to be scraped. if (!mMetaDataDecl.at(i).shouldScrape) diff --git a/es-app/src/guis/GuiScraperSearch.h b/es-app/src/guis/GuiScraperSearch.h index 25fda7fce..3e7340f69 100644 --- a/es-app/src/guis/GuiScraperSearch.h +++ b/es-app/src/guis/GuiScraperSearch.h @@ -108,6 +108,7 @@ private: void resizeMetadata(); void onSearchError(const std::string& error, + const bool retry, HttpReq::Status status = HttpReq::REQ_UNDEFINED_ERROR); void onSearchDone(std::vector& results); diff --git a/es-app/src/scrapers/GamesDBJSONScraper.cpp b/es-app/src/scrapers/GamesDBJSONScraper.cpp index 5535084b5..2e3f8cbcf 100644 --- a/es-app/src/scrapers/GamesDBJSONScraper.cpp +++ b/es-app/src/scrapers/GamesDBJSONScraper.cpp @@ -200,10 +200,11 @@ void thegamesdb_generate_json_scraper_requests( if (Settings::getInstance()->getBool("ScraperConvertUnderscores")) cleanName = Utils::String::replace(cleanName, "_", " "); - path += "/Games/ByGameName?" + apiKey + - "&fields=players,publishers,genres,overview,last_updated,rating," - "platform,coop,youtube,os,processor,ram,hdd,video,sound,alternates&name=" + - HttpReq::urlEncode(cleanName); + path.append("/Games/ByGameName?") + .append(apiKey) + .append("&fields=players,publishers,genres,overview,last_updated,rating," + "platform,coop,youtube,os,processor,ram,hdd,video,sound,alternates&name=") + .append(HttpReq::urlEncode(cleanName)); } if (usingGameID) { @@ -248,10 +249,10 @@ void thegamesdb_generate_json_scraper_requests( std::vector& results) { resources.prepare(); - std::string path = "https://api.thegamesdb.net/v1"; + std::string path {"https://api.thegamesdb.net/v1"}; const std::string apiKey {std::string("apikey=") + resources.getApiKey()}; - path += "/Games/Images/GamesImages?" + apiKey + "&games_id=" + gameIDs; + path.append("/Games/Images/GamesImages?").append(apiKey).append("&games_id=").append(gameIDs); requests.push( std::unique_ptr(new TheGamesDBJSONRequest(requests, results, path))); @@ -290,9 +291,9 @@ namespace if (!v.IsArray()) return ""; - std::string out = ""; + std::string out; bool first {true}; - for (int i = 0; i < static_cast(v.Size()); ++i) { + for (int i {0}; i < static_cast(v.Size()); ++i) { auto mapIt = resources.gamesdb_new_developers_map.find(getIntOrThrow(v[i])); if (mapIt == resources.gamesdb_new_developers_map.cend()) @@ -314,7 +315,7 @@ namespace std::string out; bool first {true}; - for (int i = 0; i < static_cast(v.Size()); ++i) { + for (int i {0}; i < static_cast(v.Size()); ++i) { auto mapIt = resources.gamesdb_new_publishers_map.find(getIntOrThrow(v[i])); if (mapIt == resources.gamesdb_new_publishers_map.cend()) @@ -336,7 +337,7 @@ namespace std::string out; bool first {true}; - for (int i = 0; i < static_cast(v.Size()); ++i) { + for (int i {0}; i < static_cast(v.Size()); ++i) { auto mapIt = resources.gamesdb_new_genres_map.find(getIntOrThrow(v[i])); if (mapIt == resources.gamesdb_new_genres_map.cend()) @@ -432,7 +433,7 @@ void processMediaURLs(const Value& images, // Quite excessive testing for valid values, but you never know what the server has // returned and we don't want to crash the program due to malformed data. if (gameMedia.IsArray()) { - for (SizeType i = 0; i < gameMedia.Size(); ++i) { + for (SizeType i {0}; i < gameMedia.Size(); ++i) { std::string mediatype; std::string mediaside; if (gameMedia[i]["type"].IsString()) @@ -477,7 +478,7 @@ void TheGamesDBJSONRequest::process(const std::unique_ptr& req, if (doc.HasParseError()) { std::string err {std::string("TheGamesDBJSONRequest - Error parsing JSON \n\t") + GetParseError_En(doc.GetParseError())}; - setError(err); + setError(err, true); LOG(LogError) << err; return; } @@ -508,7 +509,7 @@ void TheGamesDBJSONRequest::process(const std::unique_ptr& req, // Find how many more requests we can make before the scraper // request allowance counter is reset. if (doc.HasMember("remaining_monthly_allowance") && doc.HasMember("extra_allowance")) { - for (size_t i = 0; i < results.size(); ++i) { + for (size_t i {0}; i < results.size(); ++i) { results[i].scraperRequestAllowance = doc["remaining_monthly_allowance"].GetInt() + doc["extra_allowance"].GetInt(); } @@ -529,7 +530,7 @@ void TheGamesDBJSONRequest::process(const std::unique_ptr& req, const Value& games {doc["data"]["games"]}; resources.ensureResources(); - for (int i = 0; i < static_cast(games.Size()); ++i) { + for (int i {0}; i < static_cast(games.Size()); ++i) { auto& v = games[i]; try { processGame(v, results); diff --git a/es-app/src/scrapers/Scraper.cpp b/es-app/src/scrapers/Scraper.cpp index c87231f77..6c6f702bf 100644 --- a/es-app/src/scrapers/Scraper.cpp +++ b/es-app/src/scrapers/Scraper.cpp @@ -62,7 +62,7 @@ std::unique_ptr startScraperSearch(const ScraperSearchParam std::unique_ptr startMediaURLsFetch(const std::string& gameIDs) { - const std::string& name = Settings::getInstance()->getString("Scraper"); + const std::string& name {Settings::getInstance()->getString("Scraper")}; std::unique_ptr handle(new ScraperSearchHandle()); ScraperSearchParams params; @@ -90,7 +90,7 @@ std::vector getScraperList() bool isValidConfiguredScraper() { - const std::string& name = Settings::getInstance()->getString("Scraper"); + const std::string& name {Settings::getInstance()->getString("Scraper")}; return scraper_request_funcs.find(name) != scraper_request_funcs.end(); } @@ -107,7 +107,7 @@ void ScraperSearchHandle::update() if (status == ASYNC_ERROR) { // Propagate error. - setError(req.getStatusString()); + setError(req.getStatusString(), req.getRetry()); // Empty our queue. while (!mRequestQueue.empty()) @@ -147,7 +147,7 @@ ScraperHttpRequest::ScraperHttpRequest(std::vector& results void ScraperHttpRequest::update() { - HttpReq::Status status = mReq->status(); + HttpReq::Status status {mReq->status()}; if (status == HttpReq::REQ_SUCCESS) { // If process() has an error, status will be changed to ASYNC_ERROR. setStatus(ASYNC_DONE); @@ -162,7 +162,7 @@ void ScraperHttpRequest::update() // Everything else is some sort of error. LOG(LogError) << "ScraperHttpRequest network error (status: " << status << ") - " << mReq->getErrorMsg(); - setError("Network error: " + mReq->getErrorMsg()); + setError("Network error: " + mReq->getErrorMsg(), true); } // Download and write the media files to disk. @@ -264,9 +264,16 @@ MDResolveHandle::MDResolveHandle(const ScraperSearchResult& result, ViewController::getInstance()->stopViewVideos(); #endif } + if (Settings::getInstance()->getBool("ScrapeManuals") && result.manualUrl != "") { + mediaFileInfo.fileURL = result.manualUrl; + mediaFileInfo.fileFormat = result.manualFormat; + mediaFileInfo.subDirectory = "manuals"; + mediaFileInfo.existingMediaFile = search.game->getManualPath(); + mediaFileInfo.resizeFile = false; + scrapeFiles.push_back(mediaFileInfo); + } for (auto it = scrapeFiles.cbegin(); it != scrapeFiles.cend(); ++it) { - std::string ext; // If we have a file extension returned by the scraper, then use it. @@ -275,13 +282,13 @@ MDResolveHandle::MDResolveHandle(const ScraperSearchResult& result, ext = it->fileFormat; } else { - size_t dot = it->fileURL.find_last_of('.'); + size_t dot {it->fileURL.find_last_of('.')}; if (dot != std::string::npos) ext = it->fileURL.substr(dot, std::string::npos); } - std::string filePath = getSaveAsPath(search, it->subDirectory, ext); + std::string filePath {getSaveAsPath(search, it->subDirectory, ext)}; // If there is an existing media file on disk and the setting to overwrite data // has been set to no, then don't proceed with downloading or saving a new file. @@ -304,17 +311,19 @@ MDResolveHandle::MDResolveHandle(const ScraperSearchResult& result, if (Settings::getInstance()->getBool("ScraperHaltOnInvalidMedia") && mResult.thumbnailImageData.size() < 350) { - FIMEMORY* memoryStream = + FIMEMORY* memoryStream { FreeImage_OpenMemory(reinterpret_cast(&mResult.thumbnailImageData.at(0)), - static_cast(mResult.thumbnailImageData.size())); + static_cast(mResult.thumbnailImageData.size()))}; - FREE_IMAGE_FORMAT imageFormat = FreeImage_GetFileTypeFromMemory(memoryStream, 0); + FREE_IMAGE_FORMAT imageFormat {FreeImage_GetFileTypeFromMemory(memoryStream, 0)}; FreeImage_CloseMemory(memoryStream); if (imageFormat == FIF_UNKNOWN) { - setError("The file \"" + Utils::FileSystem::getFileName(filePath) + - "\" returned by the scraper seems to be invalid as it's less than " + - "350 bytes in size"); + setError( + "The file \"" + Utils::FileSystem::getFileName(filePath) + + "\" returned by the scraper seems to be invalid as it's less than " + + "350 bytes in size", + true); return; } } @@ -330,7 +339,8 @@ MDResolveHandle::MDResolveHandle(const ScraperSearchResult& result, // problems or the MediaDirectory setting points to a file instead of a directory. if (!Utils::FileSystem::isDirectory(Utils::FileSystem::getParent(filePath))) { setError("Media directory does not exist and can't be created. " - "Permission problems?"); + "Permission problems?", + false); LOG(LogError) << "Couldn't create media directory: \"" << Utils::FileSystem::getParent(filePath) << "\""; return; @@ -343,22 +353,22 @@ MDResolveHandle::MDResolveHandle(const ScraperSearchResult& result, std::ofstream stream(filePath, std::ios_base::out | std::ios_base::binary); #endif if (!stream || stream.bad()) { - setError("Failed to open path for writing media file.\nPermission error?"); + setError("Failed to open path for writing media file.\nPermission error?", false); return; } - const std::string& content = mResult.thumbnailImageData; + const std::string& content {mResult.thumbnailImageData}; stream.write(content.data(), content.length()); stream.close(); if (stream.bad()) { - setError("Failed to save media file.\nDisk full?"); + setError("Failed to save media file.\nDisk full?", false); return; } // Resize it. if (it->resizeFile) { if (!resizeImage(filePath, it->subDirectory)) { - setError("Error saving resized image.\nOut of memory? Disk full?"); + setError("Error saving resized image.\nOut of memory? Disk full?", false); return; } } @@ -384,7 +394,7 @@ void MDResolveHandle::update() while (it != mFuncs.cend()) { if (it->first->status() == ASYNC_ERROR) { - setError(it->first->getStatusString()); + setError(it->first->getStatusString(), it->first->getRetry()); return; } else if (it->first->status() == ASYNC_DONE) { @@ -433,7 +443,7 @@ void MediaDownloadHandle::update() if (mReq->status() != HttpReq::REQ_SUCCESS) { std::stringstream ss; ss << "Network error: " << mReq->getErrorMsg(); - setError(ss.str()); + setError(ss.str(), true); return; } @@ -450,22 +460,22 @@ void MediaDownloadHandle::update() // and skip them so they're not saved to disk. if (Settings::getInstance()->getString("Scraper") == "screenscraper" && mMediaType == "backcovers") { - bool emptyImage = false; - FREE_IMAGE_FORMAT imageFormat = FIF_UNKNOWN; - std::string imageData = mReq->getContent(); - FIMEMORY* memoryStream = FreeImage_OpenMemory(reinterpret_cast(&imageData.at(0)), - static_cast(imageData.size())); + bool emptyImage {false}; + FREE_IMAGE_FORMAT imageFormat {FIF_UNKNOWN}; + std::string imageData {mReq->getContent()}; + FIMEMORY* memoryStream {FreeImage_OpenMemory(reinterpret_cast(&imageData.at(0)), + static_cast(imageData.size()))}; imageFormat = FreeImage_GetFileTypeFromMemory(memoryStream, 0); if (imageFormat != FIF_UNKNOWN) { emptyImage = true; - FIBITMAP* tempImage = FreeImage_LoadFromMemory(imageFormat, memoryStream); + FIBITMAP* tempImage {FreeImage_LoadFromMemory(imageFormat, memoryStream)}; RGBQUAD firstPixel; RGBQUAD currPixel; - unsigned int width = FreeImage_GetWidth(tempImage); - unsigned int height = FreeImage_GetHeight(tempImage); + unsigned int width {FreeImage_GetWidth(tempImage)}; + unsigned int height {FreeImage_GetHeight(tempImage)}; // Skip really small images as they're obviously not valid. if (width < 50) { @@ -477,7 +487,7 @@ void MediaDownloadHandle::update() else { // Remove the alpha channel which will convert fully transparent pixels to black. if (FreeImage_GetBPP(tempImage) != 24) { - FIBITMAP* convertImage = FreeImage_ConvertTo24Bits(tempImage); + FIBITMAP* convertImage {FreeImage_ConvertTo24Bits(tempImage)}; FreeImage_Unload(tempImage); tempImage = convertImage; } @@ -485,11 +495,11 @@ void MediaDownloadHandle::update() // Skip the first line as this can apparently lead to false positives. FreeImage_GetPixelColor(tempImage, 0, 1, &firstPixel); - for (unsigned int x = 0; x < width; ++x) { + for (unsigned int x {0}; x < width; ++x) { if (!emptyImage) break; // Skip the last line as well. - for (unsigned int y = 1; y < height - 1; ++y) { + for (unsigned int y {1}; y < height - 1; ++y) { FreeImage_GetPixelColor(tempImage, x, y, &currPixel); if (currPixel.rgbBlue != firstPixel.rgbBlue || currPixel.rgbGreen != firstPixel.rgbGreen || @@ -527,20 +537,21 @@ void MediaDownloadHandle::update() if (Settings::getInstance()->getBool("ScraperHaltOnInvalidMedia") && mReq->getContent().size() < 350) { - FREE_IMAGE_FORMAT imageFormat = FIF_UNKNOWN; + FREE_IMAGE_FORMAT imageFormat {FIF_UNKNOWN}; if (mMediaType != "videos") { - std::string imageData = mReq->getContent(); - FIMEMORY* memoryStream = FreeImage_OpenMemory(reinterpret_cast(&imageData.at(0)), - static_cast(imageData.size())); + std::string imageData {mReq->getContent()}; + FIMEMORY* memoryStream {FreeImage_OpenMemory(reinterpret_cast(&imageData.at(0)), + static_cast(imageData.size()))}; imageFormat = FreeImage_GetFileTypeFromMemory(memoryStream, 0); FreeImage_CloseMemory(memoryStream); } if (imageFormat == FIF_UNKNOWN) { setError("The file \"" + Utils::FileSystem::getFileName(mSavePath) + - "\" returned by the scraper seems to be invalid as it's less than " + - "350 bytes in size"); + "\" returned by the scraper seems to be invalid as it's less than " + + "350 bytes in size", + true); return; } } @@ -555,7 +566,8 @@ void MediaDownloadHandle::update() // If the media directory does not exist, something is wrong, possibly permission // problems or the MediaDirectory setting points to a file instead of a directory. if (!Utils::FileSystem::isDirectory(Utils::FileSystem::getParent(mSavePath))) { - setError("Media directory does not exist and can't be created. Permission problems?"); + setError("Media directory does not exist and can't be created. Permission problems?", + false); LOG(LogError) << "Couldn't create media directory: \"" << Utils::FileSystem::getParent(mSavePath) << "\""; return; @@ -568,22 +580,29 @@ void MediaDownloadHandle::update() std::ofstream stream(mSavePath, std::ios_base::out | std::ios_base::binary); #endif if (!stream || stream.bad()) { - setError("Failed to open path for writing media file.\nPermission error?"); + setError("Failed to open path for writing media file.\nPermission error?", false); return; } - const std::string& content = mReq->getContent(); + const std::string& content {mReq->getContent()}; stream.write(content.data(), content.length()); stream.close(); if (stream.bad()) { - setError("Failed to save media file.\nDisk full?"); + setError("Failed to save media file.\nDisk full?", false); return; } + if (mMediaType == "manuals") { + LOG(LogDebug) << "Scraper::update(): Saving game manual \"" << mSavePath << "\""; + } + else if (mMediaType == "videos") { + LOG(LogDebug) << "Scraper::update(): Saving video \"" << mSavePath << "\""; + } + // Resize it. if (mResizeFile) { if (!resizeImage(mSavePath, mMediaType)) { - setError("Error saving resized image.\nOut of memory? Disk full?"); + setError("Error saving resized image.\nOut of memory? Disk full?", false); return; } } @@ -713,8 +732,8 @@ std::string getSaveAsPath(const ScraperSearchParams& params, const std::string& filetypeSubdirectory, const std::string& extension) { - const std::string systemsubdirectory = params.system->getName(); - const std::string name = Utils::FileSystem::getStem(params.game->getPath()); + const std::string systemsubdirectory {params.system->getName()}; + const std::string name {Utils::FileSystem::getStem(params.game->getPath())}; std::string subFolders; // Extract possible subfolders from the path. @@ -722,16 +741,20 @@ std::string getSaveAsPath(const ScraperSearchParams& params, subFolders = Utils::String::replace(Utils::FileSystem::getParent(params.game->getPath()), params.system->getSystemEnvData()->mStartPath, ""); - std::string path = FileData::getMediaDirectory(); + std::string path {FileData::getMediaDirectory()}; if (!Utils::FileSystem::exists(path)) Utils::FileSystem::createDirectory(path); - path += systemsubdirectory + "/" + filetypeSubdirectory + subFolders + "/"; + path.append(systemsubdirectory) + .append("/") + .append(filetypeSubdirectory) + .append(subFolders) + .append("/"); if (!Utils::FileSystem::exists(path)) Utils::FileSystem::createDirectory(path); - path += name + extension; + path.append(name).append(extension); return path; } diff --git a/es-app/src/scrapers/Scraper.h b/es-app/src/scrapers/Scraper.h index 8a76dd3b5..a4c7c7c69 100644 --- a/es-app/src/scrapers/Scraper.h +++ b/es-app/src/scrapers/Scraper.h @@ -81,6 +81,7 @@ struct ScraperSearchResult { std::string screenshotUrl; std::string titlescreenUrl; std::string videoUrl; + std::string manualUrl; // Needed to pre-set the image type. std::string box3DFormat; @@ -92,6 +93,7 @@ struct ScraperSearchResult { std::string screenshotFormat; std::string titlescreenFormat; std::string videoFormat; + std::string manualFormat; // Indicates whether any new media files were downloaded and saved. bool savedNewMedia; diff --git a/es-app/src/scrapers/ScreenScraper.cpp b/es-app/src/scrapers/ScreenScraper.cpp index 7afe2d936..f81902cae 100644 --- a/es-app/src/scrapers/ScreenScraper.cpp +++ b/es-app/src/scrapers/ScreenScraper.cpp @@ -266,7 +266,7 @@ void ScreenScraperRequest::process(const std::unique_ptr& req, std::string err = ss.str(); LOG(LogError) << err; - setError("ScreenScraper error: \n" + req->getContent()); + setError("ScreenScraper error: \n" + req->getContent(), true); return; } @@ -606,6 +606,9 @@ void ScreenScraperRequest::processGame(const pugi::xml_document& xmldoc, if (result.videoUrl == "") processMedia(result, media_list, ssConfig.media_video_normalized, result.videoUrl, result.videoFormat, region); + // Game manuals. + processMedia(result, media_list, ssConfig.media_manual, result.manualUrl, + result.manualFormat, region); } result.mediaURLFetch = COMPLETED; out_results.emplace_back(result); diff --git a/es-app/src/scrapers/ScreenScraper.h b/es-app/src/scrapers/ScreenScraper.h index 474a14a1d..c0556a236 100644 --- a/es-app/src/scrapers/ScreenScraper.h +++ b/es-app/src/scrapers/ScreenScraper.h @@ -97,6 +97,7 @@ public: std::string media_titlescreen = "sstitle"; std::string media_video = "video"; std::string media_video_normalized = "video-normalized"; + std::string media_manual = "manuel"; bool isArcadeSystem; bool automaticMode; diff --git a/es-app/src/views/GamelistBase.cpp b/es-app/src/views/GamelistBase.cpp index 98b307d37..030a81f45 100644 --- a/es-app/src/views/GamelistBase.cpp +++ b/es-app/src/views/GamelistBase.cpp @@ -668,6 +668,14 @@ void GamelistBase::removeMedia(FileData* game) removeEmptyDirFunc(systemMediaDir, mediaType, path); } + while (Utils::FileSystem::exists(game->getManualPath())) { + mediaType = "manuals"; + path = game->getManualPath(); + if (Utils::FileSystem::removeFile(path)) + break; + removeEmptyDirFunc(systemMediaDir, mediaType, path); + } + while (Utils::FileSystem::exists(game->getMiximagePath())) { mediaType = "miximages"; path = game->getMiximagePath(); diff --git a/es-core/src/AsyncHandle.h b/es-core/src/AsyncHandle.h index 89d382c4e..1b4c57349 100644 --- a/es-core/src/AsyncHandle.h +++ b/es-core/src/AsyncHandle.h @@ -23,6 +23,7 @@ class AsyncHandle public: AsyncHandle() : mStatus(ASYNC_IN_PROGRESS) + , mRetry {true} { } virtual ~AsyncHandle() {} @@ -36,6 +37,8 @@ public: return mStatus; } + const bool getRetry() { return mRetry; } + // User-friendly string of our current status. // Will return error message if status() == SEARCH_ERROR. std::string getStatusString() @@ -54,14 +57,16 @@ public: protected: void setStatus(AsyncHandleStatus status) { mStatus = status; } - void setError(const std::string& error) + void setError(const std::string& error, bool retry) { setStatus(ASYNC_ERROR); mError = error; + mRetry = retry; } std::string mError; AsyncHandleStatus mStatus; + bool mRetry; }; #endif // ES_CORE_ASYNC_HANDLE_H diff --git a/es-core/src/Settings.cpp b/es-core/src/Settings.cpp index d294617c0..5b2a0726f 100644 --- a/es-core/src/Settings.cpp +++ b/es-core/src/Settings.cpp @@ -118,6 +118,7 @@ void Settings::setDefaults() mBoolMap["Scrape3DBoxes"] = {true, true}; mBoolMap["ScrapePhysicalMedia"] = {true, true}; mBoolMap["ScrapeFanArt"] = {true, true}; + mBoolMap["ScrapeManuals"] = {false, false}; mStringMap["MiximageResolution"] = {"1280x960", "1280x960"}; mStringMap["MiximageScreenshotScaling"] = {"sharp", "sharp"};