Added an option to scrape game manuals using ScreenScraper

Also changed the scraper auto-retry functionality to not run on non-recoverable errors or duing manual scraping

es-app/src/FileData.cpp

@@ -322,6 +322,29 @@ const std::string FileData::getVideoPath() const
     return "";
 }
 
+const std::string FileData::getManualPath() const
+{
+    const std::vector<std::string> extList {".pdf"};
+    std::string subFolders;
+
+    // Extract possible subfolders from the path.
+    if (mEnvData->mStartPath != "")
+        subFolders =
+            Utils::String::replace(Utils::FileSystem::getParent(mPath), mEnvData->mStartPath, "");
+
+    const std::string tempPath {getMediaDirectory() + mSystemName + "/manuals" + subFolders + "/" +
+                                getDisplayName()};
+
+    // Look for manuals in the media directory.
+    for (size_t i {0}; i < extList.size(); ++i) {
+        std::string mediaPath {tempPath + extList[i]};
+        if (Utils::FileSystem::exists(mediaPath))
+            return mediaPath;
+    }
+
+    return "";
+}
+
 const std::vector<FileData*>& FileData::getChildrenListToDisplay()
 {
     FileFilterIndex* idx {mSystem->getIndex()};

es-app/src/FileData.h

@@ -96,6 +96,7 @@ public:
     const std::string getScreenshotPath() const;
     const std::string getTitleScreenPath() const;
     const std::string getVideoPath() const;
+    const std::string getManualPath() const;
 
     const bool getDeletionFlag() const { return mDeletionFlag; }
     void setDeletionFlag(bool setting) { mDeletionFlag = setting; }

es-app/src/guis/GuiScraperMenu.cpp

@@ -419,6 +419,27 @@ void GuiScraperMenu::openContentOptions()
         }
     });
 
+    // Scrape game manuals.
+    auto scrapeManuals = std::make_shared<SwitchComponent>();
+    scrapeManuals->setState(Settings::getInstance()->getBool("ScrapeManuals"));
+    s->addWithLabel("GAME MANUALS", scrapeManuals);
+    s->addSaveFunc([scrapeManuals, s] {
+        if (scrapeManuals->getState() != Settings::getInstance()->getBool("ScrapeManuals")) {
+            Settings::getInstance()->setBool("ScrapeManuals", scrapeManuals->getState());
+            s->setNeedsSaving();
+        }
+    });
+
+    // Game manuals are not supported by TheGamesDB, so gray out the option if this scraper
+    // is selected.
+    if (Settings::getInstance()->getString("Scraper") == "thegamesdb") {
+        scrapeManuals->setEnabled(false);
+        scrapeManuals->setOpacity(DISABLED_OPACITY);
+        scrapeManuals->getParent()
+            ->getChild(scrapeManuals->getChildIndex() - 1)
+            ->setOpacity(DISABLED_OPACITY);
+    }
+
     mWindow->pushGui(s);
 }
 
@@ -1126,6 +1147,11 @@ void GuiScraperMenu::start()
             contentToScrape = true;
             break;
         }
+        if (scraperService == "screenscraper" &&
+            Settings::getInstance()->getBool("ScrapeManuals")) {
+            contentToScrape = true;
+            break;
+        }
         if (Settings::getInstance()->getBool("ScrapeMarquees")) {
             contentToScrape = true;
             break;

es-app/src/guis/GuiScraperSearch.cpp

@@ -255,7 +255,7 @@ void GuiScraperSearch::resizeMetadata()
                                                            mRenderer->getScreenWidthModifier()));
         }
 
-        for (unsigned int i = 0; i < mMD_Pairs.size(); ++i)
+        for (unsigned int i {0}; i < mMD_Pairs.size(); ++i)
             mMD_Grid->setRowHeightPerc(
                 i * 2,
                 (fontLbl->getLetterHeight() + (2.0f * (mRenderer->getIsVerticalOrientation() ?
@@ -406,7 +406,7 @@ void GuiScraperSearch::onSearchDone(std::vector<ScraperSearchResult>& results)
         mFoundGame = true;
         ComponentListRow row;
 
-        for (size_t i = 0; i < results.size(); ++i) {
+        for (size_t i {0}; i < results.size(); ++i) {
             // If the platform IDs returned by the scraper do not match the platform IDs of the
             // scraped game, then add the additional platform information to the end of the game
             // name (within square brackets).
@@ -425,7 +425,7 @@ void GuiScraperSearch::onSearchDone(std::vector<ScraperSearchResult>& results)
                 }
             }
 
-            bool hasOtherPlatforms = false;
+            bool hasOtherPlatforms {false};
 
             for (auto& platformID : mLastSearch.system->getSystemEnvData()->mPlatformIds) {
                 if (!results.at(i).platformIDs.empty() &&
@@ -495,11 +495,13 @@ void GuiScraperSearch::onSearchDone(std::vector<ScraperSearchResult>& results)
     }
 }
 
-void GuiScraperSearch::onSearchError(const std::string& error, HttpReq::Status status)
+void GuiScraperSearch::onSearchError(const std::string& error,
+                                     const bool retry,
+                                     HttpReq::Status status)
 {
     const int retries {
         glm::clamp(Settings::getInstance()->getInt("ScraperRetryOnErrorCount"), 0, 10)};
-    if (retries > 0 && mRetryCount < retries) {
+    if (retry && mSearchType != NEVER_AUTO_ACCEPT && retries > 0 && mRetryCount < retries) {
         LOG(LogError) << "GuiScraperSearch: " << Utils::String::replace(error, "\n", "");
         mRetrySearch = true;
         ++mRetryCount;
@@ -535,7 +537,7 @@ int GuiScraperSearch::getSelectedIndex()
 
 void GuiScraperSearch::updateInfoPane()
 {
-    int i = getSelectedIndex();
+    int i {getSelectedIndex()};
     if (mSearchType == ALWAYS_ACCEPT_FIRST_RESULT && mScraperResults.size())
         i = 0;
 
@@ -670,6 +672,8 @@ void GuiScraperSearch::returnResult(ScraperSearchResult result)
     // Resolve metadata image before returning.
     if (result.mediaFilesDownloadStatus != COMPLETED) {
         result.mediaFilesDownloadStatus = IN_PROGRESS;
+        LOG(LogDebug) << "GuiScraperSearch::returnResult(): Selected game \""
+                      << result.mdl.get("name") << "\"";
         mMDResolveHandle = resolveMetaDataAssets(result, mLastSearch);
         return;
     }
@@ -710,7 +714,8 @@ void GuiScraperSearch::update(int deltaTime)
     if (mSearchHandle && mSearchHandle->status() != ASYNC_IN_PROGRESS) {
         auto status = mSearchHandle->status();
         mScraperResults = mSearchHandle->getResults();
-        auto statusString = mSearchHandle->getStatusString();
+        const std::string statusString {mSearchHandle->getStatusString()};
+        const bool retryFlag {mSearchHandle->getRetry()};
 
         // We reset here because onSearchDone in auto mode can call mSkipCallback() which
         // can call another search() which will set our mSearchHandle to something important.
@@ -734,7 +739,7 @@ void GuiScraperSearch::update(int deltaTime)
             }
         }
         else if (status == ASYNC_ERROR) {
-            onSearchError(statusString);
+            onSearchError(statusString, retryFlag);
         }
     }
 
@@ -767,7 +772,8 @@ void GuiScraperSearch::update(int deltaTime)
             onSearchDone(results_scrape);
         }
         else if (mMDRetrieveURLsHandle->status() == ASYNC_ERROR) {
-            onSearchError(mMDRetrieveURLsHandle->getStatusString());
+            onSearchError(mMDRetrieveURLsHandle->getStatusString(),
+                          mMDRetrieveURLsHandle->getRetry());
             mMDRetrieveURLsHandle.reset();
         }
     }
@@ -823,7 +829,7 @@ void GuiScraperSearch::update(int deltaTime)
             }
         }
         else if (mMDResolveHandle->status() == ASYNC_ERROR) {
-            onSearchError(mMDResolveHandle->getStatusString());
+            onSearchError(mMDResolveHandle->getStatusString(), mMDResolveHandle->getRetry());
             mMDResolveHandle.reset();
         }
     }
@@ -850,7 +856,7 @@ void GuiScraperSearch::updateThumbnail()
     }
     else {
         mResultThumbnail->setImage("");
-        onSearchError("Error downloading thumbnail:\n " + it->second->getErrorMsg(),
+        onSearchError("Error downloading thumbnail:\n " + it->second->getErrorMsg(), true,
                       it->second->status());
     }
 
@@ -954,7 +960,7 @@ bool GuiScraperSearch::saveMetadata(const ScraperSearchResult& result,
     if (defaultName == metadata.get("name"))
         hasDefaultName = true;
 
-    for (unsigned int i = 0; i < mMetaDataDecl.size(); ++i) {
+    for (unsigned int i {0}; i < mMetaDataDecl.size(); ++i) {
 
         // Skip elements that are tagged not to be scraped.
         if (!mMetaDataDecl.at(i).shouldScrape)

es-app/src/guis/GuiScraperSearch.h

@@ -108,6 +108,7 @@ private:
     void resizeMetadata();
 
     void onSearchError(const std::string& error,
+                       const bool retry,
                        HttpReq::Status status = HttpReq::REQ_UNDEFINED_ERROR);
     void onSearchDone(std::vector<ScraperSearchResult>& results);
 

es-app/src/scrapers/GamesDBJSONScraper.cpp

@@ -200,10 +200,11 @@ void thegamesdb_generate_json_scraper_requests(
         if (Settings::getInstance()->getBool("ScraperConvertUnderscores"))
             cleanName = Utils::String::replace(cleanName, "_", " ");
 
-        path += "/Games/ByGameName?" + apiKey +
+        path.append("/Games/ByGameName?")
-                "&fields=players,publishers,genres,overview,last_updated,rating,"
+            .append(apiKey)
-                "platform,coop,youtube,os,processor,ram,hdd,video,sound,alternates&name=" +
+            .append("&fields=players,publishers,genres,overview,last_updated,rating,"
-                HttpReq::urlEncode(cleanName);
+                    "platform,coop,youtube,os,processor,ram,hdd,video,sound,alternates&name=")
+            .append(HttpReq::urlEncode(cleanName));
     }
 
     if (usingGameID) {
@@ -248,10 +249,10 @@ void thegamesdb_generate_json_scraper_requests(
     std::vector<ScraperSearchResult>& results)
 {
     resources.prepare();
-    std::string path = "https://api.thegamesdb.net/v1";
+    std::string path {"https://api.thegamesdb.net/v1"};
     const std::string apiKey {std::string("apikey=") + resources.getApiKey()};
 
-    path += "/Games/Images/GamesImages?" + apiKey + "&games_id=" + gameIDs;
+    path.append("/Games/Images/GamesImages?").append(apiKey).append("&games_id=").append(gameIDs);
 
     requests.push(
         std::unique_ptr<ScraperRequest>(new TheGamesDBJSONRequest(requests, results, path)));
@@ -290,9 +291,9 @@ namespace
         if (!v.IsArray())
             return "";
 
-        std::string out = "";
+        std::string out;
         bool first {true};
-        for (int i = 0; i < static_cast<int>(v.Size()); ++i) {
+        for (int i {0}; i < static_cast<int>(v.Size()); ++i) {
             auto mapIt = resources.gamesdb_new_developers_map.find(getIntOrThrow(v[i]));
 
             if (mapIt == resources.gamesdb_new_developers_map.cend())
@@ -314,7 +315,7 @@ namespace
 
         std::string out;
         bool first {true};
-        for (int i = 0; i < static_cast<int>(v.Size()); ++i) {
+        for (int i {0}; i < static_cast<int>(v.Size()); ++i) {
             auto mapIt = resources.gamesdb_new_publishers_map.find(getIntOrThrow(v[i]));
 
             if (mapIt == resources.gamesdb_new_publishers_map.cend())
@@ -336,7 +337,7 @@ namespace
 
         std::string out;
         bool first {true};
-        for (int i = 0; i < static_cast<int>(v.Size()); ++i) {
+        for (int i {0}; i < static_cast<int>(v.Size()); ++i) {
             auto mapIt = resources.gamesdb_new_genres_map.find(getIntOrThrow(v[i]));
 
             if (mapIt == resources.gamesdb_new_genres_map.cend())
@@ -432,7 +433,7 @@ void processMediaURLs(const Value& images,
         // Quite excessive testing for valid values, but you never know what the server has
         // returned and we don't want to crash the program due to malformed data.
         if (gameMedia.IsArray()) {
-            for (SizeType i = 0; i < gameMedia.Size(); ++i) {
+            for (SizeType i {0}; i < gameMedia.Size(); ++i) {
                 std::string mediatype;
                 std::string mediaside;
                 if (gameMedia[i]["type"].IsString())
@@ -477,7 +478,7 @@ void TheGamesDBJSONRequest::process(const std::unique_ptr<HttpReq>& req,
     if (doc.HasParseError()) {
         std::string err {std::string("TheGamesDBJSONRequest - Error parsing JSON \n\t") +
                          GetParseError_En(doc.GetParseError())};
-        setError(err);
+        setError(err, true);
         LOG(LogError) << err;
         return;
     }
@@ -508,7 +509,7 @@ void TheGamesDBJSONRequest::process(const std::unique_ptr<HttpReq>& req,
         // Find how many more requests we can make before the scraper
         // request allowance counter is reset.
         if (doc.HasMember("remaining_monthly_allowance") && doc.HasMember("extra_allowance")) {
-            for (size_t i = 0; i < results.size(); ++i) {
+            for (size_t i {0}; i < results.size(); ++i) {
                 results[i].scraperRequestAllowance =
                     doc["remaining_monthly_allowance"].GetInt() + doc["extra_allowance"].GetInt();
             }
@@ -529,7 +530,7 @@ void TheGamesDBJSONRequest::process(const std::unique_ptr<HttpReq>& req,
     const Value& games {doc["data"]["games"]};
     resources.ensureResources();
 
-    for (int i = 0; i < static_cast<int>(games.Size()); ++i) {
+    for (int i {0}; i < static_cast<int>(games.Size()); ++i) {
         auto& v = games[i];
         try {
             processGame(v, results);

es-app/src/scrapers/Scraper.cpp

@@ -62,7 +62,7 @@ std::unique_ptr<ScraperSearchHandle> startScraperSearch(const ScraperSearchParam
 
 std::unique_ptr<ScraperSearchHandle> startMediaURLsFetch(const std::string& gameIDs)
 {
-    const std::string& name = Settings::getInstance()->getString("Scraper");
+    const std::string& name {Settings::getInstance()->getString("Scraper")};
     std::unique_ptr<ScraperSearchHandle> handle(new ScraperSearchHandle());
 
     ScraperSearchParams params;
@@ -90,7 +90,7 @@ std::vector<std::string> getScraperList()
 
 bool isValidConfiguredScraper()
 {
-    const std::string& name = Settings::getInstance()->getString("Scraper");
+    const std::string& name {Settings::getInstance()->getString("Scraper")};
     return scraper_request_funcs.find(name) != scraper_request_funcs.end();
 }
 
@@ -107,7 +107,7 @@ void ScraperSearchHandle::update()
 
         if (status == ASYNC_ERROR) {
             // Propagate error.
-            setError(req.getStatusString());
+            setError(req.getStatusString(), req.getRetry());
 
             // Empty our queue.
             while (!mRequestQueue.empty())
@@ -147,7 +147,7 @@ ScraperHttpRequest::ScraperHttpRequest(std::vector<ScraperSearchResult>& results
 
 void ScraperHttpRequest::update()
 {
-    HttpReq::Status status = mReq->status();
+    HttpReq::Status status {mReq->status()};
     if (status == HttpReq::REQ_SUCCESS) {
         // If process() has an error, status will be changed to ASYNC_ERROR.
         setStatus(ASYNC_DONE);
@@ -162,7 +162,7 @@ void ScraperHttpRequest::update()
     // Everything else is some sort of error.
     LOG(LogError) << "ScraperHttpRequest network error (status: " << status << ") - "
                   << mReq->getErrorMsg();
-    setError("Network error: " + mReq->getErrorMsg());
+    setError("Network error: " + mReq->getErrorMsg(), true);
 }
 
 // Download and write the media files to disk.
@@ -264,9 +264,16 @@ MDResolveHandle::MDResolveHandle(const ScraperSearchResult& result,
         ViewController::getInstance()->stopViewVideos();
 #endif
     }
+    if (Settings::getInstance()->getBool("ScrapeManuals") && result.manualUrl != "") {
+        mediaFileInfo.fileURL = result.manualUrl;
+        mediaFileInfo.fileFormat = result.manualFormat;
+        mediaFileInfo.subDirectory = "manuals";
+        mediaFileInfo.existingMediaFile = search.game->getManualPath();
+        mediaFileInfo.resizeFile = false;
+        scrapeFiles.push_back(mediaFileInfo);
+    }
 
     for (auto it = scrapeFiles.cbegin(); it != scrapeFiles.cend(); ++it) {
-
         std::string ext;
 
         // If we have a file extension returned by the scraper, then use it.
@@ -275,13 +282,13 @@ MDResolveHandle::MDResolveHandle(const ScraperSearchResult& result,
             ext = it->fileFormat;
         }
         else {
-            size_t dot = it->fileURL.find_last_of('.');
+            size_t dot {it->fileURL.find_last_of('.')};
 
             if (dot != std::string::npos)
                 ext = it->fileURL.substr(dot, std::string::npos);
         }
 
-        std::string filePath = getSaveAsPath(search, it->subDirectory, ext);
+        std::string filePath {getSaveAsPath(search, it->subDirectory, ext)};
 
         // If there is an existing media file on disk and the setting to overwrite data
         // has been set to no, then don't proceed with downloading or saving a new file.
@@ -304,17 +311,19 @@ MDResolveHandle::MDResolveHandle(const ScraperSearchResult& result,
             if (Settings::getInstance()->getBool("ScraperHaltOnInvalidMedia") &&
                 mResult.thumbnailImageData.size() < 350) {
 
-                FIMEMORY* memoryStream =
+                FIMEMORY* memoryStream {
                     FreeImage_OpenMemory(reinterpret_cast<BYTE*>(&mResult.thumbnailImageData.at(0)),
-                                         static_cast<DWORD>(mResult.thumbnailImageData.size()));
+                                         static_cast<DWORD>(mResult.thumbnailImageData.size()))};
 
-                FREE_IMAGE_FORMAT imageFormat = FreeImage_GetFileTypeFromMemory(memoryStream, 0);
+                FREE_IMAGE_FORMAT imageFormat {FreeImage_GetFileTypeFromMemory(memoryStream, 0)};
                 FreeImage_CloseMemory(memoryStream);
 
                 if (imageFormat == FIF_UNKNOWN) {
-                    setError("The file \"" + Utils::FileSystem::getFileName(filePath) +
+                    setError(
-                             "\" returned by the scraper seems to be invalid as it's less than " +
+                        "The file \"" + Utils::FileSystem::getFileName(filePath) +
-                             "350 bytes in size");
+                            "\" returned by the scraper seems to be invalid as it's less than " +
+                            "350 bytes in size",
+                        true);
                     return;
                 }
             }
@@ -330,7 +339,8 @@ MDResolveHandle::MDResolveHandle(const ScraperSearchResult& result,
             // problems or the MediaDirectory setting points to a file instead of a directory.
             if (!Utils::FileSystem::isDirectory(Utils::FileSystem::getParent(filePath))) {
                 setError("Media directory does not exist and can't be created. "
-                         "Permission problems?");
+                         "Permission problems?",
+                         false);
                 LOG(LogError) << "Couldn't create media directory: \""
                               << Utils::FileSystem::getParent(filePath) << "\"";
                 return;
@@ -343,22 +353,22 @@ MDResolveHandle::MDResolveHandle(const ScraperSearchResult& result,
             std::ofstream stream(filePath, std::ios_base::out | std::ios_base::binary);
 #endif
             if (!stream || stream.bad()) {
-                setError("Failed to open path for writing media file.\nPermission error?");
+                setError("Failed to open path for writing media file.\nPermission error?", false);
                 return;
             }
 
-            const std::string& content = mResult.thumbnailImageData;
+            const std::string& content {mResult.thumbnailImageData};
             stream.write(content.data(), content.length());
             stream.close();
             if (stream.bad()) {
-                setError("Failed to save media file.\nDisk full?");
+                setError("Failed to save media file.\nDisk full?", false);
                 return;
             }
 
             // Resize it.
             if (it->resizeFile) {
                 if (!resizeImage(filePath, it->subDirectory)) {
-                    setError("Error saving resized image.\nOut of memory? Disk full?");
+                    setError("Error saving resized image.\nOut of memory? Disk full?", false);
                     return;
                 }
             }
@@ -384,7 +394,7 @@ void MDResolveHandle::update()
     while (it != mFuncs.cend()) {
 
         if (it->first->status() == ASYNC_ERROR) {
-            setError(it->first->getStatusString());
+            setError(it->first->getStatusString(), it->first->getRetry());
             return;
         }
         else if (it->first->status() == ASYNC_DONE) {
@@ -433,7 +443,7 @@ void MediaDownloadHandle::update()
     if (mReq->status() != HttpReq::REQ_SUCCESS) {
         std::stringstream ss;
         ss << "Network error: " << mReq->getErrorMsg();
-        setError(ss.str());
+        setError(ss.str(), true);
         return;
     }
 
@@ -450,22 +460,22 @@ void MediaDownloadHandle::update()
     // and skip them so they're not saved to disk.
     if (Settings::getInstance()->getString("Scraper") == "screenscraper" &&
         mMediaType == "backcovers") {
-        bool emptyImage = false;
+        bool emptyImage {false};
-        FREE_IMAGE_FORMAT imageFormat = FIF_UNKNOWN;
+        FREE_IMAGE_FORMAT imageFormat {FIF_UNKNOWN};
-        std::string imageData = mReq->getContent();
+        std::string imageData {mReq->getContent()};
-        FIMEMORY* memoryStream = FreeImage_OpenMemory(reinterpret_cast<BYTE*>(&imageData.at(0)),
+        FIMEMORY* memoryStream {FreeImage_OpenMemory(reinterpret_cast<BYTE*>(&imageData.at(0)),
-                                                      static_cast<DWORD>(imageData.size()));
+                                                     static_cast<DWORD>(imageData.size()))};
         imageFormat = FreeImage_GetFileTypeFromMemory(memoryStream, 0);
 
         if (imageFormat != FIF_UNKNOWN) {
             emptyImage = true;
 
-            FIBITMAP* tempImage = FreeImage_LoadFromMemory(imageFormat, memoryStream);
+            FIBITMAP* tempImage {FreeImage_LoadFromMemory(imageFormat, memoryStream)};
             RGBQUAD firstPixel;
             RGBQUAD currPixel;
 
-            unsigned int width = FreeImage_GetWidth(tempImage);
+            unsigned int width {FreeImage_GetWidth(tempImage)};
-            unsigned int height = FreeImage_GetHeight(tempImage);
+            unsigned int height {FreeImage_GetHeight(tempImage)};
 
             // Skip really small images as they're obviously not valid.
             if (width < 50) {
@@ -477,7 +487,7 @@ void MediaDownloadHandle::update()
             else {
                 // Remove the alpha channel which will convert fully transparent pixels to black.
                 if (FreeImage_GetBPP(tempImage) != 24) {
-                    FIBITMAP* convertImage = FreeImage_ConvertTo24Bits(tempImage);
+                    FIBITMAP* convertImage {FreeImage_ConvertTo24Bits(tempImage)};
                     FreeImage_Unload(tempImage);
                     tempImage = convertImage;
                 }
@@ -485,11 +495,11 @@ void MediaDownloadHandle::update()
                 // Skip the first line as this can apparently lead to false positives.
                 FreeImage_GetPixelColor(tempImage, 0, 1, &firstPixel);
 
-                for (unsigned int x = 0; x < width; ++x) {
+                for (unsigned int x {0}; x < width; ++x) {
                     if (!emptyImage)
                         break;
                     // Skip the last line as well.
-                    for (unsigned int y = 1; y < height - 1; ++y) {
+                    for (unsigned int y {1}; y < height - 1; ++y) {
                         FreeImage_GetPixelColor(tempImage, x, y, &currPixel);
                         if (currPixel.rgbBlue != firstPixel.rgbBlue ||
                             currPixel.rgbGreen != firstPixel.rgbGreen ||
@@ -527,20 +537,21 @@ void MediaDownloadHandle::update()
     if (Settings::getInstance()->getBool("ScraperHaltOnInvalidMedia") &&
         mReq->getContent().size() < 350) {
 
-        FREE_IMAGE_FORMAT imageFormat = FIF_UNKNOWN;
+        FREE_IMAGE_FORMAT imageFormat {FIF_UNKNOWN};
 
         if (mMediaType != "videos") {
-            std::string imageData = mReq->getContent();
+            std::string imageData {mReq->getContent()};
-            FIMEMORY* memoryStream = FreeImage_OpenMemory(reinterpret_cast<BYTE*>(&imageData.at(0)),
+            FIMEMORY* memoryStream {FreeImage_OpenMemory(reinterpret_cast<BYTE*>(&imageData.at(0)),
-                                                          static_cast<DWORD>(imageData.size()));
+                                                         static_cast<DWORD>(imageData.size()))};
             imageFormat = FreeImage_GetFileTypeFromMemory(memoryStream, 0);
             FreeImage_CloseMemory(memoryStream);
         }
 
         if (imageFormat == FIF_UNKNOWN) {
             setError("The file \"" + Utils::FileSystem::getFileName(mSavePath) +
-                     "\" returned by the scraper seems to be invalid as it's less than " +
+                         "\" returned by the scraper seems to be invalid as it's less than " +
-                     "350 bytes in size");
+                         "350 bytes in size",
+                     true);
             return;
         }
     }
@@ -555,7 +566,8 @@ void MediaDownloadHandle::update()
     // If the media directory does not exist, something is wrong, possibly permission
     // problems or the MediaDirectory setting points to a file instead of a directory.
     if (!Utils::FileSystem::isDirectory(Utils::FileSystem::getParent(mSavePath))) {
-        setError("Media directory does not exist and can't be created. Permission problems?");
+        setError("Media directory does not exist and can't be created. Permission problems?",
+                 false);
         LOG(LogError) << "Couldn't create media directory: \""
                       << Utils::FileSystem::getParent(mSavePath) << "\"";
         return;
@@ -568,22 +580,29 @@ void MediaDownloadHandle::update()
     std::ofstream stream(mSavePath, std::ios_base::out | std::ios_base::binary);
 #endif
     if (!stream || stream.bad()) {
-        setError("Failed to open path for writing media file.\nPermission error?");
+        setError("Failed to open path for writing media file.\nPermission error?", false);
         return;
     }
 
-    const std::string& content = mReq->getContent();
+    const std::string& content {mReq->getContent()};
     stream.write(content.data(), content.length());
     stream.close();
     if (stream.bad()) {
-        setError("Failed to save media file.\nDisk full?");
+        setError("Failed to save media file.\nDisk full?", false);
         return;
     }
 
+    if (mMediaType == "manuals") {
+        LOG(LogDebug) << "Scraper::update(): Saving game manual \"" << mSavePath << "\"";
+    }
+    else if (mMediaType == "videos") {
+        LOG(LogDebug) << "Scraper::update(): Saving video \"" << mSavePath << "\"";
+    }
+
     // Resize it.
     if (mResizeFile) {
         if (!resizeImage(mSavePath, mMediaType)) {
-            setError("Error saving resized image.\nOut of memory? Disk full?");
+            setError("Error saving resized image.\nOut of memory? Disk full?", false);
             return;
         }
     }
@@ -713,8 +732,8 @@ std::string getSaveAsPath(const ScraperSearchParams& params,
                           const std::string& filetypeSubdirectory,
                           const std::string& extension)
 {
-    const std::string systemsubdirectory = params.system->getName();
+    const std::string systemsubdirectory {params.system->getName()};
-    const std::string name = Utils::FileSystem::getStem(params.game->getPath());
+    const std::string name {Utils::FileSystem::getStem(params.game->getPath())};
     std::string subFolders;
 
     // Extract possible subfolders from the path.
@@ -722,16 +741,20 @@ std::string getSaveAsPath(const ScraperSearchParams& params,
         subFolders = Utils::String::replace(Utils::FileSystem::getParent(params.game->getPath()),
                                             params.system->getSystemEnvData()->mStartPath, "");
 
-    std::string path = FileData::getMediaDirectory();
+    std::string path {FileData::getMediaDirectory()};
 
     if (!Utils::FileSystem::exists(path))
         Utils::FileSystem::createDirectory(path);
 
-    path += systemsubdirectory + "/" + filetypeSubdirectory + subFolders + "/";
+    path.append(systemsubdirectory)
+        .append("/")
+        .append(filetypeSubdirectory)
+        .append(subFolders)
+        .append("/");
 
     if (!Utils::FileSystem::exists(path))
         Utils::FileSystem::createDirectory(path);
 
-    path += name + extension;
+    path.append(name).append(extension);
     return path;
 }

es-app/src/scrapers/Scraper.h

@@ -81,6 +81,7 @@ struct ScraperSearchResult {
     std::string screenshotUrl;
     std::string titlescreenUrl;
     std::string videoUrl;
+    std::string manualUrl;
 
     // Needed to pre-set the image type.
     std::string box3DFormat;
@@ -92,6 +93,7 @@ struct ScraperSearchResult {
     std::string screenshotFormat;
     std::string titlescreenFormat;
     std::string videoFormat;
+    std::string manualFormat;
 
     // Indicates whether any new media files were downloaded and saved.
     bool savedNewMedia;

es-app/src/scrapers/ScreenScraper.cpp

@@ -266,7 +266,7 @@ void ScreenScraperRequest::process(const std::unique_ptr<HttpReq>& req,
 
         std::string err = ss.str();
         LOG(LogError) << err;
-        setError("ScreenScraper error: \n" + req->getContent());
+        setError("ScreenScraper error: \n" + req->getContent(), true);
 
         return;
     }
@@ -606,6 +606,9 @@ void ScreenScraperRequest::processGame(const pugi::xml_document& xmldoc,
             if (result.videoUrl == "")
                 processMedia(result, media_list, ssConfig.media_video_normalized, result.videoUrl,
                              result.videoFormat, region);
+            // Game manuals.
+            processMedia(result, media_list, ssConfig.media_manual, result.manualUrl,
+                         result.manualFormat, region);
         }
         result.mediaURLFetch = COMPLETED;
         out_results.emplace_back(result);

es-app/src/scrapers/ScreenScraper.h

@@ -97,6 +97,7 @@ public:
         std::string media_titlescreen = "sstitle";
         std::string media_video = "video";
         std::string media_video_normalized = "video-normalized";
+        std::string media_manual = "manuel";
 
         bool isArcadeSystem;
         bool automaticMode;

es-app/src/views/GamelistBase.cpp

@@ -668,6 +668,14 @@ void GamelistBase::removeMedia(FileData* game)
         removeEmptyDirFunc(systemMediaDir, mediaType, path);
     }
 
+    while (Utils::FileSystem::exists(game->getManualPath())) {
+        mediaType = "manuals";
+        path = game->getManualPath();
+        if (Utils::FileSystem::removeFile(path))
+            break;
+        removeEmptyDirFunc(systemMediaDir, mediaType, path);
+    }
+
     while (Utils::FileSystem::exists(game->getMiximagePath())) {
         mediaType = "miximages";
         path = game->getMiximagePath();

es-core/src/AsyncHandle.h

@@ -23,6 +23,7 @@ class AsyncHandle
 public:
     AsyncHandle()
         : mStatus(ASYNC_IN_PROGRESS)
+        , mRetry {true}
     {
     }
     virtual ~AsyncHandle() {}
@@ -36,6 +37,8 @@ public:
         return mStatus;
     }
 
+    const bool getRetry() { return mRetry; }
+
     // User-friendly string of our current status.
     // Will return error message if status() == SEARCH_ERROR.
     std::string getStatusString()
@@ -54,14 +57,16 @@ public:
 
 protected:
     void setStatus(AsyncHandleStatus status) { mStatus = status; }
-    void setError(const std::string& error)
+    void setError(const std::string& error, bool retry)
     {
         setStatus(ASYNC_ERROR);
         mError = error;
+        mRetry = retry;
     }
 
     std::string mError;
     AsyncHandleStatus mStatus;
+    bool mRetry;
 };
 
 #endif // ES_CORE_ASYNC_HANDLE_H

es-core/src/Settings.cpp

@@ -118,6 +118,7 @@ void Settings::setDefaults()
     mBoolMap["Scrape3DBoxes"] = {true, true};
     mBoolMap["ScrapePhysicalMedia"] = {true, true};
     mBoolMap["ScrapeFanArt"] = {true, true};
+    mBoolMap["ScrapeManuals"] = {false, false};
 
     mStringMap["MiximageResolution"] = {"1280x960", "1280x960"};
     mStringMap["MiximageScreenshotScaling"] = {"sharp", "sharp"};