From 408be5120cb3dc0959d660f2ec541006ee12745e Mon Sep 17 00:00:00 2001 From: Leon Styhre Date: Thu, 6 Aug 2020 15:12:04 +0200 Subject: [PATCH] Multi-scraper folder scraping and folder scraper exclusions added. A quick-skip shortcut was added to the multi-scraper as well and some minor changes were done to the ScreenScraper debug logging. --- es-app/src/FileData.cpp | 30 +++++++++++++++++++++++++++ es-app/src/FileData.h | 2 ++ es-app/src/MetaData.cpp | 4 ++-- es-app/src/guis/GuiMetaDataEd.cpp | 6 ++++-- es-app/src/guis/GuiScraperMenu.cpp | 29 ++++++++++++++++++++------ es-app/src/guis/GuiScraperSearch.cpp | 4 ++++ es-app/src/scrapers/ScreenScraper.cpp | 7 ++++--- es-core/src/Settings.cpp | 2 ++ 8 files changed, 71 insertions(+), 13 deletions(-) diff --git a/es-app/src/FileData.cpp b/es-app/src/FileData.cpp index aa19a09f8..2090cf869 100644 --- a/es-app/src/FileData.cpp +++ b/es-app/src/FileData.cpp @@ -345,6 +345,36 @@ std::vector FileData::getFilesRecursive(unsigned int typeMask, return out; } +std::vector FileData::getScrapeFilesRecursive(bool includeFolders, + bool excludeRecursively, bool respectExclusions) const +{ + std::vector out; + + for (auto it = mChildren.cbegin(); it != mChildren.cend(); it++) { + if (includeFolders && (*it)->getType() == FOLDER) { + if (!(respectExclusions && (*it)->getExcludeFromScraper())) + out.push_back(*it); + } + else if ((*it)->getType() == GAME) { + if (!(respectExclusions && (*it)->getExcludeFromScraper())) + out.push_back(*it); + } + + // If the flag has been passed to exclude directories recursively, then skip the entire + // folder at this point if the folder is marked for scrape exclusion. + if (excludeRecursively && (*it)->getType() == FOLDER && (*it)->getExcludeFromScraper()) + continue; + + if ((*it)->getChildren().size() > 0) { + std::vector subChildren = (*it)->getScrapeFilesRecursive( + includeFolders, excludeRecursively, respectExclusions); + out.insert(out.cend(), subChildren.cbegin(), subChildren.cend()); + } + } + + return out; +} + std::string FileData::getKey() { return getFileName(); } diff --git a/es-app/src/FileData.h b/es-app/src/FileData.h index a071f9e97..bcb1e2664 100644 --- a/es-app/src/FileData.h +++ b/es-app/src/FileData.h @@ -80,6 +80,8 @@ public: const std::vector& getChildrenListToDisplay(); std::vector getFilesRecursive(unsigned int typeMask, bool displayedOnly = false, bool countAllGames = true) const; + std::vector getScrapeFilesRecursive(bool includeFolders, bool excludeRecursively, + bool respectExclusions) const; void addChild(FileData* file); // Error if mType != FOLDER void removeChild(FileData* file); //Error if mType != FOLDER diff --git a/es-app/src/MetaData.cpp b/es-app/src/MetaData.cpp index ca3afa6b1..3399dde5d 100644 --- a/es-app/src/MetaData.cpp +++ b/es-app/src/MetaData.cpp @@ -28,7 +28,7 @@ MetaDataDecl gameDecls[] = { {"hidden", MD_BOOL, "false", false, "hidden", "enter hidden off/on", false}, {"broken", MD_BOOL, "false", false, "broken/not working", "enter broken off/on", false}, {"nogamecount", MD_BOOL, "false", false, "exclude from game counter", "enter don't count as game off/on", false}, -{"nomultiscrape", MD_BOOL, "false", false, "exclude from multi-scraper", "enter no autoscrape off/on", false}, +{"nomultiscrape", MD_BOOL, "false", false, "exclude from multi-scraper", "enter no multi-scrape off/on", false}, {"launchcommand", MD_LAUNCHCOMMAND, "", false, "launch command", "enter game launch command " "(emulator override)", false}, {"playcount", MD_INT, "0", false, "play count", "enter number of times played", false}, @@ -51,7 +51,7 @@ MetaDataDecl folderDecls[] = { {"completed", MD_BOOL, "false", false, "completed", "enter completed off/on", false}, {"hidden", MD_BOOL, "false", false, "hidden", "enter hidden off/on", false}, {"broken", MD_BOOL, "false", false, "broken/not working", "enter broken off/on", false}, -{"nomultiscrape", MD_BOOL, "false", false, "exclude from multi-scraper", "enter no autoscrape off/on", false}, +{"nomultiscrape", MD_BOOL, "false", false, "exclude from multi-scraper", "enter no multi-scrape off/on", false}, {"lastplayed", MD_TIME, "0", true, "last played", "enter last played date", false} }; const std::vector folderMDD(folderDecls, folderDecls + diff --git a/es-app/src/guis/GuiMetaDataEd.cpp b/es-app/src/guis/GuiMetaDataEd.cpp index d961f665c..bf9ce9edb 100644 --- a/es-app/src/guis/GuiMetaDataEd.cpp +++ b/es-app/src/guis/GuiMetaDataEd.cpp @@ -415,8 +415,10 @@ void GuiMetaDataEd::fetchDone(const ScraperSearchResult& result) mEditors.at(i)->setColor(TEXTCOLOR_SCRAPERMARKED); } // Save all the keys, except the following which can't be scraped. - if (key != "favorite" && key != "completed" && key != "broken" && - key != "hidden" && key != "kidgame") + if (key != "favorite" && key != "completed" && key != "kidgame" && + key != "hidden" && key != "broken" && key != "nogamecount" && + key != "nomultiscrape" && key != "nocontentscrape" && + key != "nocontentscrape") mEditors.at(i)->setValue(metadata->get(key)); } diff --git a/es-app/src/guis/GuiScraperMenu.cpp b/es-app/src/guis/GuiScraperMenu.cpp index 9ed512c7b..a9bee62d2 100644 --- a/es-app/src/guis/GuiScraperMenu.cpp +++ b/es-app/src/guis/GuiScraperMenu.cpp @@ -292,6 +292,24 @@ void GuiScraperMenu::openOtherSettings() Settings::getInstance()->setBool("ScraperRespectExclusions", scraper_respect_exclusions->getState()); }); + // Exclude files recursively for excluded folders. + auto scraper_exclude_recursively = std::make_shared(mWindow); + scraper_exclude_recursively->setState( + Settings::getInstance()->getBool("ScraperExcludeRecursively")); + s->addWithLabel("EXCLUDE FOLDERS RECURSIVELY", scraper_exclude_recursively); + s->addSaveFunc([scraper_exclude_recursively] { + Settings::getInstance()->setBool("ScraperExcludeRecursively", + scraper_exclude_recursively->getState()); }); + + // Include actual folders when scraping. + auto scraper_include_folders = std::make_shared(mWindow); + scraper_include_folders->setState( + Settings::getInstance()->getBool("ScraperIncludeFolders")); + s->addWithLabel("SCRAPE ACTUAL FOLDERS", scraper_include_folders); + s->addSaveFunc([scraper_include_folders] { + Settings::getInstance()->setBool("ScraperIncludeFolders", + scraper_include_folders->getState()); }); + mWindow->pushGui(s); } @@ -337,13 +355,12 @@ std::queue GuiScraperMenu::getSearches( { std::queue queue; for (auto sys = systems.cbegin(); sys != systems.cend(); sys++) { - std::vector games = (*sys)->getRootFolder()->getFilesRecursive(GAME); + std::vector games = (*sys)->getRootFolder()->getScrapeFilesRecursive( + Settings::getInstance()->getBool("ScraperIncludeFolders"), + Settings::getInstance()->getBool("ScraperExcludeRecursively"), + Settings::getInstance()->getBool("ScraperRespectExclusions")); for (auto game = games.cbegin(); game != games.cend(); game++) { - // Skip scraping the game if the multi-scraper exclusion flag has been - // set for the file, and the flag to respect this value is also enabled. - bool skipGame = ((*game)->getExcludeFromScraper() && - Settings::getInstance()->getBool("ScraperRespectExclusions") == true); - if (!skipGame && selector((*sys), (*game))) { + if (selector((*sys), (*game))) { ScraperSearchParams search; search.game = *game; search.system = *sys; diff --git a/es-app/src/guis/GuiScraperSearch.cpp b/es-app/src/guis/GuiScraperSearch.cpp index 13a6517b2..149527e38 100644 --- a/es-app/src/guis/GuiScraperSearch.cpp +++ b/es-app/src/guis/GuiScraperSearch.cpp @@ -451,6 +451,10 @@ bool GuiScraperSearch::input(InputConfig* config, Input input) return true; } + // Quick-skip option activated by pressing 's' on the keyboard. + if (config->getDeviceId() == DEVICE_KEYBOARD && input.id == SDLK_s && input.value != 0) + mSkipCallback(); + return GuiComponent::input(config, input); } diff --git a/es-app/src/scrapers/ScreenScraper.cpp b/es-app/src/scrapers/ScreenScraper.cpp index cf91b71c4..962fb9278 100644 --- a/es-app/src/scrapers/ScreenScraper.cpp +++ b/es-app/src/scrapers/ScreenScraper.cpp @@ -419,7 +419,8 @@ void ScreenScraperRequest::processMedia( fileFormat = "." + media_type; } else { - LOG(LogDebug) << "Failed to find media XML node with name=" << mediaType; + LOG(LogDebug) << "ScreenScraperRequest::processMedia(): " + "Failed to find media XML node with name '" << mediaType << "'."; } } @@ -429,13 +430,13 @@ void ScreenScraperRequest::processList(const pugi::xml_document& xmldoc, { assert(mRequestQueue != nullptr); - LOG(LogDebug) << "ScreenScraper: Processing a list of results"; + LOG(LogDebug) << "ScreenScraperRequest::processList(): Processing a list of results."; pugi::xml_node data = xmldoc.child("Data"); pugi::xml_node game = data.child("jeu"); if (!game) { - LOG(LogDebug) << "ScreenScraper: Found nothing"; + LOG(LogDebug) << "ScreenScraperRequest::processList(): Found nothing."; } ScreenScraperRequest::ScreenScraperConfig ssConfig; diff --git a/es-core/src/Settings.cpp b/es-core/src/Settings.cpp index 9c343b981..c408053e8 100644 --- a/es-core/src/Settings.cpp +++ b/es-core/src/Settings.cpp @@ -155,6 +155,8 @@ void Settings::setDefaults() mBoolMap["ScraperSemiautomatic"] = true; mBoolMap["ScraperOverwriteData"] = true; mBoolMap["ScraperRespectExclusions"] = true; + mBoolMap["ScraperExcludeRecursively"] = true; + mBoolMap["ScraperIncludeFolders"] = false; mBoolMap["ScrapeMetadata"] = true; mBoolMap["ScrapeGameNames"] = true; mBoolMap["ScrapeRatings"] = true;