Xargon/ES-DE
1
0
Fork 0
mirror of https://github.com/RetroDECK/ES-DE.git synced 2025-01-17 22:55:38 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Multi-scraper folder scraping and folder scraper exclusions added.

Browse source 
A quick-skip shortcut was added to the multi-scraper as well and some minor changes were done to the ScreenScraper debug logging.
This commit is contained in:
Leon Styhre 2020-08-06 15:12:04 +02:00
parent 2ee67407d8
commit 408be5120c
8 changed files with 71 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
es-app/src/FileData.cpp
View file

@ -345,6 +345,36 @@ std::vector<FileData*> FileData::getFilesRecursive(unsigned int typeMask,
return out;
}
std::vector<FileData*> FileData::getScrapeFilesRecursive(bool includeFolders,
bool excludeRecursively, bool respectExclusions) const
{
std::vector<FileData*> out;
for (auto it = mChildren.cbegin(); it != mChildren.cend(); it++) {
if (includeFolders && (*it)->getType() == FOLDER) {
if (!(respectExclusions && (*it)->getExcludeFromScraper()))
out.push_back(*it);
}
else if ((*it)->getType() == GAME) {
if (!(respectExclusions && (*it)->getExcludeFromScraper()))
out.push_back(*it);
}
// If the flag has been passed to exclude directories recursively, then skip the entire
// folder at this point if the folder is marked for scrape exclusion.
if (excludeRecursively && (*it)->getType() == FOLDER && (*it)->getExcludeFromScraper())
continue;
if ((*it)->getChildren().size() > 0) {
std::vector<FileData*> subChildren = (*it)->getScrapeFilesRecursive(
includeFolders, excludeRecursively, respectExclusions);
out.insert(out.cend(), subChildren.cbegin(), subChildren.cend());
}
}
return out;
}
std::string FileData::getKey() {
return getFileName();
}

2
es-app/src/FileData.h
View file

@ -80,6 +80,8 @@ public:
const std::vector<FileData*>& getChildrenListToDisplay();
std::vector<FileData*> getFilesRecursive(unsigned int typeMask,
bool displayedOnly = false, bool countAllGames = true) const;
std::vector<FileData*> getScrapeFilesRecursive(bool includeFolders, bool excludeRecursively,
bool respectExclusions) const;
void addChild(FileData* file); // Error if mType != FOLDER
void removeChild(FileData* file); //Error if mType != FOLDER

4
es-app/src/MetaData.cpp
View file

@ -28,7 +28,7 @@ MetaDataDecl gameDecls[] = {
{"hidden", MD_BOOL, "false", false, "hidden", "enter hidden off/on", false},
{"broken", MD_BOOL, "false", false, "broken/not working", "enter broken off/on", false},
{"nogamecount", MD_BOOL, "false", false, "exclude from game counter", "enter don't count as game off/on", false},
{"nomultiscrape", MD_BOOL, "false", false, "exclude from multi-scraper", "enter no autoscrape off/on", false},
{"nomultiscrape", MD_BOOL, "false", false, "exclude from multi-scraper", "enter no multi-scrape off/on", false},
{"launchcommand", MD_LAUNCHCOMMAND, "", false, "launch command", "enter game launch command "
"(emulator override)", false},
{"playcount", MD_INT, "0", false, "play count", "enter number of times played", false},
@ -51,7 +51,7 @@ MetaDataDecl folderDecls[] = {
{"completed", MD_BOOL, "false", false, "completed", "enter completed off/on", false},
{"hidden", MD_BOOL, "false", false, "hidden", "enter hidden off/on", false},
{"broken", MD_BOOL, "false", false, "broken/not working", "enter broken off/on", false},
{"nomultiscrape", MD_BOOL, "false", false, "exclude from multi-scraper", "enter no autoscrape off/on", false},
{"nomultiscrape", MD_BOOL, "false", false, "exclude from multi-scraper", "enter no multi-scrape off/on", false},
{"lastplayed", MD_TIME, "0", true, "last played", "enter last played date", false}
};
const std::vector<MetaDataDecl> folderMDD(folderDecls, folderDecls +

6
es-app/src/guis/GuiMetaDataEd.cpp
View file

@ -415,8 +415,10 @@ void GuiMetaDataEd::fetchDone(const ScraperSearchResult& result)
mEditors.at(i)->setColor(TEXTCOLOR_SCRAPERMARKED);
}
// Save all the keys, except the following which can't be scraped.
if (key != "favorite" && key != "completed" && key != "broken" &&
key != "hidden" && key != "kidgame")
if (key != "favorite" && key != "completed" && key != "kidgame" &&
key != "hidden" && key != "broken" && key != "nogamecount" &&
key != "nomultiscrape" && key != "nocontentscrape" &&
key != "nocontentscrape")
mEditors.at(i)->setValue(metadata->get(key));
}

29
es-app/src/guis/GuiScraperMenu.cpp
View file

@ -292,6 +292,24 @@ void GuiScraperMenu::openOtherSettings()
Settings::getInstance()->setBool("ScraperRespectExclusions",
scraper_respect_exclusions->getState()); });
// Exclude files recursively for excluded folders.
auto scraper_exclude_recursively = std::make_shared<SwitchComponent>(mWindow);
scraper_exclude_recursively->setState(
Settings::getInstance()->getBool("ScraperExcludeRecursively"));
s->addWithLabel("EXCLUDE FOLDERS RECURSIVELY", scraper_exclude_recursively);
s->addSaveFunc([scraper_exclude_recursively] {
Settings::getInstance()->setBool("ScraperExcludeRecursively",
scraper_exclude_recursively->getState()); });
// Include actual folders when scraping.
auto scraper_include_folders = std::make_shared<SwitchComponent>(mWindow);
scraper_include_folders->setState(
Settings::getInstance()->getBool("ScraperIncludeFolders"));
s->addWithLabel("SCRAPE ACTUAL FOLDERS", scraper_include_folders);
s->addSaveFunc([scraper_include_folders] {
Settings::getInstance()->setBool("ScraperIncludeFolders",
scraper_include_folders->getState()); });
mWindow->pushGui(s);
}
@ -337,13 +355,12 @@ std::queue<ScraperSearchParams> GuiScraperMenu::getSearches(
{
std::queue<ScraperSearchParams> queue;
for (auto sys = systems.cbegin(); sys != systems.cend(); sys++) {
std::vector<FileData*> games = (*sys)->getRootFolder()->getFilesRecursive(GAME);
std::vector<FileData*> games = (*sys)->getRootFolder()->getScrapeFilesRecursive(
Settings::getInstance()->getBool("ScraperIncludeFolders"),
Settings::getInstance()->getBool("ScraperExcludeRecursively"),
Settings::getInstance()->getBool("ScraperRespectExclusions"));
for (auto game = games.cbegin(); game != games.cend(); game++) {
// Skip scraping the game if the multi-scraper exclusion flag has been
// set for the file, and the flag to respect this value is also enabled.
bool skipGame = ((*game)->getExcludeFromScraper() &&
Settings::getInstance()->getBool("ScraperRespectExclusions") == true);
if (!skipGame && selector((*sys), (*game))) {
if (selector((*sys), (*game))) {
ScraperSearchParams search;
search.game = *game;
search.system = *sys;

4
es-app/src/guis/GuiScraperSearch.cpp
View file

@ -451,6 +451,10 @@ bool GuiScraperSearch::input(InputConfig* config, Input input)
return true;
}
// Quick-skip option activated by pressing 's' on the keyboard.
if (config->getDeviceId() == DEVICE_KEYBOARD && input.id == SDLK_s && input.value != 0)
mSkipCallback();
return GuiComponent::input(config, input);
}

7
es-app/src/scrapers/ScreenScraper.cpp
View file

@ -419,7 +419,8 @@ void ScreenScraperRequest::processMedia(
fileFormat = "." + media_type;
}
else {
LOG(LogDebug) << "Failed to find media XML node with name=" << mediaType;
LOG(LogDebug) << "ScreenScraperRequest::processMedia(): "
"Failed to find media XML node with name '" << mediaType << "'.";
}
}
@ -429,13 +430,13 @@ void ScreenScraperRequest::processList(const pugi::xml_document& xmldoc,
{
assert(mRequestQueue != nullptr);
LOG(LogDebug) << "ScreenScraper: Processing a list of results";
LOG(LogDebug) << "ScreenScraperRequest::processList(): Processing a list of results.";
pugi::xml_node data = xmldoc.child("Data");
pugi::xml_node game = data.child("jeu");
if (!game) {
LOG(LogDebug) << "ScreenScraper: Found nothing";
LOG(LogDebug) << "ScreenScraperRequest::processList(): Found nothing.";
}
ScreenScraperRequest::ScreenScraperConfig ssConfig;

2
es-core/src/Settings.cpp
View file

@ -155,6 +155,8 @@ void Settings::setDefaults()
mBoolMap["ScraperSemiautomatic"] = true;
mBoolMap["ScraperOverwriteData"] = true;
mBoolMap["ScraperRespectExclusions"] = true;
mBoolMap["ScraperExcludeRecursively"] = true;
mBoolMap["ScraperIncludeFolders"] = false;
mBoolMap["ScrapeMetadata"] = true;
mBoolMap["ScrapeGameNames"] = true;
mBoolMap["ScrapeRatings"] = true;