From 69852af751755cf0002607f8a3ad9d6641b038a7 Mon Sep 17 00:00:00 2001 From: Aloshi Date: Sun, 13 Oct 2013 14:07:48 -0500 Subject: [PATCH] Multi-game scraper seems to be functional! --- CMakeLists.txt | 2 + src/ScraperCmdLine.cpp | 6 +- src/Settings.cpp | 2 + src/XMLReader.cpp | 4 + src/components/GuiMetaDataEd.cpp | 4 +- src/components/GuiScraperLog.cpp | 158 +++++++++++++++++++++++++++++ src/components/GuiScraperLog.h | 39 +++++++ src/components/GuiScraperStart.cpp | 7 +- src/scrapers/Scraper.cpp | 33 +++++- src/scrapers/Scraper.h | 6 +- 10 files changed, 254 insertions(+), 7 deletions(-) create mode 100644 src/components/GuiScraperLog.cpp create mode 100644 src/components/GuiScraperLog.h diff --git a/CMakeLists.txt b/CMakeLists.txt index ab3020b15..56b6b4c8c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -184,6 +184,7 @@ set(ES_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/src/components/GuiMenu.h ${CMAKE_CURRENT_SOURCE_DIR}/src/components/GuiSettingsMenu.h ${CMAKE_CURRENT_SOURCE_DIR}/src/components/GuiScraperStart.h + ${CMAKE_CURRENT_SOURCE_DIR}/src/components/GuiScraperLog.h ${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/Scraper.h ${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/GamesDBScraper.h ${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/TheArchiveScraper.h @@ -241,6 +242,7 @@ set(ES_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/src/components/GuiMenu.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/components/GuiSettingsMenu.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/components/GuiScraperStart.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/components/GuiScraperLog.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/Scraper.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/GamesDBScraper.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/TheArchiveScraper.cpp diff --git a/src/ScraperCmdLine.cpp b/src/ScraperCmdLine.cpp index 1daac6b23..f62b64a03 100644 --- a/src/ScraperCmdLine.cpp +++ b/src/ScraperCmdLine.cpp @@ -255,7 +255,11 @@ int run_scraper_cmdline() if(url.length() != urlShort.length()) urlShort += "..."; out << " " << game->metadata()->get("name") << " [from: " << urlShort << "]...\n"; - game->metadata()->set(key, downloadImage(url, getSaveAsPath((*sysIt)->getName(), game->getCleanName(), url))); + + ScraperSearchParams p; + p.game = game; + p.system = *sysIt; + game->metadata()->set(key, downloadImage(url, getSaveAsPath(p, key, url))); if(game->metadata()->get(key).empty()) { out << " FAILED! Skipping.\n"; diff --git a/src/Settings.cpp b/src/Settings.cpp index d5d34bb16..89f28e312 100644 --- a/src/Settings.cpp +++ b/src/Settings.cpp @@ -33,6 +33,7 @@ void Settings::setDefaults() mBoolMap["DEBUG"] = false; mBoolMap["WINDOWED"] = false; mBoolMap["DISABLESOUNDS"] = false; + mBoolMap["DisableGamelistWrites"] = false; mIntMap["DIMTIME"] = 30*1000; mIntMap["ScraperResizeWidth"] = 450; @@ -40,6 +41,7 @@ void Settings::setDefaults() mIntMap["GameListSortIndex"] = 0; + mScraper = std::shared_ptr(new GamesDBScraper()); } diff --git a/src/XMLReader.cpp b/src/XMLReader.cpp index 80d83f3c2..4fa20d655 100644 --- a/src/XMLReader.cpp +++ b/src/XMLReader.cpp @@ -4,6 +4,7 @@ #include "pugiXML/pugixml.hpp" #include #include "Log.h" +#include "Settings.h" //this is obviously an incredibly inefficient way to go about searching //but I don't think it'll matter too much with the size of most collections @@ -193,6 +194,9 @@ void updateGamelist(SystemData* system) //We have the complete information for every game though, so we can simply remove a game //we already have in the system from the XML, and then add it back from its GameData information... + if(Settings::getInstance()->getBool("DisableGamelistWrites")) + return; + std::string xmlpath = system->getGamelistPath(); if(xmlpath.empty()) return; diff --git a/src/components/GuiMetaDataEd.cpp b/src/components/GuiMetaDataEd.cpp index 13831c943..61246dd06 100644 --- a/src/components/GuiMetaDataEd.cpp +++ b/src/components/GuiMetaDataEd.cpp @@ -126,6 +126,8 @@ void GuiMetaDataEd::fetch() void GuiMetaDataEd::fetchDone(MetaDataList result) { + //TODO - replace this with resolveMetaDataAssetsAsync! + //this is a little tricky: //go through the list of returned results, if anything is an image and the path looks like a URL: // (1) start an async download + resize (will create an AsyncReq that blocks further user input) @@ -139,7 +141,7 @@ void GuiMetaDataEd::fetchDone(MetaDataList result) //val is /probably/ a URL if(it->type == MD_IMAGE_PATH && HttpReq::isUrl(val)) { - downloadImageAsync(mWindow, val, getSaveAsPath(mScraperParams.system->getName(), mScraperParams.game->getCleanName() + "-" + key, val), + downloadImageAsync(mWindow, val, getSaveAsPath(mScraperParams, key, val), [this, result, key] (std::string filePath) mutable -> void { //skip it if(filePath.empty()) diff --git a/src/components/GuiScraperLog.cpp b/src/components/GuiScraperLog.cpp new file mode 100644 index 000000000..a768e3bc3 --- /dev/null +++ b/src/components/GuiScraperLog.cpp @@ -0,0 +1,158 @@ +#include "GuiScraperLog.h" +#include "../Settings.h" +#include "GuiGameScraper.h" +#include "../Renderer.h" +#include "../Log.h" + +GuiScraperLog::GuiScraperLog(Window* window, const std::queue& searches, bool manualMode) : GuiComponent(window), + mManualMode(manualMode), + mBox(window, ":/frame.png"), + mSearches(searches), + mStatus(window), + mTextLines(40) +{ + addChild(&mBox); + addChild(&mStatus); + + setSize(Renderer::getScreenWidth() * 0.8f, (float)Renderer::getScreenHeight()); + setPosition((Renderer::getScreenWidth() - mSize.x()) / 2, 0); + + mStatus.setColor(0x000000FF); + mStatus.setSize(mSize.x(), (float)mStatus.getFont()->getHeight()); + mStatus.setCentered(true); + updateStatus(); + + mBox.setEdgeColor(0x111111FF); + mBox.setCenterColor(0xDFDFDFFF); + mBox.fitTo(mSize, Eigen::Vector3f::Zero(), Eigen::Vector2f(8, 0)); +} + +void GuiScraperLog::start() +{ + next(); +} + +void GuiScraperLog::next() +{ + if(mSearches.empty()) + { + done(); + return; + } + + ScraperSearchParams search = mSearches.front(); + mSearches.pop(); + + writeLine(search.game->getCleanName(), 0x0000FFFF); + + if(mManualMode) + { + GuiGameScraper* ggs = new GuiGameScraper(mWindow, search, + [this, search] (MetaDataList result) { resultFetched(search, result); }, + [this, search] { resultEmpty(search); }); + + mWindow->pushGui(ggs); + ggs->search(); + }else{ + std::shared_ptr scraper = Settings::getInstance()->getScraper(); + scraper->getResultsAsync(search, mWindow, [this, search] (std::vector mdls) { + if(mdls.empty()) + resultEmpty(search); + else + resultFetched(search, mdls[0]); + }); + } + + updateStatus(); +} + +void GuiScraperLog::resultFetched(ScraperSearchParams params, MetaDataList mdl) +{ + writeLine(" -> \"" + mdl.get("name") + "\"", 0x0000FFFF); + writeLine(" Downloading images...", 0x0000FFFF); + resolveMetaDataAssetsAsync(mWindow, params, mdl, [this, params] (MetaDataList meta) { resultResolved(params, meta); }); + + //-> resultResolved +} + +void GuiScraperLog::resultResolved(ScraperSearchParams params, MetaDataList mdl) +{ + writeLine(" Success!", 0x00FF00FF); + + next(); +} + +void GuiScraperLog::resultEmpty(ScraperSearchParams search) +{ + if(mManualMode) + writeLine(" SKIPPING", 0xFF0000FF); + else + writeLine(" NO RESULTS, skipping", 0xFF0000FF); + + LOG(LogInfo) << "Scraper skipping [" << search.game->getCleanName() << "]"; + + next(); +} + +void GuiScraperLog::done() +{ + writeLine("====================", 0x000000FF); + writeLine("DONE!!", 0x00FF00FF); + writeLine("====================", 0x000000FF); + + //done with everything! +} + +void GuiScraperLog::writeLine(const std::string& line, unsigned int color) +{ + std::shared_ptr cmp(new TextComponent(mWindow)); + cmp->setText(line); + cmp->setColor(color); + cmp->setSize(mSize.x(), (float)cmp->getFont()->getHeight()); + + mTextLines.push_back(cmp); +} + +void GuiScraperLog::render(const Eigen::Affine3f& parentTrans) +{ + renderChildren(parentTrans * getTransform()); + + Eigen::Affine3f trans = parentTrans * getTransform(); + + //draw messages + float fontHeight = (float)Font::get(FONT_SIZE_MEDIUM)->getHeight(); + trans = trans.translate(Eigen::Vector3f(0, mSize.y() - fontHeight, 0)); + + for(auto it = mTextLines.rbegin(); it != mTextLines.rend(); it++) + { + (*it)->render(trans); + trans = trans.translate(Eigen::Vector3f(0, -fontHeight, 0)); + + if(trans.translation().y() < fontHeight) + break; + } +} + +bool GuiScraperLog::input(InputConfig* config, Input input) +{ + if(input.value != 0) + { + //we're done + if(mSearches.empty()) + { + delete this; + return true; + } + } + + return GuiComponent::input(config, input); +} + +void GuiScraperLog::updateStatus() +{ + std::stringstream ss; + + ss << mSearches.size() << " games remaining"; + + mStatus.setText(ss.str()); +} diff --git a/src/components/GuiScraperLog.h b/src/components/GuiScraperLog.h new file mode 100644 index 000000000..04a7d14cc --- /dev/null +++ b/src/components/GuiScraperLog.h @@ -0,0 +1,39 @@ +#pragma once + +#include "../GuiComponent.h" +#include "NinePatchComponent.h" +#include +#include "../scrapers/Scraper.h" +#include +#include "TextComponent.h" + +class GuiScraperLog : public GuiComponent +{ +public: + GuiScraperLog(Window* window, const std::queue& params, bool manualMode); + + void start(); + + void render(const Eigen::Affine3f& parentTrans) override; + bool input(InputConfig* config, Input input) override; + +private: + void updateStatus(); + void writeLine(const std::string& line, unsigned int color); + + void resultFetched(ScraperSearchParams params, MetaDataList mdl); + void resultResolved(ScraperSearchParams params, MetaDataList mdl); + void resultEmpty(ScraperSearchParams params); + + void next(); + void done(); + + bool mManualMode; + + NinePatchComponent mBox; + + std::queue mSearches; + + TextComponent mStatus; + boost::circular_buffer< std::shared_ptr > mTextLines; +}; diff --git a/src/components/GuiScraperStart.cpp b/src/components/GuiScraperStart.cpp index 6dd3e9bef..a54eef47f 100644 --- a/src/components/GuiScraperStart.cpp +++ b/src/components/GuiScraperStart.cpp @@ -1,5 +1,5 @@ #include "GuiScraperStart.h" -#include "GuiMsgBoxOk.h" +#include "GuiScraperLog.h" GuiScraperStart::GuiScraperStart(Window* window) : GuiComponent(window), mBox(window, ":/frame.png"), @@ -57,7 +57,10 @@ void GuiScraperStart::start() { std::queue searches = getSearches(mSystemsOpt.getSelectedObjects(), mFiltersOpt.getSelectedObjects()[0]); - mWindow->pushGui(new GuiMsgBoxOk(mWindow, "this isn't implemented yet")); + GuiScraperLog* gsl = new GuiScraperLog(mWindow, searches, mManualSwitch.getState()); + mWindow->pushGui(gsl); + gsl->start(); + delete this; } std::queue GuiScraperStart::getSearches(std::vector systems, GameFilterFunc selector) diff --git a/src/scrapers/Scraper.cpp b/src/scrapers/Scraper.cpp index 5e4837e0d..8f8058ac6 100644 --- a/src/scrapers/Scraper.cpp +++ b/src/scrapers/Scraper.cpp @@ -140,8 +140,11 @@ std::string downloadImage(const std::string& url, const std::string& saveAs) return file; } -std::string getSaveAsPath(const std::string& subdirectory, const std::string& name, const std::string& url) +std::string getSaveAsPath(const ScraperSearchParams& params, const std::string& suffix, const std::string& url) { + const std::string subdirectory = params.system->getName(); + const std::string name = params.game->getCleanName() + "-" + suffix; + std::string path = getHomePath() + "/.emulationstation/downloaded_images/"; if(!boost::filesystem::exists(path)) @@ -171,3 +174,31 @@ std::shared_ptr createScraperByName(const std::string& name) return nullptr; } + +void resolveMetaDataAssetsAsync(Window* window, const ScraperSearchParams& params, MetaDataList mdl, std::function returnFunc) +{ + std::vector mdd = params.system->getGameMDD(); + for(auto it = mdd.begin(); it != mdd.end(); it++) + { + std::string key = it->key; + std::string val = mdl.get(key); + if(it->type == MD_IMAGE_PATH && HttpReq::isUrl(val)) + { + downloadImageAsync(window, val, getSaveAsPath(params, key, val), [window, params, mdl, key, returnFunc] (std::string savedAs) mutable -> + void + { + if(savedAs.empty()) + { + //error + LOG(LogError) << "Could not resolve image for [" << params.game->getCleanName() << "]! Skipping."; + } + + mdl.set(key, savedAs); + resolveMetaDataAssetsAsync(window, params, mdl, returnFunc); + }); + return; + } + } + + returnFunc(mdl); +} diff --git a/src/scrapers/Scraper.h b/src/scrapers/Scraper.h index 9243f6396..854aa3a11 100644 --- a/src/scrapers/Scraper.h +++ b/src/scrapers/Scraper.h @@ -32,9 +32,9 @@ private: std::shared_ptr createScraperByName(const std::string& name); -//About the same as "~/.emulationstation/downloaded_images/[subdirectory]/[name].[url's extension]". +//About the same as "~/.emulationstation/downloaded_images/[system_name]/[game_name].[url's extension]". //Will create the "downloaded_images" and "subdirectory" directories if they do not exist. -std::string getSaveAsPath(const std::string& subdirectory, const std::string& name, const std::string& url); +std::string getSaveAsPath(const ScraperSearchParams& params, const std::string& suffix, const std::string& url); //Returns the path to the downloaded file (saveAs) on completion. //Returns empty string if an error occured. @@ -47,6 +47,8 @@ std::string downloadImage(const std::string& url, const std::string& saveAs); //Same as downloadImage, just async. void downloadImageAsync(Window* window, const std::string& url, const std::string& saveAs, std::function returnFunc); +void resolveMetaDataAssetsAsync(Window* window, const ScraperSearchParams& params, MetaDataList mdl, std::function returnFunc); + //You can pass 0 for maxWidth or maxHeight to automatically keep the aspect ratio. //Will overwrite the image at [path] with the new resized one. void resizeImage(const std::string& path, int maxWidth, int maxHeight);