From dbde9006294bedabbf948a8c938ef0c96b4b5605 Mon Sep 17 00:00:00 2001 From: Aloshi Date: Tue, 18 Mar 2014 16:05:56 -0500 Subject: [PATCH] Redid Scrapers to return ScraperSearchHandles for async searches. This allows for much better error handling and doesn't take over the UI. Redid GuiScraperLog to fit new UI concept. --- CMakeLists.txt | 4 +- src/ScraperCmdLine.cpp | 5 + src/components/ButtonComponent.cpp | 3 +- src/components/MenuComponent.cpp | 5 +- src/components/ScraperSearchComponent.cpp | 93 +++++++---- src/components/ScraperSearchComponent.h | 24 ++- src/guis/GuiGameScraper.cpp | 66 ++------ src/guis/GuiGameScraper.h | 10 +- src/guis/GuiMetaDataEd.cpp | 32 +--- src/guis/GuiMetaDataEd.h | 2 +- src/guis/GuiScraperLog.cpp | 181 ---------------------- src/guis/GuiScraperLog.h | 46 ------ src/guis/GuiScraperMulti.cpp | 102 ++++++++++++ src/guis/GuiScraperMulti.h | 39 +++++ src/guis/GuiScraperStart.cpp | 11 +- src/scrapers/GamesDBScraper.cpp | 65 +++++--- src/scrapers/GamesDBScraper.h | 17 +- src/scrapers/Scraper.cpp | 32 ++-- src/scrapers/Scraper.h | 46 +++++- src/scrapers/TheArchiveScraper.cpp | 63 +++++--- src/scrapers/TheArchiveScraper.h | 20 ++- 21 files changed, 422 insertions(+), 444 deletions(-) delete mode 100644 src/guis/GuiScraperLog.cpp delete mode 100644 src/guis/GuiScraperLog.h create mode 100644 src/guis/GuiScraperMulti.cpp create mode 100644 src/guis/GuiScraperMulti.h diff --git a/CMakeLists.txt b/CMakeLists.txt index b0eecf5e6..09fe248a6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -189,8 +189,8 @@ set(ES_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/src/guis/GuiInputConfig.h ${CMAKE_CURRENT_SOURCE_DIR}/src/guis/GuiMenu.h ${CMAKE_CURRENT_SOURCE_DIR}/src/guis/GuiSettings.h + ${CMAKE_CURRENT_SOURCE_DIR}/src/guis/GuiScraperMulti.h ${CMAKE_CURRENT_SOURCE_DIR}/src/guis/GuiScraperStart.h - ${CMAKE_CURRENT_SOURCE_DIR}/src/guis/GuiScraperLog.h ${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/Scraper.h ${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/GamesDBScraper.h @@ -269,8 +269,8 @@ set(ES_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/src/guis/GuiInputConfig.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/guis/GuiMenu.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/guis/GuiSettings.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/guis/GuiScraperMulti.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/guis/GuiScraperStart.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/src/guis/GuiScraperLog.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/Scraper.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/GamesDBScraper.cpp diff --git a/src/ScraperCmdLine.cpp b/src/ScraperCmdLine.cpp index 4fd535e78..f6f695252 100644 --- a/src/ScraperCmdLine.cpp +++ b/src/ScraperCmdLine.cpp @@ -138,6 +138,7 @@ int run_scraper_cmdline() out << "Alright, let's do this thing!\n"; out << "=============================\n"; + /* std::shared_ptr scraper = Settings::getInstance()->getScraper(); for(auto sysIt = systems.begin(); sysIt != systems.end(); sysIt++) { @@ -275,6 +276,10 @@ int run_scraper_cmdline() out << "==============================\n"; out << "SCRAPE COMPLETE!\n"; out << "==============================\n"; + */ + + out << "\n\n"; + out << "ACTUALLY THIS IS STILL TODO\n"; return 0; } diff --git a/src/components/ButtonComponent.cpp b/src/components/ButtonComponent.cpp index 2336e3698..3a914113e 100644 --- a/src/components/ButtonComponent.cpp +++ b/src/components/ButtonComponent.cpp @@ -13,6 +13,7 @@ ButtonComponent::ButtonComponent(Window* window, const std::string& text, const { setPressedFunc(func); setText(text, helpText); + updateImage(); } void ButtonComponent::onSizeChanged() @@ -71,7 +72,7 @@ void ButtonComponent::updateImage() { if(!mEnabled || !mPressedFunc) { - mBox.setImagePath(":/button.png"); + mBox.setImagePath(":/button_filled.png"); mBox.setCenterColor(0x770000FF); mBox.setEdgeColor(0x770000FF); return; diff --git a/src/components/MenuComponent.cpp b/src/components/MenuComponent.cpp index 06f706c74..59b4187fa 100644 --- a/src/components/MenuComponent.cpp +++ b/src/components/MenuComponent.cpp @@ -59,13 +59,12 @@ void MenuComponent::updateGrid() if(mButtonGrid) mGrid.removeEntry(mButtonGrid); + mButtonGrid.reset(); + if(mButtons.size()) { mButtonGrid = makeButtonGrid(mWindow, mButtons); - mGrid.setEntry(mButtonGrid, Vector2i(0, 2), true, false); - }else{ - mButtonGrid.reset(); } } diff --git a/src/components/ScraperSearchComponent.cpp b/src/components/ScraperSearchComponent.cpp index 8974df44f..d87414bfb 100644 --- a/src/components/ScraperSearchComponent.cpp +++ b/src/components/ScraperSearchComponent.cpp @@ -1,9 +1,10 @@ #include "ScraperSearchComponent.h" -#include "../components/TextComponent.h" -#include "../components/ScrollableContainer.h" -#include "../components/ImageComponent.h" -#include "../components/ComponentList.h" +#include "../guis/GuiMsgBox.h" +#include "TextComponent.h" +#include "ScrollableContainer.h" +#include "ImageComponent.h" +#include "ComponentList.h" #include "../HttpReq.h" #include "../Settings.h" #include "../Log.h" @@ -12,9 +13,6 @@ ScraperSearchComponent::ScraperSearchComponent(Window* window, SearchType type) mGrid(window, Eigen::Vector2i(4, 3)), mSearchType(type) { - mSearchParams.system = NULL; - mSearchParams.game = NULL; - addChild(&mGrid); using namespace Eigen; @@ -85,18 +83,17 @@ void ScraperSearchComponent::updateViewStyle() } } -void ScraperSearchComponent::setSearchParams(const ScraperSearchParams& params) +void ScraperSearchComponent::search(const ScraperSearchParams& params) { - mSearchParams = params; - search(); + mResultList->clear(); + mScraperResults.clear(); + updateInfoPane(); + + mLastSearch = params; + mSearchHandle = Settings::getInstance()->getScraper()->getResultsAsync(params); } -void ScraperSearchComponent::search() -{ - Settings::getInstance()->getScraper()->getResultsAsync(mSearchParams, mWindow, std::bind(&ScraperSearchComponent::onSearchReceived, this, std::placeholders::_1)); -} - -void ScraperSearchComponent::onSearchReceived(std::vector results) +void ScraperSearchComponent::onSearchDone(const std::vector& results) { mResultList->clear(); @@ -117,7 +114,7 @@ void ScraperSearchComponent::onSearchReceived(std::vector results) for(int i = 0; i < end; i++) { row.elements.clear(); - row.addElement(std::make_shared(mWindow, results.at(i).get("name"), font, color), true); + row.addElement(std::make_shared(mWindow, results.at(i).mdl.get("name"), font, color), true); mResultList->addRow(row); } mGrid.resetCursor(); @@ -128,16 +125,23 @@ void ScraperSearchComponent::onSearchReceived(std::vector results) if(mSearchType == ALWAYS_ACCEPT_FIRST_RESULT) { if(mScraperResults.size() == 0) - returnResult(NULL); + mSkipCallback(); else - returnResult(&mScraperResults.front()); + returnResult(mScraperResults.front()); }else if(mSearchType == ALWAYS_ACCEPT_MATCHING_CRC) { // TODO - assert(false); } } +void ScraperSearchComponent::onSearchError(const std::string& error) +{ + mWindow->pushGui(new GuiMsgBox(mWindow, error, + "RETRY", std::bind(&ScraperSearchComponent::search, this, mLastSearch), + "SKIP", mSkipCallback, + "CANCEL", mCancelCallback)); +} + int ScraperSearchComponent::getSelectedIndex() { if(mScraperResults.size() && mGrid.getSelectedComponent() != mResultList) @@ -151,17 +155,21 @@ void ScraperSearchComponent::updateInfoPane() int i = getSelectedIndex(); if(i != -1 && (int)mScraperResults.size() > i) { - mResultName->setText(mScraperResults.at(i).get("name")); - mResultDesc->setText(mScraperResults.at(i).get("desc")); + mResultName->setText(mScraperResults.at(i).mdl.get("name")); + mResultDesc->setText(mScraperResults.at(i).mdl.get("desc")); mDescContainer->setScrollPos(Eigen::Vector2d(0, 0)); mDescContainer->resetAutoScrollTimer(); - std::string thumb = mScraperResults.at(i).get("thumbnail"); mResultThumbnail->setImage(""); + const std::string& thumb = mScraperResults.at(i).thumbnailUrl; if(!thumb.empty()) mThumbnailReq = std::unique_ptr(new HttpReq(thumb)); else mThumbnailReq.reset(); + }else{ + mResultName->setText(" "); + mResultDesc->setText(" "); + mResultThumbnail->setImage(""); } } @@ -172,7 +180,7 @@ bool ScraperSearchComponent::input(InputConfig* config, Input input) //if you're on a result if(getSelectedIndex() != -1) { - returnResult(&mScraperResults.at(getSelectedIndex())); + returnResult(mScraperResults.at(getSelectedIndex())); return true; } } @@ -187,9 +195,27 @@ bool ScraperSearchComponent::input(InputConfig* config, Input input) return ret; } -void ScraperSearchComponent::returnResult(MetaDataList* result) +void ScraperSearchComponent::returnResult(ScraperSearchResult result) { - assert(mAcceptCallback); + // resolve metadata image before returning + if(!result.imageUrl.empty()) + { + downloadImageAsync(mWindow, result.imageUrl, getSaveAsPath(mLastSearch, "image", result.imageUrl), + [this, result] (std::string filePath) mutable -> void + { + if(filePath.empty()) + { + onSearchError("Error downloading boxart."); + return; + } + + result.mdl.set("image", filePath); + result.imageUrl = ""; + this->returnResult(result); // re-enter this function + }); + return; + } + mAcceptCallback(result); } @@ -200,6 +226,19 @@ void ScraperSearchComponent::update(int deltaTime) updateThumbnail(); } + if(mSearchHandle && mSearchHandle->status() != SEARCH_IN_PROGRESS) + { + if(mSearchHandle->status() == SEARCH_DONE) + { + onSearchDone(mSearchHandle->getResults()); + }else if(mSearchHandle->status() == SEARCH_ERROR) + { + onSearchError(mSearchHandle->getStatusString()); + } + + mSearchHandle.reset(); + } + GuiComponent::update(deltaTime); } @@ -235,4 +274,4 @@ void ScraperSearchComponent::onFocusGained() void ScraperSearchComponent::onFocusLost() { mGrid.onFocusLost(); -} \ No newline at end of file +} diff --git a/src/components/ScraperSearchComponent.h b/src/components/ScraperSearchComponent.h index 09a7bf221..1e6b134a1 100644 --- a/src/components/ScraperSearchComponent.h +++ b/src/components/ScraperSearchComponent.h @@ -25,8 +25,12 @@ public: ScraperSearchComponent(Window* window, SearchType searchType = NEVER_AUTO_ACCEPT); - void setSearchParams(const ScraperSearchParams& params); - inline void setAcceptCallback(const std::function& acceptCallback) { mAcceptCallback = acceptCallback; } + void search(const ScraperSearchParams& params); + + // Metadata assets will be resolved before calling the accept callback (e.g. result.mdl's "image" is automatically downloaded and properly set). + inline void setAcceptCallback(const std::function& acceptCallback) { mAcceptCallback = acceptCallback; } + inline void setSkipCallback(const std::function& skipCallback) { mSkipCallback = skipCallback; }; + inline void setCancelCallback(const std::function& cancelCallback) { mCancelCallback = cancelCallback; } bool input(InputConfig* config, Input input) override; void update(int deltaTime) override; @@ -40,12 +44,13 @@ private: void updateThumbnail(); void updateInfoPane(); - void search(); - void onSearchReceived(std::vector results); + void onSearchError(const std::string& error); + void onSearchDone(const std::vector& results); int getSelectedIndex(); - void returnResult(MetaDataList* result); + // resolve any metadata assets that need to be downloaded and return + void returnResult(ScraperSearchResult result); ComponentGrid mGrid; @@ -56,9 +61,12 @@ private: std::shared_ptr mResultList; SearchType mSearchType; - ScraperSearchParams mSearchParams; - std::function mAcceptCallback; + ScraperSearchParams mLastSearch; + std::function mAcceptCallback; + std::function mSkipCallback; + std::function mCancelCallback; - std::vector mScraperResults; + std::unique_ptr mSearchHandle; + std::vector mScraperResults; std::unique_ptr mThumbnailReq; }; diff --git a/src/guis/GuiGameScraper.cpp b/src/guis/GuiGameScraper.cpp index e9ec0734a..7ac7dcf6f 100644 --- a/src/guis/GuiGameScraper.cpp +++ b/src/guis/GuiGameScraper.cpp @@ -6,31 +6,13 @@ #include "../components/TextComponent.h" #include "../components/ButtonComponent.h" +#include "../components/MenuComponent.h" -GuiGameScraper::GuiGameScraper(Window* window, ScraperSearchParams params, std::function doneFunc, std::function skipFunc) : GuiComponent(window), +GuiGameScraper::GuiGameScraper(Window* window, ScraperSearchParams params, std::function doneFunc) : GuiComponent(window), mGrid(window, Eigen::Vector2i(1, 3)), mBox(window, ":/frame.png"), - mSearchParams(params), - mDoneFunc(doneFunc), - mSkipFunc(skipFunc), - mSearchCountdown(2) + mSearchParams(params) { - // new screen: - - // FILE NAME - //-------------------------------------- - // Result Title | Result #1 - // |-------| ..... | Result #2 - // | IMG | info | Result #3 - // |-------| ..... | ......... - // | ......... - // DESCRIPTION........| ......... - // ...................| ......... - // ...................| ......... - //-------------------------------------- - // [SEARCH NAME] [CANCEL] - - addChild(&mBox); addChild(&mGrid); @@ -52,17 +34,10 @@ GuiGameScraper::GuiGameScraper(Window* window, ScraperSearchParams params, std:: mGrid.setEntry(mSearch, Eigen::Vector2i(0, 1), true); // buttons - auto buttonGrid = std::make_shared(mWindow, Eigen::Vector2i(3, 1)); - - auto manualSearchBtn = std::make_shared(mWindow, "MANUAL SEARCH", "enter search terms"); - auto cancelBtn = std::make_shared(mWindow, "CANCEL", "cancel"); - - buttonGrid->setSize(manualSearchBtn->getSize().x() + cancelBtn->getSize().x() + 18, manualSearchBtn->getSize().y()); - buttonGrid->setColWidthPerc(0, 0.5f); - buttonGrid->setColWidthPerc(1, 0.5f); - - buttonGrid->setEntry(manualSearchBtn, Eigen::Vector2i(0, 0), true, false); - buttonGrid->setEntry(cancelBtn, Eigen::Vector2i(1, 0), true, false); + std::vector< std::shared_ptr > buttons; + buttons.push_back(std::make_shared(mWindow, "INPUT", "manually search")); + buttons.push_back(std::make_shared(mWindow, "CANCEL", "cancel", [&] { delete this; })); + auto buttonGrid = makeButtonGrid(mWindow, buttons); mGrid.setEntry(buttonGrid, Eigen::Vector2i(0, 2), true, false); @@ -70,25 +45,17 @@ GuiGameScraper::GuiGameScraper(Window* window, ScraperSearchParams params, std:: mGrid.setPosition((mSize.x() - mGrid.getSize().x()) / 2, (mSize.y() - mGrid.getSize().y()) / 2); mBox.fitTo(mGrid.getSize(), mGrid.getPosition(), Eigen::Vector2f(-32, -32)); - mSearch->setAcceptCallback( [this](MetaDataList* result) { - if(result != NULL) - this->mDoneFunc(*result); - else if(this->mSkipFunc) - this->mSkipFunc(); - - delete this; - }); + mSearch->setAcceptCallback([this, doneFunc](const ScraperSearchResult& result) { doneFunc(result); delete this; }); + mSearch->setCancelCallback([&] { delete this; }); mGrid.resetCursor(); - //mSearch->setSearchParams(params); // also starts the search + mSearch->search(params); // start the search } bool GuiGameScraper::input(InputConfig* config, Input input) { if(config->isMappedTo("b", input) && input.value) { - if(mSkipFunc) - mSkipFunc(); delete this; return true; } @@ -96,19 +63,6 @@ bool GuiGameScraper::input(InputConfig* config, Input input) return GuiComponent::input(config, input); } -void GuiGameScraper::update(int deltaTime) -{ - // HAAACK because AsyncReq wont get pushed in the right order if search happens on creation - if(mSearchCountdown > 0) - { - mSearchCountdown--; - if(mSearchCountdown == 0) - mSearch->setSearchParams(mSearchParams); - } - - GuiComponent::update(deltaTime); -} - std::vector GuiGameScraper::getHelpPrompts() { return mGrid.getHelpPrompts(); diff --git a/src/guis/GuiGameScraper.h b/src/guis/GuiGameScraper.h index 74e1e01e1..f87d7f0f8 100644 --- a/src/guis/GuiGameScraper.h +++ b/src/guis/GuiGameScraper.h @@ -7,16 +7,13 @@ class GuiGameScraper : public GuiComponent { public: - GuiGameScraper(Window* window, ScraperSearchParams params, std::function doneFunc, std::function skipFunc = nullptr); + GuiGameScraper(Window* window, ScraperSearchParams params, std::function doneFunc); bool input(InputConfig* config, Input input) override; - void update(int deltaTime) override; - + virtual std::vector getHelpPrompts() override; private: - int mSearchCountdown; // haaack - ComponentGrid mGrid; NinePatchComponent mBox; @@ -24,6 +21,5 @@ private: ScraperSearchParams mSearchParams; - std::function mDoneFunc; - std::function mSkipFunc; + std::function mCancelFunc; }; diff --git a/src/guis/GuiMetaDataEd.cpp b/src/guis/GuiMetaDataEd.cpp index b59915dbe..5cee1ed57 100644 --- a/src/guis/GuiMetaDataEd.cpp +++ b/src/guis/GuiMetaDataEd.cpp @@ -63,36 +63,8 @@ void GuiMetaDataEd::fetch() mWindow->pushGui(scr); } -void GuiMetaDataEd::fetchDone(MetaDataList result) +void GuiMetaDataEd::fetchDone(const ScraperSearchResult& result) { - //TODO - replace this with resolveMetaDataAssetsAsync! - - //this is a little tricky: - //go through the list of returned results, if anything is an image and the path looks like a URL: - // (1) start an async download + resize (will create an AsyncReq that blocks further user input) - // (when this is finished, call result.set(key, newly_downloaded_file_path) and call fetchDone() again) - // (2) return from this function immediately - for(auto it = mMetaDataDecl.begin(); it != mMetaDataDecl.end(); it++) - { - std::string key = it->key; - std::string val = result.get(it->key); - - //val is /probably/ a URL - if(it->type == MD_IMAGE_PATH && HttpReq::isUrl(val)) - { - downloadImageAsync(mWindow, val, getSaveAsPath(mScraperParams, key, val), - [this, result, key] (std::string filePath) mutable -> void { - //skip it - if(filePath.empty()) - LOG(LogError) << "Error resolving boxart"; - - result.set(key, filePath); - this->fetchDone(result); - }); - return; - } - } - for(unsigned int i = 0; i < mEditors.size(); i++) { //don't overwrite statistics @@ -100,7 +72,7 @@ void GuiMetaDataEd::fetchDone(MetaDataList result) continue; const std::string& key = mMetaDataDecl.at(i).key; - mEditors.at(i)->setValue(result.get(key)); + mEditors.at(i)->setValue(result.mdl.get(key)); } } diff --git a/src/guis/GuiMetaDataEd.h b/src/guis/GuiMetaDataEd.h index 69751da9a..295f4e312 100644 --- a/src/guis/GuiMetaDataEd.h +++ b/src/guis/GuiMetaDataEd.h @@ -20,7 +20,7 @@ public: private: void save(); void fetch(); - void fetchDone(MetaDataList result); + void fetchDone(const ScraperSearchResult& result); MenuComponent mMenu; diff --git a/src/guis/GuiScraperLog.cpp b/src/guis/GuiScraperLog.cpp deleted file mode 100644 index b24a8bd2d..000000000 --- a/src/guis/GuiScraperLog.cpp +++ /dev/null @@ -1,181 +0,0 @@ -#include "GuiScraperLog.h" -#include "GuiGameScraper.h" -#include "../Settings.h" -#include "../Renderer.h" -#include "../Log.h" -#include "../XMLReader.h" - -GuiScraperLog::GuiScraperLog(Window* window, const std::queue& searches, bool manualMode) : GuiComponent(window), - mManualMode(manualMode), - mBox(window, ":/frame.png"), - mSearches(searches), - mStatus(window), - mTextLines(40), - mSuccessCount(0), mSkippedCount(0) -{ - addChild(&mBox); - addChild(&mStatus); - - setSize(Renderer::getScreenWidth() * 0.8f, (float)Renderer::getScreenHeight()); - setPosition((Renderer::getScreenWidth() - mSize.x()) / 2, 0); - - mStatus.setColor(0x000000FF); - mStatus.setSize(mSize.x(), (float)mStatus.getFont()->getHeight()); - mStatus.setCentered(true); - updateStatus(); - - mBox.setEdgeColor(0x111111FF); - mBox.setCenterColor(0xDFDFDFFF); - mBox.fitTo(mSize, Eigen::Vector3f::Zero(), Eigen::Vector2f(8, 0)); - - mWindow->setAllowSleep(false); -} - -GuiScraperLog::~GuiScraperLog() -{ - mWindow->setAllowSleep(true); -} - -void GuiScraperLog::start() -{ - next(); -} - -void GuiScraperLog::next() -{ - if(mSearches.empty()) - { - done(); - return; - } - - ScraperSearchParams search = mSearches.front(); - mSearches.pop(); - - writeLine(getCleanFileName(search.game->getPath()), 0x0000FFFF); - - if(mManualMode) - { - GuiGameScraper* ggs = new GuiGameScraper(mWindow, search, - [this, search] (MetaDataList result) { resultFetched(search, result); }, - [this, search] { resultEmpty(search); }); - - mWindow->pushGui(ggs); - }else{ - std::shared_ptr scraper = Settings::getInstance()->getScraper(); - scraper->getResultsAsync(search, mWindow, [this, search] (std::vector mdls) { - if(mdls.empty()) - resultEmpty(search); - else - resultFetched(search, mdls[0]); - }); - } - - updateStatus(); -} - -void GuiScraperLog::resultFetched(ScraperSearchParams params, MetaDataList mdl) -{ - writeLine(" -> \"" + mdl.get("name") + "\"", 0x0000FFFF); - writeLine(" Downloading images...", 0x0000FFFF); - resolveMetaDataAssetsAsync(mWindow, params, mdl, [this, params] (MetaDataList meta) { resultResolved(params, meta); }); - - //-> resultResolved -} - -void GuiScraperLog::resultResolved(ScraperSearchParams params, MetaDataList mdl) -{ - //apply new metadata - params.game->metadata = mdl; - - writeLine(" Success!", 0x00FF00FF); - - //write changes to gamelist.xml - updateGamelist(params.system); - - mSuccessCount++; - - next(); -} - -void GuiScraperLog::resultEmpty(ScraperSearchParams search) -{ - if(mManualMode) - writeLine(" SKIPPING", 0xFF0000FF); - else - writeLine(" NO RESULTS, skipping", 0xFF0000FF); - - LOG(LogInfo) << "Scraper skipping [" << getCleanFileName(search.game->getPath()) << "]"; - - mSkippedCount++; - - next(); -} - -void GuiScraperLog::done() -{ - writeLine("===================================", 0x000000FF); - - std::stringstream ss; - ss << mSuccessCount << " successful, " << mSkippedCount << " skipped"; - writeLine(ss.str(), 0x000000FF); - - writeLine("DONE!! Press anything to continue.", 0x00FF00FF); - writeLine("===================================", 0x000000FF); - - //done with everything! -} - -void GuiScraperLog::writeLine(const std::string& line, unsigned int color) -{ - std::shared_ptr cmp(new TextComponent(mWindow)); - cmp->setText(line); - cmp->setColor(color); - cmp->setSize(mSize.x(), (float)cmp->getFont()->getHeight()); - - mTextLines.push_back(cmp); -} - -void GuiScraperLog::render(const Eigen::Affine3f& parentTrans) -{ - renderChildren(parentTrans * getTransform()); - - Eigen::Affine3f trans = parentTrans * getTransform(); - - //draw messages - float fontHeight = (float)Font::get(FONT_SIZE_MEDIUM)->getHeight(); - trans = trans.translate(Eigen::Vector3f(0, mSize.y() - fontHeight, 0)); - - for(auto it = mTextLines.rbegin(); it != mTextLines.rend(); it++) - { - (*it)->render(trans); - trans = trans.translate(Eigen::Vector3f(0, -fontHeight, 0)); - - if(trans.translation().y() < fontHeight) - break; - } -} - -bool GuiScraperLog::input(InputConfig* config, Input input) -{ - if(input.value != 0) - { - //we're done - if(mSearches.empty()) - { - delete this; - return true; - } - } - - return GuiComponent::input(config, input); -} - -void GuiScraperLog::updateStatus() -{ - std::stringstream ss; - - ss << mSearches.size() << " games remaining"; - - mStatus.setText(ss.str()); -} diff --git a/src/guis/GuiScraperLog.h b/src/guis/GuiScraperLog.h deleted file mode 100644 index 62634ce6f..000000000 --- a/src/guis/GuiScraperLog.h +++ /dev/null @@ -1,46 +0,0 @@ -#pragma once - -#include "../GuiComponent.h" -#include "../components/NinePatchComponent.h" -#include "../scrapers/Scraper.h" -#include "../components/TextComponent.h" - -#include -#include - -//A "terminal" of sorts for scraping. -//Doesn't accept input, but renders log-style messages and handles the callback chain for multi-game scraping. -class GuiScraperLog : public GuiComponent -{ -public: - GuiScraperLog(Window* window, const std::queue& params, bool manualMode); - ~GuiScraperLog(); - - void start(); - - void render(const Eigen::Affine3f& parentTrans) override; - bool input(InputConfig* config, Input input) override; - -private: - void updateStatus(); - void writeLine(const std::string& line, unsigned int color); - - void resultFetched(ScraperSearchParams params, MetaDataList mdl); - void resultResolved(ScraperSearchParams params, MetaDataList mdl); - void resultEmpty(ScraperSearchParams params); - - void next(); - void done(); - - bool mManualMode; - - NinePatchComponent mBox; - - std::queue mSearches; - - TextComponent mStatus; - boost::circular_buffer< std::shared_ptr > mTextLines; - - unsigned int mSuccessCount; - unsigned int mSkippedCount; -}; diff --git a/src/guis/GuiScraperMulti.cpp b/src/guis/GuiScraperMulti.cpp new file mode 100644 index 000000000..179dfcd50 --- /dev/null +++ b/src/guis/GuiScraperMulti.cpp @@ -0,0 +1,102 @@ +#include "GuiScraperMulti.h" +#include "../Renderer.h" + +#include "../components/TextComponent.h" +#include "../components/ButtonComponent.h" +#include "../components/ScraperSearchComponent.h" +#include "../components/MenuComponent.h" // for makeButtonGrid +#include "GuiMsgBox.h" + +using namespace Eigen; + +GuiScraperMulti::GuiScraperMulti(Window* window, const std::queue& searches, bool approveResults) : + GuiComponent(window), mBackground(window, ":/frame.png"), mGrid(window, Vector2i(1, 4)), + mSearchQueue(searches) +{ + addChild(&mBackground); + addChild(&mGrid); + + mTotalGames = mSearchQueue.size(); + mCurrentGame = 0; + + // set up grid + mTitle = std::make_shared(mWindow, "SCRAPING IN PROGRESS", Font::get(FONT_SIZE_SMALL), 0x777777FF, true); + mGrid.setEntry(mTitle, Vector2i(0, 0), false, true); + + mSubtitle = std::make_shared(mWindow, "subtitle text", Font::get(FONT_SIZE_SMALL), 0x888888FF, true); + mGrid.setEntry(mSubtitle, Vector2i(0, 1), false, true); + + mSearchComp = std::make_shared(mWindow, + approveResults ? ScraperSearchComponent::ALWAYS_ACCEPT_MATCHING_CRC : ScraperSearchComponent::ALWAYS_ACCEPT_FIRST_RESULT); + mSearchComp->setAcceptCallback(std::bind(&GuiScraperMulti::acceptResult, this, std::placeholders::_1)); + mSearchComp->setCancelCallback(std::bind(&GuiScraperMulti::finish, this)); + mGrid.setEntry(mSearchComp, Vector2i(0, 2), approveResults, true); + + std::vector< std::shared_ptr > buttons; + buttons.push_back(std::make_shared(mWindow, "INPUT", "manually search by name", nullptr)); + buttons.push_back(std::make_shared(mWindow, "SKIP", "skip this game", std::bind(&GuiScraperMulti::skip, this))); + buttons.push_back(std::make_shared(mWindow, "STOP", "cancel scraping", std::bind(&GuiScraperMulti::finish, this))); + mButtonGrid = makeButtonGrid(mWindow, buttons); + mButtonGrid->setSize(mButtonGrid->getSize().x(), mButtonGrid->getSize().y() + 32); + mGrid.setEntry(mButtonGrid, Vector2i(0, 3), true, false); + + setSize(Renderer::getScreenWidth() * 0.7f, Renderer::getScreenHeight() * 0.65f); + setPosition((Renderer::getScreenWidth() - mSize.x()) / 2, (Renderer::getScreenHeight() - mSize.y()) / 2); + + doNextSearch(); +} + +void GuiScraperMulti::onSizeChanged() +{ + mBackground.fitTo(mSize, Vector3f::Zero(), Vector2f(-32, -32)); + mGrid.setSize(mSize); + + mGrid.setRowHeightPerc(0, mTitle->getFont()->getHeight() / mGrid.getSize().y()); + mGrid.setRowHeightPerc(1, mSubtitle->getFont()->getHeight() / mGrid.getSize().y()); + mGrid.setRowHeightPerc(3, mButtonGrid->getSize().y() / mGrid.getSize().y()); +} + +void GuiScraperMulti::doNextSearch() +{ + if(mSearchQueue.empty()) + { + finish(); + return; + } + + // update subtitle + std::stringstream ss; + ss << "GAME " << (mCurrentGame + 1) << " OF " << mTotalGames; + mSubtitle->setText(ss.str()); + + mSearchComp->search(mSearchQueue.front()); +} + +void GuiScraperMulti::acceptResult(const ScraperSearchResult& result) +{ + ScraperSearchParams& search = mSearchQueue.front(); + + search.game->metadata = result.mdl; + + mSearchQueue.pop(); + mCurrentGame++; + doNextSearch(); +} + +void GuiScraperMulti::skip() +{ + mSearchQueue.pop(); + mCurrentGame++; + doNextSearch(); +} + +void GuiScraperMulti::finish() +{ + mWindow->pushGui(new GuiMsgBox(mWindow, "SCRAPING COMPLETE!", + "OK", [&] { delete this; })); +} + +std::vector GuiScraperMulti::getHelpPrompts() +{ + return mGrid.getHelpPrompts(); +} diff --git a/src/guis/GuiScraperMulti.h b/src/guis/GuiScraperMulti.h new file mode 100644 index 000000000..1f9919a89 --- /dev/null +++ b/src/guis/GuiScraperMulti.h @@ -0,0 +1,39 @@ +#pragma once + +#include "../GuiComponent.h" +#include "../components/NinePatchComponent.h" +#include "../components/ComponentGrid.h" +#include "../scrapers/Scraper.h" + +#include + +class ScraperSearchComponent; +class TextComponent; + +class GuiScraperMulti : public GuiComponent +{ +public: + GuiScraperMulti(Window* window, const std::queue& searches, bool approveResults); + + void onSizeChanged() override; + std::vector getHelpPrompts() override; + +private: + void acceptResult(const ScraperSearchResult& result); + void skip(); + void doNextSearch(); + + void finish(); + + unsigned int mTotalGames; + unsigned int mCurrentGame; + std::queue mSearchQueue; + + NinePatchComponent mBackground; + ComponentGrid mGrid; + + std::shared_ptr mTitle; + std::shared_ptr mSubtitle; + std::shared_ptr mSearchComp; + std::shared_ptr mButtonGrid; +}; diff --git a/src/guis/GuiScraperStart.cpp b/src/guis/GuiScraperStart.cpp index 59a08495e..8125c100a 100644 --- a/src/guis/GuiScraperStart.cpp +++ b/src/guis/GuiScraperStart.cpp @@ -1,5 +1,5 @@ #include "GuiScraperStart.h" -#include "GuiScraperLog.h" +#include "GuiScraperMulti.h" #include "GuiMsgBox.h" #include "../components/TextComponent.h" @@ -14,9 +14,9 @@ GuiScraperStart::GuiScraperStart(Window* window) : GuiComponent(window), // add filters (with first one selected) mFilters = std::make_shared< OptionListComponent >(mWindow, "SCRAPE THESE GAMES", false); mFilters->add("All Games", - [](SystemData*, FileData*) -> bool { return true; }, true); + [](SystemData*, FileData*) -> bool { return true; }, false); mFilters->add("Only missing image", - [](SystemData*, FileData* g) -> bool { return g->metadata.get("image").empty(); }, false); + [](SystemData*, FileData* g) -> bool { return g->metadata.get("image").empty(); }, true); mMenu.addWithLabel("Filter", mFilters); //add systems (all with a platformid specified selected) @@ -57,9 +57,8 @@ void GuiScraperStart::start() { std::queue searches = getSearches(mSystems->getSelectedObjects(), mFilters->getSelected()); - GuiScraperLog* gsl = new GuiScraperLog(mWindow, searches, mApproveResults->getState()); - mWindow->pushGui(gsl); - gsl->start(); + GuiScraperMulti* gsm = new GuiScraperMulti(mWindow, searches, mApproveResults->getState()); + mWindow->pushGui(gsm); delete this; } diff --git a/src/scrapers/GamesDBScraper.cpp b/src/scrapers/GamesDBScraper.cpp index a4f828959..ab1e3a83d 100644 --- a/src/scrapers/GamesDBScraper.cpp +++ b/src/scrapers/GamesDBScraper.cpp @@ -55,7 +55,7 @@ const std::map gamesdb_platformid_map = boost::assign:: (TURBOGRAFX_16, "TurboGrafx 16"); -std::shared_ptr GamesDBScraper::makeHttpReq(ScraperSearchParams params) +std::unique_ptr GamesDBScraper::getResultsAsync(const ScraperSearchParams& params) { std::string path = "/api/GetGame.php?"; @@ -71,25 +71,41 @@ std::shared_ptr GamesDBScraper::makeHttpReq(ScraperSearchParams params) path += HttpReq::urlEncode(gamesdb_platformid_map.at(params.system->getPlatformId())); } - return std::make_shared("thegamesdb.net" + path); + path = "thegamesdb.net" + path; + + return std::unique_ptr(new GamesDBHandle(params, path)); } -std::vector GamesDBScraper::parseReq(ScraperSearchParams params, std::shared_ptr req) +GamesDBHandle::GamesDBHandle(const ScraperSearchParams& params, const std::string& url) : + mReq(std::unique_ptr(new HttpReq(url))) { - std::vector mdl; + setStatus(SEARCH_IN_PROGRESS); +} - if(req->status() != HttpReq::REQ_SUCCESS) +void GamesDBHandle::update() +{ + if(mReq->status() == HttpReq::REQ_IN_PROGRESS) + return; + + if(mReq->status() != HttpReq::REQ_SUCCESS) { - LOG(LogError) << "HttpReq error"; - return mdl; + std::stringstream ss; + ss << "Network error - " << mReq->getErrorMsg(); + setError(ss.str()); + return; } + // our HTTP request was successful + // try to build our result list + + std::vector results; + pugi::xml_document doc; - pugi::xml_parse_result parseResult = doc.load(req->getContent().c_str()); + pugi::xml_parse_result parseResult = doc.load(mReq->getContent().c_str()); if(!parseResult) { - LOG(LogError) << "Error parsing XML"; - return mdl; + setError("Error parsing XML"); + return; } pugi::xml_node data = doc.child("Data"); @@ -100,24 +116,25 @@ std::vector GamesDBScraper::parseReq(ScraperSearchParams params, s pugi::xml_node game = data.child("Game"); while(game && resultNum < MAX_SCRAPER_RESULTS) { - mdl.push_back(MetaDataList(GAME_METADATA)); - mdl.back().set("name", game.child("GameTitle").text().get()); - mdl.back().set("desc", game.child("Overview").text().get()); + ScraperSearchResult result; + + result.mdl.set("name", game.child("GameTitle").text().get()); + result.mdl.set("desc", game.child("Overview").text().get()); boost::posix_time::ptime rd = string_to_ptime(game.child("ReleaseDate").text().get(), "%m/%d/%Y"); - mdl.back().setTime("releasedate", rd); + result.mdl.setTime("releasedate", rd); - mdl.back().set("developer", game.child("Developer").text().get()); - mdl.back().set("publisher", game.child("Publisher").text().get()); - mdl.back().set("genre", game.child("Genres").first_child().text().get()); - mdl.back().set("players", game.child("Players").text().get()); + result.mdl.set("developer", game.child("Developer").text().get()); + result.mdl.set("publisher", game.child("Publisher").text().get()); + result.mdl.set("genre", game.child("Genres").first_child().text().get()); + result.mdl.set("players", game.child("Players").text().get()); if(Settings::getInstance()->getBool("ScrapeRatings") && game.child("Rating")) { float ratingVal = (game.child("Rating").text().as_int() / 10.0f); std::stringstream ss; ss << ratingVal; - mdl.back().set("rating", ss.str()); + result.mdl.set("rating", ss.str()); } pugi::xml_node images = game.child("Images"); @@ -128,14 +145,18 @@ std::vector GamesDBScraper::parseReq(ScraperSearchParams params, s if(art) { - mdl.back().set("thumbnail", baseImageUrl + art.attribute("thumb").as_string()); - mdl.back().set("image", baseImageUrl + art.text().get()); + result.thumbnailUrl = baseImageUrl + art.attribute("thumb").as_string(); + result.imageUrl = baseImageUrl + art.text().get(); } } + results.push_back(result); + resultNum++; game = game.next_sibling("Game"); } - return mdl; + setStatus(SEARCH_DONE); + setResults(results); + return; } diff --git a/src/scrapers/GamesDBScraper.h b/src/scrapers/GamesDBScraper.h index 3a306ee99..f94671114 100644 --- a/src/scrapers/GamesDBScraper.h +++ b/src/scrapers/GamesDBScraper.h @@ -3,11 +3,22 @@ #include "Scraper.h" #include "../HttpReq.h" +class GamesDBHandle : public ScraperSearchHandle +{ +public: + GamesDBHandle(const ScraperSearchParams& params, const std::string& url); + + void update() override; + +private: + std::unique_ptr mReq; + ScraperSearchParams mParams; +}; + class GamesDBScraper : public Scraper { public: + std::unique_ptr getResultsAsync(const ScraperSearchParams& params) override; + const char* getName(); -private: - std::shared_ptr makeHttpReq(ScraperSearchParams params) override; - std::vector parseReq(ScraperSearchParams params, std::shared_ptr) override; }; diff --git a/src/scrapers/Scraper.cpp b/src/scrapers/Scraper.cpp index ddfe6350e..e730b9b15 100644 --- a/src/scrapers/Scraper.cpp +++ b/src/scrapers/Scraper.cpp @@ -9,31 +9,21 @@ #include "GamesDBScraper.h" #include "TheArchiveScraper.h" -std::vector Scraper::getResults(ScraperSearchParams params) +std::string ScraperSearchHandle::getStatusString() { - std::shared_ptr req = makeHttpReq(params); - while(req->status() == HttpReq::REQ_IN_PROGRESS); - - return parseReq(params, req); -} - -void Scraper::getResultsAsync(ScraperSearchParams params, Window* window, std::function)> returnFunc) -{ - std::shared_ptr httpreq = makeHttpReq(params); - AsyncReqComponent* req = new AsyncReqComponent(window, httpreq, - [this, params, returnFunc] (std::shared_ptr r) + switch(mStatus) { - returnFunc(parseReq(params, r)); - }, [returnFunc] () - { - returnFunc(std::vector()); - }); - - window->pushGui(req); + case SEARCH_IN_PROGRESS: + return "search in progress"; + case SEARCH_ERROR: + return mError; + case SEARCH_DONE: + return "search done"; + default: + return "something impossible has occured"; + } } - - std::string processFileDownload(std::shared_ptr r, std::string saveAs) { if(r->status() != HttpReq::REQ_SUCCESS) diff --git a/src/scrapers/Scraper.h b/src/scrapers/Scraper.h index df9b8a42d..ef0ce2207 100644 --- a/src/scrapers/Scraper.h +++ b/src/scrapers/Scraper.h @@ -16,17 +16,53 @@ struct ScraperSearchParams std::string nameOverride; }; +struct ScraperSearchResult +{ + ScraperSearchResult() : mdl(GAME_METADATA) {}; + + MetaDataList mdl; + std::string imageUrl; + std::string thumbnailUrl; +}; + +enum ScraperSearchStatus +{ + SEARCH_IN_PROGRESS, + SEARCH_ERROR, + SEARCH_DONE +}; + +class ScraperSearchHandle +{ +public: + virtual void update() = 0; + + // Update and return the latest status. + inline ScraperSearchStatus status() { update(); return mStatus; } + + // User-friendly string of our current status. Will return error message if status() == SEARCH_ERROR. + std::string getStatusString(); + + inline const std::vector& getResults() const { assert(mStatus != SEARCH_IN_PROGRESS); return mResults; } + +protected: + inline void setError(const std::string& error) { setStatus(SEARCH_ERROR); mError = error; } + inline void setStatus(ScraperSearchStatus status) { mStatus = status; } + inline void setResults(const std::vector& results) { mResults = results; } + +private: + std::string mError; + ScraperSearchStatus mStatus; + std::vector mResults; +}; + class Scraper { public: //Get a list of potential results. - virtual std::vector getResults(ScraperSearchParams params); - virtual void getResultsAsync(ScraperSearchParams params, Window* window, std::function)> returnFunc); + virtual std::unique_ptr getResultsAsync(const ScraperSearchParams& params) = 0; virtual const char* getName() = 0; -private: - virtual std::shared_ptr makeHttpReq(ScraperSearchParams params) = 0; - virtual std::vector parseReq(ScraperSearchParams params, std::shared_ptr) = 0; }; std::shared_ptr createScraperByName(const std::string& name); diff --git a/src/scrapers/TheArchiveScraper.cpp b/src/scrapers/TheArchiveScraper.cpp index 5fa8f535f..fa008bd21 100644 --- a/src/scrapers/TheArchiveScraper.cpp +++ b/src/scrapers/TheArchiveScraper.cpp @@ -6,7 +6,7 @@ const char* TheArchiveScraper::getName() { return "TheArchive"; } -std::shared_ptr TheArchiveScraper::makeHttpReq(ScraperSearchParams params) +std::unique_ptr TheArchiveScraper::getResultsAsync(const ScraperSearchParams& params) { std::string path = "/2.0/Archive.search/xml/7TTRM4MNTIKR2NNAGASURHJOZJ3QXQC5/"; @@ -17,25 +17,44 @@ std::shared_ptr TheArchiveScraper::makeHttpReq(ScraperSearchParams para path += HttpReq::urlEncode(cleanName); //platform TODO, should use some params.system get method - return std::make_shared("api.archive.vg" + path); + path = "api.archive.vg" + path; + + return std::unique_ptr(new TheArchiveHandle(params, path)); } -std::vector TheArchiveScraper::parseReq(ScraperSearchParams params, std::shared_ptr req) +TheArchiveHandle::TheArchiveHandle(const ScraperSearchParams& params, const std::string& url) : + mReq(std::unique_ptr(new HttpReq(url))) { - std::vector mdl; + setStatus(SEARCH_IN_PROGRESS); +} - if(req->status() != HttpReq::REQ_SUCCESS) +void TheArchiveHandle::update() +{ + if(status() == SEARCH_DONE) + return; + + if(mReq->status() == HttpReq::REQ_IN_PROGRESS) + return; + + if(mReq->status() != HttpReq::REQ_SUCCESS) { - LOG(LogError) << "HttpReq error"; - return mdl; + std::stringstream ss; + ss << "Network error: " << mReq->getErrorMsg(); + setError(ss.str()); + return; } + // if we're here, our HTTP request finished successfully + + // so, let's try building our result list + std::vector results; + pugi::xml_document doc; - pugi::xml_parse_result parseResult = doc.load(req->getContent().c_str()); + pugi::xml_parse_result parseResult = doc.load(mReq->getContent().c_str()); if(!parseResult) { - LOG(LogError) << "Error parsing XML"; - return mdl; + setError("Error parsing XML"); + return; } pugi::xml_node data = doc.child("OpenSearchDescription").child("games"); @@ -44,30 +63,34 @@ std::vector TheArchiveScraper::parseReq(ScraperSearchParams params pugi::xml_node game = data.child("game"); while(game && resultNum < MAX_SCRAPER_RESULTS) { - mdl.push_back(MetaDataList(GAME_METADATA)); - mdl.back().set("name", game.child("title").text().get()); - mdl.back().set("desc", game.child("description").text().get()); + ScraperSearchResult result; + + result.mdl.set("name", game.child("title").text().get()); + result.mdl.set("desc", game.child("description").text().get()); //Archive.search does not return ratings - mdl.back().set("developer", game.child("developer").text().get()); + result.mdl.set("developer", game.child("developer").text().get()); std::string genre = game.child("genre").text().get(); size_t search = genre.find_last_of(" > "); genre = genre.substr(search == std::string::npos ? 0 : search, std::string::npos); - mdl.back().set("genre", genre); + result.mdl.set("genre", genre); pugi::xml_node image = game.child("box_front"); pugi::xml_node thumbnail = game.child("box_front_small"); - if (image) - mdl.back().set("image",image.text().get()); - if (thumbnail) - mdl.back().set("thumbnail", thumbnail.text().get()); + if(image) + result.imageUrl = image.text().get(); + if(thumbnail) + result.thumbnailUrl = thumbnail.text().get(); + + results.push_back(result); resultNum++; game = game.next_sibling("game"); } - return mdl; + setStatus(SEARCH_DONE); + setResults(results); } diff --git a/src/scrapers/TheArchiveScraper.h b/src/scrapers/TheArchiveScraper.h index bf5ef0bef..60724c946 100644 --- a/src/scrapers/TheArchiveScraper.h +++ b/src/scrapers/TheArchiveScraper.h @@ -3,12 +3,22 @@ #include "Scraper.h" #include "../HttpReq.h" +class TheArchiveHandle : public ScraperSearchHandle +{ +public: + TheArchiveHandle(const ScraperSearchParams& params, const std::string& url); + + void update() override; + +private: + std::unique_ptr mReq; + ScraperSearchParams mParams; +}; + class TheArchiveScraper : public Scraper { public: - const char* getName(); -private: - std::shared_ptr makeHttpReq(ScraperSearchParams params) override; - std::vector parseReq(ScraperSearchParams params, std::shared_ptr) override; -}; + std::unique_ptr getResultsAsync(const ScraperSearchParams& params) override; + const char* getName(); +};