Redid Scrapers to return ScraperSearchHandles for async searches.

This allows for much better error handling and doesn't take over the UI.
Redid GuiScraperLog to fit new UI concept.

CMakeLists.txt

@@ -189,8 +189,8 @@ set(ES_HEADERS
     ${CMAKE_CURRENT_SOURCE_DIR}/src/guis/GuiInputConfig.h
     ${CMAKE_CURRENT_SOURCE_DIR}/src/guis/GuiMenu.h
     ${CMAKE_CURRENT_SOURCE_DIR}/src/guis/GuiSettings.h
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/guis/GuiScraperMulti.h
     ${CMAKE_CURRENT_SOURCE_DIR}/src/guis/GuiScraperStart.h
-    ${CMAKE_CURRENT_SOURCE_DIR}/src/guis/GuiScraperLog.h
 
     ${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/Scraper.h
     ${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/GamesDBScraper.h
@@ -269,8 +269,8 @@ set(ES_SOURCES
     ${CMAKE_CURRENT_SOURCE_DIR}/src/guis/GuiInputConfig.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/guis/GuiMenu.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/guis/GuiSettings.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/guis/GuiScraperMulti.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/guis/GuiScraperStart.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/src/guis/GuiScraperLog.cpp
 
     ${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/Scraper.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/GamesDBScraper.cpp

src/ScraperCmdLine.cpp

@@ -138,6 +138,7 @@ int run_scraper_cmdline()
 	out << "Alright, let's do this thing!\n";
 	out << "=============================\n";
 
+	/*
 	std::shared_ptr<Scraper> scraper = Settings::getInstance()->getScraper();
 	for(auto sysIt = systems.begin(); sysIt != systems.end(); sysIt++)
 	{
@@ -275,6 +276,10 @@ int run_scraper_cmdline()
 	out << "==============================\n";
 	out << "SCRAPE COMPLETE!\n";
 	out << "==============================\n";
+	*/
+
+	out << "\n\n";
+	out << "ACTUALLY THIS IS STILL TODO\n";
 
 	return 0;
 }

src/components/ButtonComponent.cpp

@@ -13,6 +13,7 @@ ButtonComponent::ButtonComponent(Window* window, const std::string& text, const
 {
 	setPressedFunc(func);
 	setText(text, helpText);
+	updateImage();
 }
 
 void ButtonComponent::onSizeChanged()
@@ -71,7 +72,7 @@ void ButtonComponent::updateImage()
 {
 	if(!mEnabled || !mPressedFunc)
 	{
-		mBox.setImagePath(":/button.png");
+		mBox.setImagePath(":/button_filled.png");
 		mBox.setCenterColor(0x770000FF);
 		mBox.setEdgeColor(0x770000FF);
 		return;

src/components/MenuComponent.cpp

@@ -59,13 +59,12 @@ void MenuComponent::updateGrid()
 	if(mButtonGrid)
 		mGrid.removeEntry(mButtonGrid);
 
+	mButtonGrid.reset();
+
 	if(mButtons.size())
 	{
 		mButtonGrid = makeButtonGrid(mWindow, mButtons);
-		
 		mGrid.setEntry(mButtonGrid, Vector2i(0, 2), true, false);
-	}else{
-		mButtonGrid.reset();
 	}
 }
 

src/components/ScraperSearchComponent.cpp

@@ -1,9 +1,10 @@
 #include "ScraperSearchComponent.h"
 
-#include "../components/TextComponent.h"
+#include "../guis/GuiMsgBox.h"
-#include "../components/ScrollableContainer.h"
+#include "TextComponent.h"
-#include "../components/ImageComponent.h"
+#include "ScrollableContainer.h"
-#include "../components/ComponentList.h"
+#include "ImageComponent.h"
+#include "ComponentList.h"
 #include "../HttpReq.h"
 #include "../Settings.h"
 #include "../Log.h"
@@ -12,9 +13,6 @@ ScraperSearchComponent::ScraperSearchComponent(Window* window, SearchType type)
 	mGrid(window, Eigen::Vector2i(4, 3)),
 	mSearchType(type)
 {
-	mSearchParams.system = NULL;
-	mSearchParams.game = NULL;
-
 	addChild(&mGrid);
 
 	using namespace Eigen;
@@ -85,18 +83,17 @@ void ScraperSearchComponent::updateViewStyle()
 	}
 }
 
-void ScraperSearchComponent::setSearchParams(const ScraperSearchParams& params)
+void ScraperSearchComponent::search(const ScraperSearchParams& params)
 {
-	mSearchParams = params;
+	mResultList->clear();
-	search();
+	mScraperResults.clear();
+	updateInfoPane();
+
+	mLastSearch = params;
+	mSearchHandle = Settings::getInstance()->getScraper()->getResultsAsync(params);
 }
 
-void ScraperSearchComponent::search()
+void ScraperSearchComponent::onSearchDone(const std::vector<ScraperSearchResult>& results)
-{
-	Settings::getInstance()->getScraper()->getResultsAsync(mSearchParams, mWindow, std::bind(&ScraperSearchComponent::onSearchReceived, this, std::placeholders::_1));
-}
-
-void ScraperSearchComponent::onSearchReceived(std::vector<MetaDataList> results)
 {
 	mResultList->clear();
 
@@ -117,7 +114,7 @@ void ScraperSearchComponent::onSearchReceived(std::vector<MetaDataList> results)
 		for(int i = 0; i < end; i++)
 		{
 			row.elements.clear();
-			row.addElement(std::make_shared<TextComponent>(mWindow, results.at(i).get("name"), font, color), true);
+			row.addElement(std::make_shared<TextComponent>(mWindow, results.at(i).mdl.get("name"), font, color), true);
 			mResultList->addRow(row);
 		}
 		mGrid.resetCursor();
@@ -128,16 +125,23 @@ void ScraperSearchComponent::onSearchReceived(std::vector<MetaDataList> results)
 	if(mSearchType == ALWAYS_ACCEPT_FIRST_RESULT)
 	{
 		if(mScraperResults.size() == 0)
-			returnResult(NULL);
+			mSkipCallback();
 		else
-			returnResult(&mScraperResults.front());
+			returnResult(mScraperResults.front());
 	}else if(mSearchType == ALWAYS_ACCEPT_MATCHING_CRC)
 	{
 		// TODO
-		assert(false);
 	}
 }
 
+void ScraperSearchComponent::onSearchError(const std::string& error)
+{
+	mWindow->pushGui(new GuiMsgBox(mWindow, error,
+		"RETRY", std::bind(&ScraperSearchComponent::search, this, mLastSearch),
+		"SKIP", mSkipCallback,
+		"CANCEL", mCancelCallback));
+}
+
 int ScraperSearchComponent::getSelectedIndex()
 {
 	if(mScraperResults.size() && mGrid.getSelectedComponent() != mResultList)
@@ -151,17 +155,21 @@ void ScraperSearchComponent::updateInfoPane()
 	int i = getSelectedIndex();
 	if(i != -1 && (int)mScraperResults.size() > i)
 	{
-		mResultName->setText(mScraperResults.at(i).get("name"));
+		mResultName->setText(mScraperResults.at(i).mdl.get("name"));
-		mResultDesc->setText(mScraperResults.at(i).get("desc"));
+		mResultDesc->setText(mScraperResults.at(i).mdl.get("desc"));
 		mDescContainer->setScrollPos(Eigen::Vector2d(0, 0));
 		mDescContainer->resetAutoScrollTimer();
 
-		std::string thumb = mScraperResults.at(i).get("thumbnail");
 		mResultThumbnail->setImage("");
+		const std::string& thumb = mScraperResults.at(i).thumbnailUrl;
 		if(!thumb.empty())
 			mThumbnailReq = std::unique_ptr<HttpReq>(new HttpReq(thumb));
 		else
 			mThumbnailReq.reset();
+	}else{
+		mResultName->setText(" ");
+		mResultDesc->setText(" ");
+		mResultThumbnail->setImage("");
 	}
 }
 
@@ -172,7 +180,7 @@ bool ScraperSearchComponent::input(InputConfig* config, Input input)
 		//if you're on a result
 		if(getSelectedIndex() != -1)
 		{
-			returnResult(&mScraperResults.at(getSelectedIndex()));
+			returnResult(mScraperResults.at(getSelectedIndex()));
 			return true;
 		}
 	}
@@ -187,9 +195,27 @@ bool ScraperSearchComponent::input(InputConfig* config, Input input)
 	return ret;
 }
 
-void ScraperSearchComponent::returnResult(MetaDataList* result)
+void ScraperSearchComponent::returnResult(ScraperSearchResult result)
 {
-	assert(mAcceptCallback);
+	// resolve metadata image before returning
+	if(!result.imageUrl.empty())
+	{
+		downloadImageAsync(mWindow, result.imageUrl, getSaveAsPath(mLastSearch, "image", result.imageUrl), 
+			[this, result] (std::string filePath) mutable -> void
+			{
+					if(filePath.empty())
+					{
+						onSearchError("Error downloading boxart.");
+						return;
+					}
+					
+					result.mdl.set("image", filePath);
+					result.imageUrl = "";
+					this->returnResult(result); // re-enter this function
+			});
+		return;
+	}
+
 	mAcceptCallback(result);
 }
 
@@ -200,6 +226,19 @@ void ScraperSearchComponent::update(int deltaTime)
 		updateThumbnail();
 	}
 
+	if(mSearchHandle && mSearchHandle->status() != SEARCH_IN_PROGRESS)
+	{
+		if(mSearchHandle->status() == SEARCH_DONE)
+		{
+			onSearchDone(mSearchHandle->getResults());
+		}else if(mSearchHandle->status() == SEARCH_ERROR)
+		{
+			onSearchError(mSearchHandle->getStatusString());
+		}
+
+		mSearchHandle.reset();
+	}
+
 	GuiComponent::update(deltaTime);
 }
 

src/components/ScraperSearchComponent.h

@@ -25,8 +25,12 @@ public:
 
 	ScraperSearchComponent(Window* window, SearchType searchType = NEVER_AUTO_ACCEPT);
 
-	void setSearchParams(const ScraperSearchParams& params);
+	void search(const ScraperSearchParams& params);
-	inline void setAcceptCallback(const std::function<void(MetaDataList*)>& acceptCallback) { mAcceptCallback = acceptCallback; }
+
+	// Metadata assets will be resolved before calling the accept callback (e.g. result.mdl's "image" is automatically downloaded and properly set).
+	inline void setAcceptCallback(const std::function<void(const ScraperSearchResult&)>& acceptCallback) { mAcceptCallback = acceptCallback; }
+	inline void setSkipCallback(const std::function<void()>& skipCallback) { mSkipCallback = skipCallback; };
+	inline void setCancelCallback(const std::function<void()>& cancelCallback) { mCancelCallback = cancelCallback; }
 
 	bool input(InputConfig* config, Input input) override;
 	void update(int deltaTime) override;
@@ -40,12 +44,13 @@ private:
 	void updateThumbnail();
 	void updateInfoPane();
 
-	void search();
+	void onSearchError(const std::string& error);
-	void onSearchReceived(std::vector<MetaDataList> results);
+	void onSearchDone(const std::vector<ScraperSearchResult>& results);
 
 	int getSelectedIndex();
 
-	void returnResult(MetaDataList* result);
+	// resolve any metadata assets that need to be downloaded and return
+	void returnResult(ScraperSearchResult result);
 
 	ComponentGrid mGrid;
 
@@ -56,9 +61,12 @@ private:
 	std::shared_ptr<ComponentList> mResultList;
 
 	SearchType mSearchType;
-	ScraperSearchParams mSearchParams;
+	ScraperSearchParams mLastSearch;
-	std::function<void(MetaDataList*)> mAcceptCallback;
+	std::function<void(const ScraperSearchResult&)> mAcceptCallback;
+	std::function<void()> mSkipCallback;
+	std::function<void()> mCancelCallback;
 
-	std::vector<MetaDataList> mScraperResults;
+	std::unique_ptr<ScraperSearchHandle> mSearchHandle;
+	std::vector<ScraperSearchResult> mScraperResults;
 	std::unique_ptr<HttpReq> mThumbnailReq;
 };

src/guis/GuiGameScraper.cpp

@@ -6,31 +6,13 @@
 
 #include "../components/TextComponent.h"
 #include "../components/ButtonComponent.h"
+#include "../components/MenuComponent.h"
 
-GuiGameScraper::GuiGameScraper(Window* window, ScraperSearchParams params, std::function<void(MetaDataList)> doneFunc, std::function<void()> skipFunc) : GuiComponent(window), 
+GuiGameScraper::GuiGameScraper(Window* window, ScraperSearchParams params, std::function<void(const ScraperSearchResult&)> doneFunc) : GuiComponent(window), 
 	mGrid(window, Eigen::Vector2i(1, 3)), 
 	mBox(window, ":/frame.png"),
-	mSearchParams(params),
+	mSearchParams(params)
-	mDoneFunc(doneFunc),
-	mSkipFunc(skipFunc),
-	mSearchCountdown(2)
 {
-	// new screen:
-	
-	//				  FILE NAME
-	//--------------------------------------
-	//   Result Title     |  Result #1
-	// |-------|  .....   |  Result #2
-	// |  IMG  |  info    |  Result #3
-	// |-------|  .....   |  .........
-	//                    |  .........
-	// DESCRIPTION........|  .........
-	// ...................|  .........
-	// ...................|  .........
-	//--------------------------------------
-	//       [SEARCH NAME] [CANCEL]
-
-
 	addChild(&mBox);
 	addChild(&mGrid);
 
@@ -52,17 +34,10 @@ GuiGameScraper::GuiGameScraper(Window* window, ScraperSearchParams params, std::
 	mGrid.setEntry(mSearch, Eigen::Vector2i(0, 1), true);
 
 	// buttons
-	auto buttonGrid = std::make_shared<ComponentGrid>(mWindow, Eigen::Vector2i(3, 1));
+	std::vector< std::shared_ptr<ButtonComponent> > buttons;
-	
+	buttons.push_back(std::make_shared<ButtonComponent>(mWindow, "INPUT", "manually search"));
-	auto manualSearchBtn = std::make_shared<ButtonComponent>(mWindow, "MANUAL SEARCH", "enter search terms");
+	buttons.push_back(std::make_shared<ButtonComponent>(mWindow, "CANCEL", "cancel", [&] { delete this; }));
-	auto cancelBtn = std::make_shared<ButtonComponent>(mWindow, "CANCEL", "cancel");
+	auto buttonGrid = makeButtonGrid(mWindow, buttons);
-
-	buttonGrid->setSize(manualSearchBtn->getSize().x() + cancelBtn->getSize().x() + 18, manualSearchBtn->getSize().y());
-	buttonGrid->setColWidthPerc(0, 0.5f);
-	buttonGrid->setColWidthPerc(1, 0.5f);
-
-	buttonGrid->setEntry(manualSearchBtn, Eigen::Vector2i(0, 0), true, false);
-	buttonGrid->setEntry(cancelBtn, Eigen::Vector2i(1, 0), true, false);
 
 	mGrid.setEntry(buttonGrid, Eigen::Vector2i(0, 2), true, false);
 
@@ -70,25 +45,17 @@ GuiGameScraper::GuiGameScraper(Window* window, ScraperSearchParams params, std::
 	mGrid.setPosition((mSize.x() - mGrid.getSize().x()) / 2, (mSize.y() - mGrid.getSize().y()) / 2);
 	mBox.fitTo(mGrid.getSize(), mGrid.getPosition(), Eigen::Vector2f(-32, -32));
 
-	mSearch->setAcceptCallback( [this](MetaDataList* result) { 
+	mSearch->setAcceptCallback([this, doneFunc](const ScraperSearchResult& result) { doneFunc(result); delete this; });
-		if(result != NULL)
+	mSearch->setCancelCallback([&] { delete this; });
-			this->mDoneFunc(*result);
-		else if(this->mSkipFunc)
-			this->mSkipFunc();
-
-		delete this;
-	});
 
 	mGrid.resetCursor();
-	//mSearch->setSearchParams(params); // also starts the search
+	mSearch->search(params); // start the search
 }
 
 bool GuiGameScraper::input(InputConfig* config, Input input)
 {
 	if(config->isMappedTo("b", input) && input.value)
 	{
-		if(mSkipFunc)
-			mSkipFunc();
 		delete this;
 		return true;
 	}
@@ -96,19 +63,6 @@ bool GuiGameScraper::input(InputConfig* config, Input input)
 	return GuiComponent::input(config, input);
 }
 
-void GuiGameScraper::update(int deltaTime)
-{
-	// HAAACK because AsyncReq wont get pushed in the right order if search happens on creation
-	if(mSearchCountdown > 0)
-	{
-		mSearchCountdown--;
-		if(mSearchCountdown == 0)
-			mSearch->setSearchParams(mSearchParams);
-	}
-
-	GuiComponent::update(deltaTime);
-}
-
 std::vector<HelpPrompt> GuiGameScraper::getHelpPrompts()
 {
 	return mGrid.getHelpPrompts();

src/guis/GuiGameScraper.h

@@ -7,16 +7,13 @@
 class GuiGameScraper : public GuiComponent
 {
 public:
-	GuiGameScraper(Window* window, ScraperSearchParams params, std::function<void(MetaDataList)> doneFunc, std::function<void()> skipFunc = nullptr);
+	GuiGameScraper(Window* window, ScraperSearchParams params, std::function<void(const ScraperSearchResult&)> doneFunc);
 
 	bool input(InputConfig* config, Input input) override;
-	void update(int deltaTime) override;
 	
 	virtual std::vector<HelpPrompt> getHelpPrompts() override;
 
 private:
-	int mSearchCountdown; // haaack
-
 	ComponentGrid mGrid;
 	NinePatchComponent mBox;
 
@@ -24,6 +21,5 @@ private:
 
 	ScraperSearchParams mSearchParams;
 
-	std::function<void(MetaDataList)> mDoneFunc;
+	std::function<void()> mCancelFunc;
-	std::function<void()> mSkipFunc;
 };

src/guis/GuiMetaDataEd.cpp

@@ -63,36 +63,8 @@ void GuiMetaDataEd::fetch()
 	mWindow->pushGui(scr);
 }
 
-void GuiMetaDataEd::fetchDone(MetaDataList result)
+void GuiMetaDataEd::fetchDone(const ScraperSearchResult& result)
 {
-	//TODO - replace this with resolveMetaDataAssetsAsync!
-
-	//this is a little tricky:
-	//go through the list of returned results, if anything is an image and the path looks like a URL:
-	//  (1) start an async download + resize (will create an AsyncReq that blocks further user input)
-	//      (when this is finished, call result.set(key, newly_downloaded_file_path) and call fetchDone() again)
-	//  (2) return from this function immediately
-	for(auto it = mMetaDataDecl.begin(); it != mMetaDataDecl.end(); it++)
-	{
-		std::string key = it->key;
-		std::string val = result.get(it->key);
-
-		//val is /probably/ a URL
-		if(it->type == MD_IMAGE_PATH && HttpReq::isUrl(val))
-		{
-			downloadImageAsync(mWindow, val, getSaveAsPath(mScraperParams, key, val), 
-				[this, result, key] (std::string filePath) mutable -> void {
-					//skip it
-					if(filePath.empty())
-						LOG(LogError) << "Error resolving boxart";
-
-					result.set(key, filePath);
-					this->fetchDone(result);
-			});
-			return;
-		}
-	}
-
 	for(unsigned int i = 0; i < mEditors.size(); i++)
 	{
 		//don't overwrite statistics
@@ -100,7 +72,7 @@ void GuiMetaDataEd::fetchDone(MetaDataList result)
 			continue;
 
 		const std::string& key = mMetaDataDecl.at(i).key;
-		mEditors.at(i)->setValue(result.get(key));
+		mEditors.at(i)->setValue(result.mdl.get(key));
 	}
 }
 

src/guis/GuiMetaDataEd.h

@@ -20,7 +20,7 @@ public:
 private:
 	void save();
 	void fetch();
-	void fetchDone(MetaDataList result);
+	void fetchDone(const ScraperSearchResult& result);
 
 	MenuComponent mMenu;
 

src/guis/GuiScraperLog.cpp

@@ -1,181 +0,0 @@
-#include "GuiScraperLog.h"
-#include "GuiGameScraper.h"
-#include "../Settings.h"
-#include "../Renderer.h"
-#include "../Log.h"
-#include "../XMLReader.h"
-
-GuiScraperLog::GuiScraperLog(Window* window, const std::queue<ScraperSearchParams>& searches, bool manualMode) : GuiComponent(window), 
-	mManualMode(manualMode), 
-	mBox(window, ":/frame.png"),
-	mSearches(searches),
-	mStatus(window), 
-	mTextLines(40), 
-	mSuccessCount(0), mSkippedCount(0)
-{
-	addChild(&mBox);
-	addChild(&mStatus);
-
-	setSize(Renderer::getScreenWidth() * 0.8f, (float)Renderer::getScreenHeight());
-	setPosition((Renderer::getScreenWidth() - mSize.x()) / 2, 0);
-
-	mStatus.setColor(0x000000FF);
-	mStatus.setSize(mSize.x(), (float)mStatus.getFont()->getHeight());
-	mStatus.setCentered(true);
-	updateStatus();
-
-	mBox.setEdgeColor(0x111111FF);
-	mBox.setCenterColor(0xDFDFDFFF);
-	mBox.fitTo(mSize, Eigen::Vector3f::Zero(), Eigen::Vector2f(8, 0));
-
-	mWindow->setAllowSleep(false);
-}
-
-GuiScraperLog::~GuiScraperLog()
-{
-	mWindow->setAllowSleep(true);
-}
-
-void GuiScraperLog::start()
-{
-	next();
-}
-
-void GuiScraperLog::next()
-{
-	if(mSearches.empty())
-	{
-		done();
-		return;
-	}
-
-	ScraperSearchParams search = mSearches.front();
-	mSearches.pop();
-
-	writeLine(getCleanFileName(search.game->getPath()), 0x0000FFFF);
-
-	if(mManualMode)
-	{
-		GuiGameScraper* ggs = new GuiGameScraper(mWindow, search, 
-			[this, search] (MetaDataList result) { resultFetched(search, result); }, 
-			[this, search] { resultEmpty(search); });
-
-		mWindow->pushGui(ggs);
-	}else{
-		std::shared_ptr<Scraper> scraper = Settings::getInstance()->getScraper();
-		scraper->getResultsAsync(search, mWindow, [this, search] (std::vector<MetaDataList> mdls) { 
-			if(mdls.empty())
-				resultEmpty(search);
-			else
-				resultFetched(search, mdls[0]);
-		});
-	}
-
-	updateStatus();
-}
-
-void GuiScraperLog::resultFetched(ScraperSearchParams params, MetaDataList mdl)
-{
-	writeLine("   -> \"" + mdl.get("name") + "\"", 0x0000FFFF);
-	writeLine("   Downloading images...", 0x0000FFFF);
-	resolveMetaDataAssetsAsync(mWindow, params, mdl, [this, params] (MetaDataList meta) { resultResolved(params, meta); });
-
-	//-> resultResolved
-}
-
-void GuiScraperLog::resultResolved(ScraperSearchParams params, MetaDataList mdl)
-{
-	//apply new metadata
-	params.game->metadata = mdl;
-
-	writeLine("   Success!", 0x00FF00FF);
-
-	//write changes to gamelist.xml
-	updateGamelist(params.system);
-
-	mSuccessCount++;
-
-	next();
-}
-
-void GuiScraperLog::resultEmpty(ScraperSearchParams search)
-{
-	if(mManualMode)
-		writeLine("   SKIPPING", 0xFF0000FF);
-	else
-		writeLine("   NO RESULTS, skipping", 0xFF0000FF);
-
-	LOG(LogInfo) << "Scraper skipping [" << getCleanFileName(search.game->getPath()) << "]";
-
-	mSkippedCount++;
-
-	next();
-}
-
-void GuiScraperLog::done()
-{
-	writeLine("===================================", 0x000000FF);
-	
-	std::stringstream ss;
-	ss << mSuccessCount << " successful, " << mSkippedCount << " skipped";
-	writeLine(ss.str(), 0x000000FF);
-
-	writeLine("DONE!! Press anything to continue.", 0x00FF00FF);
-	writeLine("===================================", 0x000000FF);
-
-	//done with everything!
-}
-
-void GuiScraperLog::writeLine(const std::string& line, unsigned int color)
-{
-	std::shared_ptr<TextComponent> cmp(new TextComponent(mWindow));
-	cmp->setText(line);
-	cmp->setColor(color);
-	cmp->setSize(mSize.x(), (float)cmp->getFont()->getHeight());
-
-	mTextLines.push_back(cmp);
-}
-
-void GuiScraperLog::render(const Eigen::Affine3f& parentTrans)
-{
-	renderChildren(parentTrans * getTransform());
-	
-	Eigen::Affine3f trans = parentTrans * getTransform();
-	
-	//draw messages
-	float fontHeight = (float)Font::get(FONT_SIZE_MEDIUM)->getHeight();
-	trans = trans.translate(Eigen::Vector3f(0, mSize.y() - fontHeight, 0));
-	
-	for(auto it = mTextLines.rbegin(); it != mTextLines.rend(); it++)
-	{
-		(*it)->render(trans);
-		trans = trans.translate(Eigen::Vector3f(0, -fontHeight, 0));
-
-		if(trans.translation().y() < fontHeight)
-			break;
-	}
-}
-
-bool GuiScraperLog::input(InputConfig* config, Input input)
-{
-	if(input.value != 0)
-	{
-		//we're done
-		if(mSearches.empty())
-		{
-			delete this;
-			return true;
-		}
-	}
-
-	return GuiComponent::input(config, input);
-}
-
-void GuiScraperLog::updateStatus()
-{
-	std::stringstream ss;
-
-	ss << mSearches.size() << " games remaining";
-
-	mStatus.setText(ss.str());
-}

src/guis/GuiScraperLog.h

@@ -1,46 +0,0 @@
-#pragma once
-
-#include "../GuiComponent.h"
-#include "../components/NinePatchComponent.h"
-#include "../scrapers/Scraper.h"
-#include "../components/TextComponent.h"
-
-#include <queue>
-#include <boost/circular_buffer.hpp>
-
-//A "terminal" of sorts for scraping.
-//Doesn't accept input, but renders log-style messages and handles the callback chain for multi-game scraping.
-class GuiScraperLog : public GuiComponent
-{
-public:
-	GuiScraperLog(Window* window, const std::queue<ScraperSearchParams>& params, bool manualMode);
-	~GuiScraperLog();
-
-	void start();
-
-	void render(const Eigen::Affine3f& parentTrans) override;
-	bool input(InputConfig* config, Input input) override;
-
-private:
-	void updateStatus();
-	void writeLine(const std::string& line, unsigned int color);
-
-	void resultFetched(ScraperSearchParams params, MetaDataList mdl);
-	void resultResolved(ScraperSearchParams params, MetaDataList mdl);
-	void resultEmpty(ScraperSearchParams params);
-
-	void next();
-	void done();
-
-	bool mManualMode;
-
-	NinePatchComponent mBox;
-
-	std::queue<ScraperSearchParams> mSearches;
-	
-	TextComponent mStatus;
-	boost::circular_buffer< std::shared_ptr<TextComponent> > mTextLines;
-
-	unsigned int mSuccessCount;
-	unsigned int mSkippedCount;
-};

src/guis/GuiScraperMulti.cpp

@@ -0,0 +1,102 @@
+#include "GuiScraperMulti.h"
+#include "../Renderer.h"
+
+#include "../components/TextComponent.h"
+#include "../components/ButtonComponent.h"
+#include "../components/ScraperSearchComponent.h"
+#include "../components/MenuComponent.h" // for makeButtonGrid
+#include "GuiMsgBox.h"
+
+using namespace Eigen;
+
+GuiScraperMulti::GuiScraperMulti(Window* window, const std::queue<ScraperSearchParams>& searches, bool approveResults) : 
+	GuiComponent(window), mBackground(window, ":/frame.png"), mGrid(window, Vector2i(1, 4)), 
+	mSearchQueue(searches)
+{
+	addChild(&mBackground);
+	addChild(&mGrid);
+
+	mTotalGames = mSearchQueue.size();
+	mCurrentGame = 0;
+
+	// set up grid
+	mTitle = std::make_shared<TextComponent>(mWindow, "SCRAPING IN PROGRESS", Font::get(FONT_SIZE_SMALL), 0x777777FF, true);
+	mGrid.setEntry(mTitle, Vector2i(0, 0), false, true);
+
+	mSubtitle = std::make_shared<TextComponent>(mWindow, "subtitle text", Font::get(FONT_SIZE_SMALL), 0x888888FF, true);
+	mGrid.setEntry(mSubtitle, Vector2i(0, 1), false, true);
+
+	mSearchComp = std::make_shared<ScraperSearchComponent>(mWindow, 
+		approveResults ? ScraperSearchComponent::ALWAYS_ACCEPT_MATCHING_CRC : ScraperSearchComponent::ALWAYS_ACCEPT_FIRST_RESULT);
+	mSearchComp->setAcceptCallback(std::bind(&GuiScraperMulti::acceptResult, this, std::placeholders::_1));
+	mSearchComp->setCancelCallback(std::bind(&GuiScraperMulti::finish, this));
+	mGrid.setEntry(mSearchComp, Vector2i(0, 2), approveResults, true);
+
+	std::vector< std::shared_ptr<ButtonComponent> > buttons;
+	buttons.push_back(std::make_shared<ButtonComponent>(mWindow, "INPUT", "manually search by name", nullptr));
+	buttons.push_back(std::make_shared<ButtonComponent>(mWindow, "SKIP", "skip this game", std::bind(&GuiScraperMulti::skip, this)));
+	buttons.push_back(std::make_shared<ButtonComponent>(mWindow, "STOP", "cancel scraping", std::bind(&GuiScraperMulti::finish, this)));
+	mButtonGrid = makeButtonGrid(mWindow, buttons);
+	mButtonGrid->setSize(mButtonGrid->getSize().x(), mButtonGrid->getSize().y() + 32);
+	mGrid.setEntry(mButtonGrid, Vector2i(0, 3), true, false);
+
+	setSize(Renderer::getScreenWidth() * 0.7f, Renderer::getScreenHeight() * 0.65f);
+	setPosition((Renderer::getScreenWidth() - mSize.x()) / 2, (Renderer::getScreenHeight() - mSize.y()) / 2);
+
+	doNextSearch();
+}
+
+void GuiScraperMulti::onSizeChanged()
+{
+	mBackground.fitTo(mSize, Vector3f::Zero(), Vector2f(-32, -32));
+	mGrid.setSize(mSize);
+
+	mGrid.setRowHeightPerc(0, mTitle->getFont()->getHeight() / mGrid.getSize().y());
+	mGrid.setRowHeightPerc(1, mSubtitle->getFont()->getHeight() / mGrid.getSize().y());
+	mGrid.setRowHeightPerc(3, mButtonGrid->getSize().y() / mGrid.getSize().y());
+}
+
+void GuiScraperMulti::doNextSearch()
+{
+	if(mSearchQueue.empty())
+	{
+		finish();
+		return;
+	}
+
+	// update subtitle
+	std::stringstream ss;
+	ss << "GAME " << (mCurrentGame + 1) << " OF " << mTotalGames;
+	mSubtitle->setText(ss.str());
+
+	mSearchComp->search(mSearchQueue.front());
+}
+
+void GuiScraperMulti::acceptResult(const ScraperSearchResult& result)
+{
+	ScraperSearchParams& search = mSearchQueue.front();
+
+	search.game->metadata = result.mdl;
+
+	mSearchQueue.pop();
+	mCurrentGame++;
+	doNextSearch();
+}
+
+void GuiScraperMulti::skip()
+{
+	mSearchQueue.pop();
+	mCurrentGame++;
+	doNextSearch();
+}
+
+void GuiScraperMulti::finish()
+{
+	mWindow->pushGui(new GuiMsgBox(mWindow, "SCRAPING COMPLETE!", 
+		"OK", [&] { delete this; }));
+}
+
+std::vector<HelpPrompt> GuiScraperMulti::getHelpPrompts()
+{
+	return mGrid.getHelpPrompts();
+}

src/guis/GuiScraperMulti.h

@@ -0,0 +1,39 @@
+#pragma once
+
+#include "../GuiComponent.h"
+#include "../components/NinePatchComponent.h"
+#include "../components/ComponentGrid.h"
+#include "../scrapers/Scraper.h"
+
+#include <queue>
+
+class ScraperSearchComponent;
+class TextComponent;
+
+class GuiScraperMulti : public GuiComponent
+{
+public:
+	GuiScraperMulti(Window* window, const std::queue<ScraperSearchParams>& searches, bool approveResults);
+
+	void onSizeChanged() override;
+	std::vector<HelpPrompt> getHelpPrompts() override;
+
+private:
+	void acceptResult(const ScraperSearchResult& result);
+	void skip();
+	void doNextSearch();
+	
+	void finish();
+
+	unsigned int mTotalGames;
+	unsigned int mCurrentGame;
+	std::queue<ScraperSearchParams> mSearchQueue;
+
+	NinePatchComponent mBackground;
+	ComponentGrid mGrid;
+
+	std::shared_ptr<TextComponent> mTitle;
+	std::shared_ptr<TextComponent> mSubtitle;
+	std::shared_ptr<ScraperSearchComponent> mSearchComp;
+	std::shared_ptr<ComponentGrid> mButtonGrid;
+};

src/guis/GuiScraperStart.cpp

@@ -1,5 +1,5 @@
 #include "GuiScraperStart.h"
-#include "GuiScraperLog.h"
+#include "GuiScraperMulti.h"
 #include "GuiMsgBox.h"
 
 #include "../components/TextComponent.h"
@@ -14,9 +14,9 @@ GuiScraperStart::GuiScraperStart(Window* window) : GuiComponent(window),
 	// add filters (with first one selected)
 	mFilters = std::make_shared< OptionListComponent<GameFilterFunc> >(mWindow, "SCRAPE THESE GAMES", false);
 	mFilters->add("All Games", 
-		[](SystemData*, FileData*) -> bool { return true; }, true);
+		[](SystemData*, FileData*) -> bool { return true; }, false);
 	mFilters->add("Only missing image", 
-		[](SystemData*, FileData* g) -> bool { return g->metadata.get("image").empty(); }, false);
+		[](SystemData*, FileData* g) -> bool { return g->metadata.get("image").empty(); }, true);
 	mMenu.addWithLabel("Filter", mFilters);
 
 	//add systems (all with a platformid specified selected)
@@ -57,9 +57,8 @@ void GuiScraperStart::start()
 {
 	std::queue<ScraperSearchParams> searches = getSearches(mSystems->getSelectedObjects(), mFilters->getSelected());
 
-	GuiScraperLog* gsl = new GuiScraperLog(mWindow, searches, mApproveResults->getState());
+	GuiScraperMulti* gsm = new GuiScraperMulti(mWindow, searches, mApproveResults->getState());
-	mWindow->pushGui(gsl);
+	mWindow->pushGui(gsm);
-	gsl->start();
 	delete this;
 }
 

src/scrapers/GamesDBScraper.cpp

@@ -55,7 +55,7 @@ const std::map<PlatformId, const char*> gamesdb_platformid_map = boost::assign::
 	(TURBOGRAFX_16, "TurboGrafx 16");
 
 
-std::shared_ptr<HttpReq> GamesDBScraper::makeHttpReq(ScraperSearchParams params)
+std::unique_ptr<ScraperSearchHandle> GamesDBScraper::getResultsAsync(const ScraperSearchParams& params)
 {
 	std::string path = "/api/GetGame.php?";
 
@@ -71,25 +71,41 @@ std::shared_ptr<HttpReq> GamesDBScraper::makeHttpReq(ScraperSearchParams params)
 		path += HttpReq::urlEncode(gamesdb_platformid_map.at(params.system->getPlatformId()));
 	}
 
-	return std::make_shared<HttpReq>("thegamesdb.net" + path);
+	path = "thegamesdb.net" + path;
+
+	return std::unique_ptr<ScraperSearchHandle>(new GamesDBHandle(params, path));
 }
 
-std::vector<MetaDataList> GamesDBScraper::parseReq(ScraperSearchParams params, std::shared_ptr<HttpReq> req)
+GamesDBHandle::GamesDBHandle(const ScraperSearchParams& params, const std::string& url) : 
+	mReq(std::unique_ptr<HttpReq>(new HttpReq(url)))
 {
-	std::vector<MetaDataList> mdl;
+	setStatus(SEARCH_IN_PROGRESS);
+}
 
-	if(req->status() != HttpReq::REQ_SUCCESS)
+void GamesDBHandle::update()
+{
+	if(mReq->status() == HttpReq::REQ_IN_PROGRESS)
+		return;
+
+	if(mReq->status() != HttpReq::REQ_SUCCESS)
 	{
-		LOG(LogError) << "HttpReq error";
+		std::stringstream ss;
-		return mdl;
+		ss << "Network error - " << mReq->getErrorMsg();
+		setError(ss.str());
+		return;
 	}
 
+	// our HTTP request was successful
+	// try to build our result list
+
+	std::vector<ScraperSearchResult> results;
+
 	pugi::xml_document doc;
-	pugi::xml_parse_result parseResult = doc.load(req->getContent().c_str());
+	pugi::xml_parse_result parseResult = doc.load(mReq->getContent().c_str());
 	if(!parseResult)
 	{
-		LOG(LogError) << "Error parsing XML";
+		setError("Error parsing XML");
-		return mdl;
+		return;
 	}
 
 	pugi::xml_node data = doc.child("Data");
@@ -100,24 +116,25 @@ std::vector<MetaDataList> GamesDBScraper::parseReq(ScraperSearchParams params, s
 	pugi::xml_node game = data.child("Game");
 	while(game && resultNum < MAX_SCRAPER_RESULTS)
 	{
-		mdl.push_back(MetaDataList(GAME_METADATA));
+		ScraperSearchResult result;
-		mdl.back().set("name", game.child("GameTitle").text().get());
+
-		mdl.back().set("desc", game.child("Overview").text().get());
+		result.mdl.set("name", game.child("GameTitle").text().get());
+		result.mdl.set("desc", game.child("Overview").text().get());
 
 		boost::posix_time::ptime rd = string_to_ptime(game.child("ReleaseDate").text().get(), "%m/%d/%Y");
-		mdl.back().setTime("releasedate", rd);
+		result.mdl.setTime("releasedate", rd);
 
-		mdl.back().set("developer", game.child("Developer").text().get());
+		result.mdl.set("developer", game.child("Developer").text().get());
-		mdl.back().set("publisher", game.child("Publisher").text().get());
+		result.mdl.set("publisher", game.child("Publisher").text().get());
-		mdl.back().set("genre", game.child("Genres").first_child().text().get());
+		result.mdl.set("genre", game.child("Genres").first_child().text().get());
-		mdl.back().set("players", game.child("Players").text().get());
+		result.mdl.set("players", game.child("Players").text().get());
 
 		if(Settings::getInstance()->getBool("ScrapeRatings") && game.child("Rating"))
 		{
 			float ratingVal = (game.child("Rating").text().as_int() / 10.0f);
 			std::stringstream ss;
 			ss << ratingVal;
-			mdl.back().set("rating", ss.str());
+			result.mdl.set("rating", ss.str());
 		}
 
 		pugi::xml_node images = game.child("Images");
@@ -128,14 +145,18 @@ std::vector<MetaDataList> GamesDBScraper::parseReq(ScraperSearchParams params, s
 
 			if(art)
 			{
-				mdl.back().set("thumbnail", baseImageUrl + art.attribute("thumb").as_string());
+				result.thumbnailUrl = baseImageUrl + art.attribute("thumb").as_string();
-				mdl.back().set("image", baseImageUrl + art.text().get());
+				result.imageUrl = baseImageUrl + art.text().get();
 			}
 		}
 
+		results.push_back(result);
+
 		resultNum++;
 		game = game.next_sibling("Game");
 	}
 
-	return mdl;
+	setStatus(SEARCH_DONE);
+	setResults(results);
+	return;
 }

src/scrapers/GamesDBScraper.h

@@ -3,11 +3,22 @@
 #include "Scraper.h"
 #include "../HttpReq.h"
 
+class GamesDBHandle : public ScraperSearchHandle
+{
+public:
+	GamesDBHandle(const ScraperSearchParams& params, const std::string& url);
+
+	void update() override;
+
+private:
+	std::unique_ptr<HttpReq> mReq;
+	ScraperSearchParams mParams;
+};
+
 class GamesDBScraper : public Scraper
 {
 public:
+	std::unique_ptr<ScraperSearchHandle> getResultsAsync(const ScraperSearchParams& params) override;
+
 	const char* getName();
-private:
-	std::shared_ptr<HttpReq> makeHttpReq(ScraperSearchParams params) override;
-	std::vector<MetaDataList> parseReq(ScraperSearchParams params, std::shared_ptr<HttpReq>) override;
 };

src/scrapers/Scraper.cpp

@@ -9,31 +9,21 @@
 #include "GamesDBScraper.h"
 #include "TheArchiveScraper.h"
 
-std::vector<MetaDataList> Scraper::getResults(ScraperSearchParams params)
+std::string ScraperSearchHandle::getStatusString()
 {
-	std::shared_ptr<HttpReq> req = makeHttpReq(params);
+	switch(mStatus)
-	while(req->status() == HttpReq::REQ_IN_PROGRESS);
-
-	return parseReq(params, req);
-}
-
-void Scraper::getResultsAsync(ScraperSearchParams params, Window* window, std::function<void(std::vector<MetaDataList>)> returnFunc)
-{
-	std::shared_ptr<HttpReq> httpreq = makeHttpReq(params);
-	AsyncReqComponent* req = new AsyncReqComponent(window, httpreq,
-		[this, params, returnFunc] (std::shared_ptr<HttpReq> r)
 	{
-		returnFunc(parseReq(params, r));
+	case SEARCH_IN_PROGRESS:
-	}, [returnFunc] ()
+		return "search in progress";
-	{
+	case SEARCH_ERROR:
-		returnFunc(std::vector<MetaDataList>());
+		return mError;
-	});
+	case SEARCH_DONE:
-
+		return "search done";
-	window->pushGui(req);
+	default:
+		return "something impossible has occured";
+	}
 }
 
-
-
 std::string processFileDownload(std::shared_ptr<HttpReq> r, std::string saveAs)
 {
 	if(r->status() != HttpReq::REQ_SUCCESS)

src/scrapers/Scraper.h

@@ -16,17 +16,53 @@ struct ScraperSearchParams
 	std::string nameOverride;
 };
 
+struct ScraperSearchResult
+{
+	ScraperSearchResult() : mdl(GAME_METADATA) {};
+
+	MetaDataList mdl;
+	std::string imageUrl;
+	std::string thumbnailUrl;
+};
+
+enum ScraperSearchStatus
+{
+	SEARCH_IN_PROGRESS,
+	SEARCH_ERROR,
+	SEARCH_DONE
+};
+
+class ScraperSearchHandle
+{
+public:
+	virtual void update() = 0;
+
+	// Update and return the latest status.
+	inline ScraperSearchStatus status() { update(); return mStatus; }
+
+	// User-friendly string of our current status.  Will return error message if status() == SEARCH_ERROR.
+	std::string getStatusString();
+
+	inline const std::vector<ScraperSearchResult>& getResults() const { assert(mStatus != SEARCH_IN_PROGRESS); return mResults; }
+
+protected:
+	inline void setError(const std::string& error) { setStatus(SEARCH_ERROR); mError = error; }
+	inline void setStatus(ScraperSearchStatus status) { mStatus = status; }
+	inline void setResults(const std::vector<ScraperSearchResult>& results) { mResults = results; }
+
+private:
+	std::string mError;
+	ScraperSearchStatus mStatus;
+	std::vector<ScraperSearchResult> mResults;
+};
+
 class Scraper
 {
 public:
 	//Get a list of potential results.
-	virtual std::vector<MetaDataList> getResults(ScraperSearchParams params);
+	virtual std::unique_ptr<ScraperSearchHandle> getResultsAsync(const ScraperSearchParams& params) = 0;
-	virtual void getResultsAsync(ScraperSearchParams params, Window* window, std::function<void(std::vector<MetaDataList>)> returnFunc);
 
 	virtual const char* getName() = 0;
-private:
-	virtual std::shared_ptr<HttpReq> makeHttpReq(ScraperSearchParams params) = 0;
-	virtual std::vector<MetaDataList> parseReq(ScraperSearchParams params, std::shared_ptr<HttpReq>) = 0;
 };
 
 std::shared_ptr<Scraper> createScraperByName(const std::string& name);

src/scrapers/TheArchiveScraper.cpp

@@ -6,7 +6,7 @@
 
 const char* TheArchiveScraper::getName() { return "TheArchive"; }
 
-std::shared_ptr<HttpReq> TheArchiveScraper::makeHttpReq(ScraperSearchParams params)
+std::unique_ptr<ScraperSearchHandle> TheArchiveScraper::getResultsAsync(const ScraperSearchParams& params)
 {
 	std::string path = "/2.0/Archive.search/xml/7TTRM4MNTIKR2NNAGASURHJOZJ3QXQC5/";
 
@@ -17,25 +17,44 @@ std::shared_ptr<HttpReq> TheArchiveScraper::makeHttpReq(ScraperSearchParams para
 	path += HttpReq::urlEncode(cleanName);
 	//platform TODO, should use some params.system get method
 
-	return std::make_shared<HttpReq>("api.archive.vg" + path);
+	path = "api.archive.vg" + path;
+
+	return std::unique_ptr<ScraperSearchHandle>(new TheArchiveHandle(params, path));
 }
 
-std::vector<MetaDataList> TheArchiveScraper::parseReq(ScraperSearchParams params, std::shared_ptr<HttpReq> req)
+TheArchiveHandle::TheArchiveHandle(const ScraperSearchParams& params, const std::string& url) : 
+	mReq(std::unique_ptr<HttpReq>(new HttpReq(url)))
 {
-	std::vector<MetaDataList> mdl;
+	setStatus(SEARCH_IN_PROGRESS);
+}
 
-	if(req->status() != HttpReq::REQ_SUCCESS)
+void TheArchiveHandle::update()
+{
+	if(status() == SEARCH_DONE)
+		return;
+
+	if(mReq->status() == HttpReq::REQ_IN_PROGRESS)
+		return;
+
+	if(mReq->status() != HttpReq::REQ_SUCCESS)
 	{
-		LOG(LogError) << "HttpReq error";
+		std::stringstream ss;
-		return mdl;
+		ss << "Network error: " << mReq->getErrorMsg();
+		setError(ss.str());
+		return;
 	}
 
+	// if we're here, our HTTP request finished successfully
+	
+	// so, let's try building our result list
+	std::vector<ScraperSearchResult> results;
+
 	pugi::xml_document doc;
-	pugi::xml_parse_result parseResult = doc.load(req->getContent().c_str());
+	pugi::xml_parse_result parseResult = doc.load(mReq->getContent().c_str());
 	if(!parseResult)
 	{
-		LOG(LogError) << "Error parsing XML";
+		setError("Error parsing XML");
-		return mdl;
+		return;
 	}
 
 	pugi::xml_node data = doc.child("OpenSearchDescription").child("games");
@@ -44,30 +63,34 @@ std::vector<MetaDataList> TheArchiveScraper::parseReq(ScraperSearchParams params
 	pugi::xml_node game = data.child("game");
 	while(game && resultNum < MAX_SCRAPER_RESULTS)
 	{
-		mdl.push_back(MetaDataList(GAME_METADATA));
+		ScraperSearchResult result;
-		mdl.back().set("name", game.child("title").text().get());
+
-		mdl.back().set("desc", game.child("description").text().get());
+		result.mdl.set("name", game.child("title").text().get());
+		result.mdl.set("desc", game.child("description").text().get());
 
 		//Archive.search does not return ratings
 
-		mdl.back().set("developer", game.child("developer").text().get());
+		result.mdl.set("developer", game.child("developer").text().get());
 
 		std::string genre = game.child("genre").text().get();
 		size_t search = genre.find_last_of(" &gt; ");
 		genre = genre.substr(search == std::string::npos ? 0 : search, std::string::npos);
-		mdl.back().set("genre", genre);
+		result.mdl.set("genre", genre);
 
 		pugi::xml_node image = game.child("box_front");
 		pugi::xml_node thumbnail = game.child("box_front_small");
 
-		if (image)
+		if(image)
-			mdl.back().set("image",image.text().get());
+			result.imageUrl = image.text().get();
-		if (thumbnail)
+		if(thumbnail)
-			mdl.back().set("thumbnail", thumbnail.text().get());
+			result.thumbnailUrl = thumbnail.text().get();
+
+		results.push_back(result);
 
 		resultNum++;
 		game = game.next_sibling("game");
 	}
 
-	return mdl;
+	setStatus(SEARCH_DONE);
+	setResults(results);
 }

src/scrapers/TheArchiveScraper.h

@@ -3,12 +3,22 @@
 #include "Scraper.h"
 #include "../HttpReq.h"
 
+class TheArchiveHandle : public ScraperSearchHandle
+{
+public:
+	TheArchiveHandle(const ScraperSearchParams& params, const std::string& url);
+
+	void update() override;
+
+private:
+	std::unique_ptr<HttpReq> mReq;
+	ScraperSearchParams mParams;
+};
+
 class TheArchiveScraper : public Scraper
 {
 public:
-	const char* getName();
+	std::unique_ptr<ScraperSearchHandle> getResultsAsync(const ScraperSearchParams& params) override;
-private:
-	std::shared_ptr<HttpReq> makeHttpReq(ScraperSearchParams params) override;
-	std::vector<MetaDataList> parseReq(ScraperSearchParams params, std::shared_ptr<HttpReq>) override;
-};
 
+	const char* getName();
+};