Multi-game scraper seems to be functional!

CMakeLists.txt

@@ -184,6 +184,7 @@ set(ES_HEADERS
     ${CMAKE_CURRENT_SOURCE_DIR}/src/components/GuiMenu.h
     ${CMAKE_CURRENT_SOURCE_DIR}/src/components/GuiSettingsMenu.h
     ${CMAKE_CURRENT_SOURCE_DIR}/src/components/GuiScraperStart.h
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/components/GuiScraperLog.h
     ${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/Scraper.h
     ${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/GamesDBScraper.h
     ${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/TheArchiveScraper.h
@@ -241,6 +242,7 @@ set(ES_SOURCES
     ${CMAKE_CURRENT_SOURCE_DIR}/src/components/GuiMenu.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/components/GuiSettingsMenu.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/components/GuiScraperStart.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/components/GuiScraperLog.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/Scraper.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/GamesDBScraper.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/TheArchiveScraper.cpp

src/ScraperCmdLine.cpp

@@ -255,7 +255,11 @@ int run_scraper_cmdline()
 					if(url.length() != urlShort.length()) urlShort += "...";
 
 					out << "   " << game->metadata()->get("name") << " [from: " << urlShort << "]...\n";
-					game->metadata()->set(key, downloadImage(url, getSaveAsPath((*sysIt)->getName(), game->getCleanName(), url)));
+
+					ScraperSearchParams p;
+					p.game = game;
+					p.system = *sysIt;
+					game->metadata()->set(key, downloadImage(url, getSaveAsPath(p, key, url)));
 					if(game->metadata()->get(key).empty())
 					{
 						out << "     FAILED! Skipping.\n";

src/Settings.cpp

@@ -33,6 +33,7 @@ void Settings::setDefaults()
 	mBoolMap["DEBUG"] = false;
 	mBoolMap["WINDOWED"] = false;
 	mBoolMap["DISABLESOUNDS"] = false;
+	mBoolMap["DisableGamelistWrites"] = false;
 
 	mIntMap["DIMTIME"] = 30*1000;
 	mIntMap["ScraperResizeWidth"] = 450;
@@ -40,6 +41,7 @@ void Settings::setDefaults()
 
 	mIntMap["GameListSortIndex"] = 0;
 
+
 	mScraper = std::shared_ptr<Scraper>(new GamesDBScraper());
 }
 

src/XMLReader.cpp

@@ -4,6 +4,7 @@
 #include "pugiXML/pugixml.hpp"
 #include <boost/filesystem.hpp>
 #include "Log.h"
+#include "Settings.h"
 
 //this is obviously an incredibly inefficient way to go about searching
 //but I don't think it'll matter too much with the size of most collections
@@ -193,6 +194,9 @@ void updateGamelist(SystemData* system)
 	//We have the complete information for every game though, so we can simply remove a game
 	//we already have in the system from the XML, and then add it back from its GameData information...
 
+	if(Settings::getInstance()->getBool("DisableGamelistWrites"))
+		return;
+
 	std::string xmlpath = system->getGamelistPath();
 	if(xmlpath.empty())
 		return;

src/components/GuiMetaDataEd.cpp

@@ -126,6 +126,8 @@ void GuiMetaDataEd::fetch()
 
 void GuiMetaDataEd::fetchDone(MetaDataList result)
 {
+	//TODO - replace this with resolveMetaDataAssetsAsync!
+
 	//this is a little tricky:
 	//go through the list of returned results, if anything is an image and the path looks like a URL:
 	//  (1) start an async download + resize (will create an AsyncReq that blocks further user input)
@@ -139,7 +141,7 @@ void GuiMetaDataEd::fetchDone(MetaDataList result)
 		//val is /probably/ a URL
 		if(it->type == MD_IMAGE_PATH && HttpReq::isUrl(val))
 		{
-			downloadImageAsync(mWindow, val, getSaveAsPath(mScraperParams.system->getName(), mScraperParams.game->getCleanName() + "-" + key, val), 
+			downloadImageAsync(mWindow, val, getSaveAsPath(mScraperParams, key, val), 
 				[this, result, key] (std::string filePath) mutable -> void {
 					//skip it
 					if(filePath.empty())

src/components/GuiScraperLog.cpp

@@ -0,0 +1,158 @@
+#include "GuiScraperLog.h"
+#include "../Settings.h"
+#include "GuiGameScraper.h"
+#include "../Renderer.h"
+#include "../Log.h"
+
+GuiScraperLog::GuiScraperLog(Window* window, const std::queue<ScraperSearchParams>& searches, bool manualMode) : GuiComponent(window), 
+	mManualMode(manualMode), 
+	mBox(window, ":/frame.png"),
+	mSearches(searches),
+	mStatus(window), 
+	mTextLines(40)
+{
+	addChild(&mBox);
+	addChild(&mStatus);
+
+	setSize(Renderer::getScreenWidth() * 0.8f, (float)Renderer::getScreenHeight());
+	setPosition((Renderer::getScreenWidth() - mSize.x()) / 2, 0);
+
+	mStatus.setColor(0x000000FF);
+	mStatus.setSize(mSize.x(), (float)mStatus.getFont()->getHeight());
+	mStatus.setCentered(true);
+	updateStatus();
+
+	mBox.setEdgeColor(0x111111FF);
+	mBox.setCenterColor(0xDFDFDFFF);
+	mBox.fitTo(mSize, Eigen::Vector3f::Zero(), Eigen::Vector2f(8, 0));
+}
+
+void GuiScraperLog::start()
+{
+	next();
+}
+
+void GuiScraperLog::next()
+{
+	if(mSearches.empty())
+	{
+		done();
+		return;
+	}
+
+	ScraperSearchParams search = mSearches.front();
+	mSearches.pop();
+
+	writeLine(search.game->getCleanName(), 0x0000FFFF);
+
+	if(mManualMode)
+	{
+		GuiGameScraper* ggs = new GuiGameScraper(mWindow, search, 
+			[this, search] (MetaDataList result) { resultFetched(search, result); }, 
+			[this, search] { resultEmpty(search); });
+
+		mWindow->pushGui(ggs);
+		ggs->search();
+	}else{
+		std::shared_ptr<Scraper> scraper = Settings::getInstance()->getScraper();
+		scraper->getResultsAsync(search, mWindow, [this, search] (std::vector<MetaDataList> mdls) { 
+			if(mdls.empty())
+				resultEmpty(search);
+			else
+				resultFetched(search, mdls[0]);
+		});
+	}
+
+	updateStatus();
+}
+
+void GuiScraperLog::resultFetched(ScraperSearchParams params, MetaDataList mdl)
+{
+	writeLine("   -> \"" + mdl.get("name") + "\"", 0x0000FFFF);
+	writeLine("   Downloading images...", 0x0000FFFF);
+	resolveMetaDataAssetsAsync(mWindow, params, mdl, [this, params] (MetaDataList meta) { resultResolved(params, meta); });
+
+	//-> resultResolved
+}
+
+void GuiScraperLog::resultResolved(ScraperSearchParams params, MetaDataList mdl)
+{
+	writeLine("   Success!", 0x00FF00FF);
+
+	next();
+}
+
+void GuiScraperLog::resultEmpty(ScraperSearchParams search)
+{
+	if(mManualMode)
+		writeLine("   SKIPPING", 0xFF0000FF);
+	else
+		writeLine("   NO RESULTS, skipping", 0xFF0000FF);
+
+	LOG(LogInfo) << "Scraper skipping [" << search.game->getCleanName() << "]";
+
+	next();
+}
+
+void GuiScraperLog::done()
+{
+	writeLine("====================", 0x000000FF);
+	writeLine("DONE!!", 0x00FF00FF);
+	writeLine("====================", 0x000000FF);
+
+	//done with everything!
+}
+
+void GuiScraperLog::writeLine(const std::string& line, unsigned int color)
+{
+	std::shared_ptr<TextComponent> cmp(new TextComponent(mWindow));
+	cmp->setText(line);
+	cmp->setColor(color);
+	cmp->setSize(mSize.x(), (float)cmp->getFont()->getHeight());
+
+	mTextLines.push_back(cmp);
+}
+
+void GuiScraperLog::render(const Eigen::Affine3f& parentTrans)
+{
+	renderChildren(parentTrans * getTransform());
+	
+	Eigen::Affine3f trans = parentTrans * getTransform();
+	
+	//draw messages
+	float fontHeight = (float)Font::get(FONT_SIZE_MEDIUM)->getHeight();
+	trans = trans.translate(Eigen::Vector3f(0, mSize.y() - fontHeight, 0));
+	
+	for(auto it = mTextLines.rbegin(); it != mTextLines.rend(); it++)
+	{
+		(*it)->render(trans);
+		trans = trans.translate(Eigen::Vector3f(0, -fontHeight, 0));
+
+		if(trans.translation().y() < fontHeight)
+			break;
+	}
+}
+
+bool GuiScraperLog::input(InputConfig* config, Input input)
+{
+	if(input.value != 0)
+	{
+		//we're done
+		if(mSearches.empty())
+		{
+			delete this;
+			return true;
+		}
+	}
+
+	return GuiComponent::input(config, input);
+}
+
+void GuiScraperLog::updateStatus()
+{
+	std::stringstream ss;
+
+	ss << mSearches.size() << " games remaining";
+
+	mStatus.setText(ss.str());
+}

src/components/GuiScraperLog.h

@@ -0,0 +1,39 @@
+#pragma once
+
+#include "../GuiComponent.h"
+#include "NinePatchComponent.h"
+#include <queue>
+#include "../scrapers/Scraper.h"
+#include <boost/circular_buffer.hpp>
+#include "TextComponent.h"
+
+class GuiScraperLog : public GuiComponent
+{
+public:
+	GuiScraperLog(Window* window, const std::queue<ScraperSearchParams>& params, bool manualMode);
+
+	void start();
+
+	void render(const Eigen::Affine3f& parentTrans) override;
+	bool input(InputConfig* config, Input input) override;
+
+private:
+	void updateStatus();
+	void writeLine(const std::string& line, unsigned int color);
+
+	void resultFetched(ScraperSearchParams params, MetaDataList mdl);
+	void resultResolved(ScraperSearchParams params, MetaDataList mdl);
+	void resultEmpty(ScraperSearchParams params);
+
+	void next();
+	void done();
+
+	bool mManualMode;
+
+	NinePatchComponent mBox;
+
+	std::queue<ScraperSearchParams> mSearches;
+	
+	TextComponent mStatus;
+	boost::circular_buffer< std::shared_ptr<TextComponent> > mTextLines;
+};

src/components/GuiScraperStart.cpp

@@ -1,5 +1,5 @@
 #include "GuiScraperStart.h"
-#include "GuiMsgBoxOk.h"
+#include "GuiScraperLog.h"
 
 GuiScraperStart::GuiScraperStart(Window* window) : GuiComponent(window),
 	mBox(window, ":/frame.png"),
@@ -57,7 +57,10 @@ void GuiScraperStart::start()
 {
 	std::queue<ScraperSearchParams> searches = getSearches(mSystemsOpt.getSelectedObjects(), mFiltersOpt.getSelectedObjects()[0]);
 
-	mWindow->pushGui(new GuiMsgBoxOk(mWindow, "this isn't implemented yet"));
+	GuiScraperLog* gsl = new GuiScraperLog(mWindow, searches, mManualSwitch.getState());
+	mWindow->pushGui(gsl);
+	gsl->start();
+	delete this;
 }
 
 std::queue<ScraperSearchParams> GuiScraperStart::getSearches(std::vector<SystemData*> systems, GameFilterFunc selector)

src/scrapers/Scraper.cpp

@@ -140,8 +140,11 @@ std::string downloadImage(const std::string& url, const std::string& saveAs)
 	return file;
 }
 
-std::string getSaveAsPath(const std::string& subdirectory, const std::string& name, const std::string& url)
+std::string getSaveAsPath(const ScraperSearchParams& params, const std::string& suffix, const std::string& url)
 {
+	const std::string subdirectory = params.system->getName();
+	const std::string name = params.game->getCleanName() + "-" + suffix;
+
 	std::string path = getHomePath() + "/.emulationstation/downloaded_images/";
 
 	if(!boost::filesystem::exists(path))
@@ -171,3 +174,31 @@ std::shared_ptr<Scraper> createScraperByName(const std::string& name)
 
 	return nullptr;
 }
+
+void resolveMetaDataAssetsAsync(Window* window, const ScraperSearchParams& params, MetaDataList mdl, std::function<void(MetaDataList)> returnFunc)
+{
+	std::vector<MetaDataDecl> mdd = params.system->getGameMDD();
+	for(auto it = mdd.begin(); it != mdd.end(); it++)
+	{
+		std::string key = it->key;
+		std::string val = mdl.get(key);
+		if(it->type == MD_IMAGE_PATH && HttpReq::isUrl(val))
+		{
+			downloadImageAsync(window, val, getSaveAsPath(params, key, val), [window, params, mdl, key, returnFunc] (std::string savedAs) mutable -> 
+				void 
+			{
+				if(savedAs.empty())
+				{
+					//error
+					LOG(LogError) << "Could not resolve image for [" << params.game->getCleanName() << "]! Skipping.";
+				}
+
+				mdl.set(key, savedAs);
+				resolveMetaDataAssetsAsync(window, params, mdl, returnFunc);
+			});
+			return;
+		}
+	}
+
+	returnFunc(mdl);
+}

src/scrapers/Scraper.h

@@ -32,9 +32,9 @@ private:
 
 std::shared_ptr<Scraper> createScraperByName(const std::string& name);
 
-//About the same as "~/.emulationstation/downloaded_images/[subdirectory]/[name].[url's extension]".
+//About the same as "~/.emulationstation/downloaded_images/[system_name]/[game_name].[url's extension]".
 //Will create the "downloaded_images" and "subdirectory" directories if they do not exist.
-std::string getSaveAsPath(const std::string& subdirectory, const std::string& name, const std::string& url);
+std::string getSaveAsPath(const ScraperSearchParams& params, const std::string& suffix, const std::string& url);
 
 //Returns the path to the downloaded file (saveAs) on completion.
 //Returns empty string if an error occured.
@@ -47,6 +47,8 @@ std::string downloadImage(const std::string& url, const std::string& saveAs);
 //Same as downloadImage, just async.
 void downloadImageAsync(Window* window, const std::string& url, const std::string& saveAs, std::function<void(std::string)> returnFunc);
 
+void resolveMetaDataAssetsAsync(Window* window, const ScraperSearchParams& params, MetaDataList mdl, std::function<void(MetaDataList)> returnFunc);
+
 //You can pass 0 for maxWidth or maxHeight to automatically keep the aspect ratio.
 //Will overwrite the image at [path] with the new resized one.
 void resizeImage(const std::string& path, int maxWidth, int maxHeight);