Initial command-line scraper.

CMakeLists.txt

@@ -150,6 +150,7 @@ set(ES_HEADERS
     ${CMAKE_CURRENT_SOURCE_DIR}/src/platform.h
     ${CMAKE_CURRENT_SOURCE_DIR}/src/PlatformId.h
     ${CMAKE_CURRENT_SOURCE_DIR}/src/Renderer.h
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/ScraperCmdLine.h
     ${CMAKE_CURRENT_SOURCE_DIR}/src/Settings.h
     ${CMAKE_CURRENT_SOURCE_DIR}/src/Sound.h
     ${CMAKE_CURRENT_SOURCE_DIR}/src/SystemData.h
@@ -205,6 +206,7 @@ set(ES_SOURCES
     ${CMAKE_CURRENT_SOURCE_DIR}/src/platform.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/Renderer_draw_gl.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/Renderer_init.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/ScraperCmdLine.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/Settings.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/Sound.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/SystemData.cpp

src/HttpReq.cpp

@@ -206,7 +206,13 @@ void HttpReq::handleReadContent(const boost::system::error_code& err)
 
 HttpReq::Status HttpReq::status()
 {
-	io_service.poll();
+	try {
+		io_service.poll();
+	} catch(boost::system::system_error& e)
+	{
+		LOG(LogError) << "Boost.ASIO system error!  " << e.what();
+	}
+
 	return mStatus;
 }
 

src/ScraperCmdLine.cpp

@@ -0,0 +1,242 @@
+#include "ScraperCmdLine.h"
+#include <iostream>
+#include <vector>
+#include "SystemData.h"
+#include "Settings.h"
+#include <signal.h>
+#include "Log.h"
+
+std::ostream& out = std::cout;
+
+void handle_interrupt_signal(int p)
+{
+	sleep(50);
+
+	LOG(LogInfo) << "Interrupt received during scrape...";
+
+	SystemData::deleteSystems();
+
+	exit(1);
+}
+
+int run_scraper_cmdline()
+{
+	out << "EmulationStation scraper\n";
+	out << "========================\n";
+	out << "\n";
+
+	signal(SIGINT, handle_interrupt_signal);
+
+	//==================================================================================
+	//filter
+	//==================================================================================
+	enum FilterChoice
+	{
+		FILTER_MISSING_IMAGES,
+		FILTER_ALL
+	};
+
+	int filter_choice;
+	do {
+		out << "Select filter for games to be scraped:\n";
+		out << FILTER_MISSING_IMAGES << " - games missing images\n";
+		out << FILTER_ALL << " - all games period, can overwrite existing metadata\n";
+
+		std::cin >> filter_choice;
+		std::cin.ignore(1, '\n'); //skip the unconsumed newline
+	} while(filter_choice < FILTER_MISSING_IMAGES || filter_choice > FILTER_ALL);
+
+	out << "\n";
+
+	//==================================================================================
+	//platforms
+	//==================================================================================
+
+	std::vector<SystemData*> systems;
+
+	out << "You can scrape only specific platforms, or scrape all of them.\n";
+	out << "Would you like to scrape all platforms? (y/n)\n";
+
+	std::string system_choice;
+	std::getline(std::cin, system_choice);
+	
+	if(system_choice == "y" || system_choice == "Y")
+	{
+		out << "Will scrape all platforms.\n";
+		for(auto i = SystemData::sSystemVector.begin(); i != SystemData::sSystemVector.end(); i++)
+		{
+			out << "   " << (*i)->getName() << " (" << (*i)->getGameCount() << " games)\n";
+			systems.push_back(*i);
+		}
+
+	}else{
+		std::string sys_name;
+
+		out << "Enter the names of the platforms you would like to scrape, one at a time.\n";
+		out << "Type nothing and press enter when you are ready to continue.\n";
+
+		do {
+			for(auto i = SystemData::sSystemVector.begin(); i != SystemData::sSystemVector.end(); i++)
+			{
+				if(std::find(systems.begin(), systems.end(), (*i)) != systems.end())
+					out << " C ";
+				else
+					out << "   ";
+
+				out << "\"" << (*i)->getName() << "\" (" << (*i)->getGameCount() << " games)\n";
+			}
+
+			std::getline(std::cin, sys_name);
+			
+			if(sys_name.empty())
+				break;
+
+			bool found = false;
+			for(auto i = SystemData::sSystemVector.begin(); i != SystemData::sSystemVector.end(); i++)
+			{
+				if((*i)->getName() == sys_name)
+				{
+					systems.push_back(*i);
+					found = true;
+					break;
+				}
+			}
+
+			if(!found)
+				out << "System not found.\n";
+
+		} while(true);
+	}
+
+	//==================================================================================
+	//manual mode
+	//==================================================================================
+
+	out << "\n";
+	out << "You can let the scraper try to automatically choose the best result, or\n";
+	out << "you can manually approve each result. This \"manual mode\" is much more accurate.\n";
+	out << "It is highly recommended you use manual mode unless you have a very large collection.\n";
+	out << "Scrape in manual mode? (y/n)\n";
+
+	std::string manual_mode_str;
+	std::getline(std::cin, manual_mode_str);
+
+	bool manual_mode = false;
+
+	if(manual_mode_str == "y" || manual_mode_str == "Y")
+	{
+		manual_mode = true;
+		out << "Scraping in manual mode!\n";
+	}else{
+		out << "Scraping in automatic mode!\n";
+	}
+
+	//==================================================================================
+	//scraping
+	//==================================================================================
+	out << "\n";
+	out << "Alright, let's do this thing!\n";
+	out << "=============================\n";
+
+	Scraper* scraper = Settings::getInstance()->getScraper();
+	for(auto sysIt = systems.begin(); sysIt != systems.end(); sysIt++)
+	{
+		std::vector<FileData*> files = (*sysIt)->getRootFolder()->getFilesRecursive(true);
+
+		ScraperSearchParams params;
+		params.system = (*sysIt);
+
+		for(auto gameIt = files.begin(); gameIt != files.end(); gameIt++)
+		{
+			params.nameOverride = "";
+			params.game = (GameData*)(*gameIt);
+
+			//print original search term
+			out << params.game->getCleanName() << "...\n";
+
+			//need to take into account filter_choice
+			if(filter_choice == FILTER_MISSING_IMAGES)
+			{
+				if(!params.game->metadata()->get("image").empty()) //maybe should also check if the image file exists/is a URL
+				{
+					out << "   Skipping, metadata \"image\" entry is not empty.\n";
+					continue;
+				}
+			}
+
+			//actually get some results
+			do {
+				std::vector<MetaDataList> mdls = scraper->getResults(params);
+
+				//no results
+				if(mdls.size() == 0)
+				{
+					if(manual_mode)
+					{
+						//in manual mode let the user enter a custom search
+						out << "   NO RESULTS FOUND! Enter a new name to search for, or nothing to skip.\n";
+
+						std::getline(std::cin, params.nameOverride);
+
+						if(params.nameOverride.empty())
+						{
+							out << "   Skipping...\n";
+							break;
+						}
+
+						continue;
+
+					}else{
+						out << "   NO RESULTS FOUND! Skipping...\n";
+						break;
+					}
+				}
+
+				//some results
+				if(manual_mode)
+				{
+					//print list of choices
+					for(unsigned int i = 0; i < mdls.size(); i++)
+					{
+						out << "   " << i << " - " << mdls.at(i).get("name") << "\n";
+					}
+
+					int choice = -1;
+					std::string choice_str;
+					
+					out << "Your choice: ";
+
+					std::getline(std::cin, choice_str);
+					std::stringstream choice_buff(choice_str); //convert to int
+					choice_buff >> choice;
+
+					if(choice >= 0 && choice < (int)mdls.size())
+					{
+						*params.game->metadata() = mdls.at(choice);
+						break;
+					}else{
+						out << "Invalid choice.\n";
+						continue;
+					}
+
+				}else{
+					//automatic mode
+					//always choose the first choice
+					out << "   name -> " << mdls.at(0).get("name") << "\n";
+					*params.game->metadata() = mdls.at(0);
+					break;
+				}
+
+			} while(true);
+
+			out << "===================\n";
+		}
+	}
+
+	out << "\n\n";
+	out << "==============================\n";
+	out << "SCRAPE COMPLETE!\n";
+	out << "==============================\n";
+
+	return 0;
+}

src/ScraperCmdLine.h

@@ -0,0 +1,3 @@
+#pragma once
+
+int run_scraper_cmdline();

src/SystemData.cpp

@@ -349,3 +349,8 @@ PlatformIds::PlatformId SystemData::getPlatformId()
 {
 	return mPlatformId;
 }
+
+unsigned int SystemData::getGameCount()
+{
+	return mRootFolder->getFilesRecursive(true).size();
+}

src/SystemData.h

@@ -26,6 +26,7 @@ public:
 	PlatformIds::PlatformId getPlatformId();
 	bool hasGamelist();
 	std::vector<MetaDataDecl> getGameMDD();
+	unsigned int getGameCount();
 
 	void launchGame(Window* window, GameData* game);
 

src/main.cpp

@@ -15,6 +15,7 @@
 #include "Window.h"
 #include "EmulationStation.h"
 #include "Settings.h"
+#include "ScraperCmdLine.h"
 
 #ifdef _RPI_
 	#include <bcm_host.h>
@@ -24,6 +25,8 @@
 
 namespace fs = boost::filesystem;
 
+bool scrape_cmdline = false;
+
 bool parseArgs(int argc, char* argv[], unsigned int* width, unsigned int* height)
 {
 	if(argc > 1)
@@ -61,6 +64,9 @@ bool parseArgs(int argc, char* argv[], unsigned int* width, unsigned int* height
 			}else if(strcmp(argv[i], "--windowed") == 0)
 			{
 				Settings::getInstance()->setBool("WINDOWED", true);
+			}else if(strcmp(argv[i], "--scrape") == 0)
+			{
+				scrape_cmdline = true;
 			}else if(strcmp(argv[i], "--help") == 0)
 			{
 				std::cout << "EmulationStation, a graphical front-end for ROM browsing.\n";
@@ -73,6 +79,7 @@ bool parseArgs(int argc, char* argv[], unsigned int* width, unsigned int* height
 				std::cout << "--no-exit			don't show the exit option in the menu\n";
 				std::cout << "--debug				even more logging\n";
 				std::cout << "--dimtime [seconds]		time to wait before dimming the screen (default 30, use 0 for never)\n";
+				std::cout << "--scrape			scrape using command line interface\n";
 
 				#ifdef USE_OPENGL_DESKTOP
 					std::cout << "--windowed			not fullscreen\n";
@@ -132,16 +139,6 @@ int main(int argc, char* argv[])
 	//always close the log and deinit the BCM library on exit
 	atexit(&onExit);
 
-	Window window;
-	if(!window.init(width, height))
-	{
-		LOG(LogError) << "Window failed to initialize!";
-		return 1;
-	}
-
-	//dont generate joystick events while we're loading (hopefully fixes "automatically started emulator" bug)
-	SDL_JoystickEventState(SDL_DISABLE);
-
 	//try loading the system config file
 	if(!SystemData::loadConfig(SystemData::getConfigPath(), true))
 	{
@@ -156,6 +153,22 @@ int main(int argc, char* argv[])
 		return 1;
 	}
 
+	//run the command line scraper ui then quit
+	if(scrape_cmdline)
+	{
+		return run_scraper_cmdline();
+	}
+
+	Window window;
+	if(!window.init(width, height))
+	{
+		LOG(LogError) << "Window failed to initialize!";
+		return 1;
+	}
+
+	//dont generate joystick events while we're loading (hopefully fixes "automatically started emulator" bug)
+	SDL_JoystickEventState(SDL_DISABLE);
+
 	//choose which GUI to open depending on if an input configuration already exists
 	if(fs::exists(InputManager::getConfigPath()))
 	{