Merge pull request #133 from elpendor/unstable

Pendor's scraper work.

CMakeLists.txt

@@ -179,6 +179,7 @@ set(ES_HEADERS
     ${CMAKE_CURRENT_SOURCE_DIR}/src/components/GuiSettingsMenu.h
     ${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/Scraper.h
     ${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/GamesDBScraper.h
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/TheArchiveScraper.h
     ${CMAKE_CURRENT_SOURCE_DIR}/src/pugiXML/pugiconfig.hpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/pugiXML/pugixml.hpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/resources/ResourceManager.h
@@ -230,6 +231,7 @@ set(ES_SOURCES
     ${CMAKE_CURRENT_SOURCE_DIR}/src/components/GuiMenu.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/components/GuiSettingsMenu.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/GamesDBScraper.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/TheArchiveScraper.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/pugiXML/pugixml.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/resources/ResourceManager.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/resources/TextureResource.cpp

src/GameData.cpp

@@ -1,5 +1,6 @@
 #include "GameData.h"
 #include <boost/filesystem.hpp>
+#include <boost/regex/v4/regex.hpp>
 #include <iostream>
 #include <ctime>
 #include <sstream>
@@ -57,6 +58,11 @@ std::string GameData::getBaseName() const
 	return mBaseName;
 }
 
+std::string GameData::getCleanName() const
+{
+	return regex_replace(mBaseName, boost::regex("\\((.*)\\)|\\[(.*)\\]"), "");
+}
+
 void GameData::incTimesPlayed()
 {
 	int timesPlayed = metadata()->getInt("playcount");

src/GameData.h

@@ -21,6 +21,7 @@ public:
 
 	std::string getBashPath() const;
 	std::string getBaseName() const;
+	std::string getCleanName() const;
 
 	bool isFolder() const override;
 

src/HttpReq.cpp

@@ -5,6 +5,28 @@
 
 boost::asio::io_service HttpReq::io_service;
 
+std::string HttpReq::urlEncode(const std::string &s)
+{
+    const std::string unreserved = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_.~";
+
+    std::string escaped="";
+    for(size_t i=0; i<s.length(); i++)
+    {
+        if (unreserved.find_first_of(s[i]) != std::string::npos)
+        {
+            escaped.push_back(s[i]);
+        }
+        else
+        {
+            escaped.append("%");
+            char buf[3];
+            sprintf(buf, "%.2X", s[i]);
+            escaped.append(buf);
+        }
+    }
+    return escaped;
+}
+
 HttpReq::HttpReq(const std::string& server, const std::string& path)
 	: mResolver(io_service), mSocket(io_service), mStatus(REQ_IN_PROGRESS)
 {

src/HttpReq.h

@@ -47,6 +47,8 @@ public:
 
 	std::string getContent();
 
+	static std::string urlEncode(const std::string &s);
+
 private:
 	static boost::asio::io_service io_service;
 

src/Settings.cpp

@@ -4,6 +4,7 @@
 #include "platform.h"
 #include <boost/filesystem.hpp>
 #include "scrapers/GamesDBScraper.h"
+#include "scrapers/TheArchiveScraper.h"
 
 Settings* Settings::sInstance = NULL;
 

src/components/GuiGameScraper.cpp

@@ -4,10 +4,8 @@
 #include "../scrapers/Scraper.h"
 #include "../Settings.h"
 
-#define RESULT_COUNT 5
-
 GuiGameScraper::GuiGameScraper(Window* window, ScraperSearchParams params, std::function<void(MetaDataList)> doneFunc, std::function<void()> skipFunc) : GuiComponent(window), 
-	mList(window, Eigen::Vector2i(2, 7 + RESULT_COUNT)), 
+	mList(window, Eigen::Vector2i(2, 7 + MAX_SCRAPER_RESULTS)), 
 	mBox(window, ":/frame.png"),
 	mHeader(window, params.game->getBaseName(), Font::get(*window->getResourceManager(), Font::getDefaultPath(), FONT_SIZE_MEDIUM)),
 	mResultName(window, "", Font::get(*window->getResourceManager(), Font::getDefaultPath(), FONT_SIZE_MEDIUM)),
@@ -71,15 +69,15 @@ GuiGameScraper::GuiGameScraper(Window* window, ScraperSearchParams params, std::
 	//y = 3 is a spacer row
 
 	mList.setEntry(Vector2i(0, 4), Vector2i(1, 1), &mSearchLabel, false, ComponentListComponent::AlignLeft);
-	mSearchText.setValue(!params.nameOverride.empty() ? params.nameOverride : params.game->getBaseName());
+	mSearchText.setValue(!params.nameOverride.empty() ? params.nameOverride : params.game->getCleanName());
 	mSearchText.setSize(colWidth * 2 - mSearchLabel.getSize().x() - 20, mSearchText.getSize().y());
 	mList.setEntry(Vector2i(1, 4), Vector2i(1, 1), &mSearchText, true, ComponentListComponent::AlignRight);
 
 	//y = 5 is a spacer row
 
 	std::shared_ptr<Font> font = Font::get(*window->getResourceManager(), Font::getDefaultPath(), FONT_SIZE_SMALL);
-	mResultNames.reserve(RESULT_COUNT);
+	mResultNames.reserve(MAX_SCRAPER_RESULTS);
-	for(int i = 0; i < RESULT_COUNT; i ++)
+	for(int i = 0; i < MAX_SCRAPER_RESULTS; i ++)
 	{
 		mResultNames.push_back(TextComponent(mWindow, "RESULT...", font));
 		mResultNames.at(i).setColor(0x111111FF);

src/components/GuiGameScraper.h

@@ -11,6 +11,8 @@
 #include "../HttpReq.h"
 #include "ImageComponent.h"
 
+#define MAX_SCRAPER_RESULTS 5
+
 class GuiGameScraper : public GuiComponent
 {
 public:

src/scrapers/GamesDBScraper.cpp

@@ -1,4 +1,5 @@
 #include "GamesDBScraper.h"
+#include "../components/GuiGameScraper.h"
 #include "../components/AsyncReqComponent.h"
 #include "../Log.h"
 #include "../pugiXML/pugixml.hpp"
@@ -17,9 +18,9 @@ std::shared_ptr<HttpReq> GamesDBScraper::makeHttpReq(ScraperSearchParams params)
 
 	std::string cleanName = params.nameOverride;
 	if(cleanName.empty())
-		cleanName = params.game->getBaseName();
+		cleanName = params.game->getCleanName();
 	
-	path += "name=" + cleanName;
+	path += "name=" + HttpReq::urlEncode(cleanName);
 	//platform TODO, should use some params.system get method
 
 	return std::make_shared<HttpReq>("thegamesdb.net", path);
@@ -49,7 +50,7 @@ std::vector<MetaDataList> GamesDBScraper::parseReq(ScraperSearchParams params, s
 	
 	unsigned int resultNum = 0;
 	pugi::xml_node game = data.child("Game");
-	while(game && resultNum < 5)
+	while(game && resultNum < MAX_SCRAPER_RESULTS)
 	{
 		mdl.push_back(MetaDataList(params.system->getGameMDD()));
 		mdl.back().set("name", game.child("GameTitle").text().get());

src/scrapers/TheArchiveScraper.cpp

@@ -0,0 +1,84 @@
+#include "TheArchiveScraper.h"
+#include "../components/GuiGameScraper.h"
+#include "../components/AsyncReqComponent.h"
+#include "../Log.h"
+#include "../pugiXML/pugixml.hpp"
+
+std::vector<MetaDataList> TheArchiveScraper::getResults(ScraperSearchParams params)
+{
+	std::shared_ptr<HttpReq> req = makeHttpReq(params);
+	while(req->status() == HttpReq::REQ_IN_PROGRESS);
+
+	return parseReq(params, req);
+}
+
+std::shared_ptr<HttpReq> TheArchiveScraper::makeHttpReq(ScraperSearchParams params)
+{
+	std::string path = "/2.0/Archive.search/xml/7TTRM4MNTIKR2NNAGASURHJOZJ3QXQC5/";
+
+	std::string cleanName = params.nameOverride;
+	if(cleanName.empty())
+		cleanName = params.game->getCleanName();
+	
+	path += HttpReq::urlEncode(cleanName);
+	//platform TODO, should use some params.system get method
+
+	return std::make_shared<HttpReq>("api.archive.vg", path);
+}
+
+std::vector<MetaDataList> TheArchiveScraper::parseReq(ScraperSearchParams params, std::shared_ptr<HttpReq> req)
+{
+	std::vector<MetaDataList> mdl;
+
+	if(req->status() != HttpReq::REQ_SUCCESS)
+	{
+		LOG(LogError) << "HttpReq error";
+		return mdl;
+	}
+
+	pugi::xml_document doc;
+	pugi::xml_parse_result parseResult = doc.load(req->getContent().c_str());
+	if(!parseResult)
+	{
+		LOG(LogError) << "Error parsing XML";
+		return mdl;
+	}
+
+	pugi::xml_node data = doc.child("OpenSearchDescription").child("games");
+
+	unsigned int resultNum = 0;
+	pugi::xml_node game = data.child("game");
+	while(game && resultNum < MAX_SCRAPER_RESULTS)
+	{
+		mdl.push_back(MetaDataList(params.system->getGameMDD()));
+		mdl.back().set("name", game.child("title").text().get());
+		mdl.back().set("desc", game.child("description").text().get());
+		
+		pugi::xml_node image = game.child("box_front");
+		pugi::xml_node thumbnail = game.child("box_front_small");
+
+		if (image)
+			mdl.back().set("image",image.text().get());
+		if (thumbnail)
+			mdl.back().set("thumbnail", thumbnail.text().get());
+
+		resultNum++;
+		game = game.next_sibling("game");
+	}
+
+	return mdl;
+}
+
+void TheArchiveScraper::getResultsAsync(ScraperSearchParams params, Window* window, std::function<void(std::vector<MetaDataList>)> returnFunc)
+{
+	std::shared_ptr<HttpReq> httpreq = makeHttpReq(params);
+	AsyncReqComponent* req = new AsyncReqComponent(window, httpreq,
+		[this, params, returnFunc] (std::shared_ptr<HttpReq> r)
+	{
+		returnFunc(parseReq(params, r));
+	}, [] ()
+	{
+	});
+
+	window->pushGui(req);
+}

src/scrapers/TheArchiveScraper.h

@@ -0,0 +1,16 @@
+#pragma once
+
+#include "Scraper.h"
+#include "../HttpReq.h"
+
+class TheArchiveScraper : public IScraper
+{
+public:
+	std::vector<MetaDataList> getResults(ScraperSearchParams params) override;
+	void getResultsAsync(ScraperSearchParams params, Window* window, std::function<void(std::vector<MetaDataList>)> returnFunc) override;
+
+private:
+	std::shared_ptr<HttpReq> makeHttpReq(ScraperSearchParams params);
+	std::vector<MetaDataList> parseReq(ScraperSearchParams params, std::shared_ptr<HttpReq>);
+};
+