From 57eb9849ffe49097e01077ec10141a9a928164a6 Mon Sep 17 00:00:00 2001 From: Juan Pablo Date: Tue, 24 Sep 2013 01:58:59 -0300 Subject: [PATCH 1/4] Renamed/moved maximum scraper results constant to be used in scraper --- src/components/GuiGameScraper.cpp | 8 +++----- src/components/GuiGameScraper.h | 2 ++ src/scrapers/GamesDBScraper.cpp | 3 ++- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/components/GuiGameScraper.cpp b/src/components/GuiGameScraper.cpp index d8111f7a4..e15eab576 100644 --- a/src/components/GuiGameScraper.cpp +++ b/src/components/GuiGameScraper.cpp @@ -4,10 +4,8 @@ #include "../scrapers/Scraper.h" #include "../Settings.h" -#define RESULT_COUNT 5 - GuiGameScraper::GuiGameScraper(Window* window, ScraperSearchParams params, std::function doneFunc, std::function skipFunc) : GuiComponent(window), - mList(window, Eigen::Vector2i(2, 7 + RESULT_COUNT)), + mList(window, Eigen::Vector2i(2, 7 + MAX_SCRAPER_RESULTS)), mBox(window, ":/frame.png"), mHeader(window, params.game->getBaseName(), Font::get(*window->getResourceManager(), Font::getDefaultPath(), FONT_SIZE_MEDIUM)), mResultName(window, "", Font::get(*window->getResourceManager(), Font::getDefaultPath(), FONT_SIZE_MEDIUM)), @@ -78,8 +76,8 @@ GuiGameScraper::GuiGameScraper(Window* window, ScraperSearchParams params, std:: //y = 5 is a spacer row std::shared_ptr font = Font::get(*window->getResourceManager(), Font::getDefaultPath(), FONT_SIZE_SMALL); - mResultNames.reserve(RESULT_COUNT); - for(int i = 0; i < RESULT_COUNT; i ++) + mResultNames.reserve(MAX_SCRAPER_RESULTS); + for(int i = 0; i < MAX_SCRAPER_RESULTS; i ++) { mResultNames.push_back(TextComponent(mWindow, "RESULT...", font)); mResultNames.at(i).setColor(0x111111FF); diff --git a/src/components/GuiGameScraper.h b/src/components/GuiGameScraper.h index db8d0191f..1a268a114 100644 --- a/src/components/GuiGameScraper.h +++ b/src/components/GuiGameScraper.h @@ -11,6 +11,8 @@ #include "../HttpReq.h" #include "ImageComponent.h" +#define MAX_SCRAPER_RESULTS 5 + class GuiGameScraper : public GuiComponent { public: diff --git a/src/scrapers/GamesDBScraper.cpp b/src/scrapers/GamesDBScraper.cpp index d44b4d390..a0cb66915 100644 --- a/src/scrapers/GamesDBScraper.cpp +++ b/src/scrapers/GamesDBScraper.cpp @@ -1,4 +1,5 @@ #include "GamesDBScraper.h" +#include "../components/GuiGameScraper.h" #include "../components/AsyncReqComponent.h" #include "../Log.h" #include "../pugiXML/pugixml.hpp" @@ -49,7 +50,7 @@ std::vector GamesDBScraper::parseReq(ScraperSearchParams params, s unsigned int resultNum = 0; pugi::xml_node game = data.child("Game"); - while(game && resultNum < 5) + while(game && resultNum < MAX_SCRAPER_RESULTS) { mdl.push_back(MetaDataList(params.system->getGameMDD())); mdl.back().set("name", game.child("GameTitle").text().get()); From 3e1ecb4a844154245471794e716e2dfdad864b3a Mon Sep 17 00:00:00 2001 From: Juan Pablo Date: Tue, 24 Sep 2013 02:20:53 -0300 Subject: [PATCH 2/4] Added a method to clean filenames before scraping --- src/GameData.cpp | 6 ++++++ src/GameData.h | 1 + src/components/GuiGameScraper.cpp | 2 +- src/scrapers/GamesDBScraper.cpp | 2 +- 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/GameData.cpp b/src/GameData.cpp index 976d3e058..d6f11ca3e 100644 --- a/src/GameData.cpp +++ b/src/GameData.cpp @@ -1,5 +1,6 @@ #include "GameData.h" #include +#include #include #include #include @@ -57,6 +58,11 @@ std::string GameData::getBaseName() const return mBaseName; } +std::string GameData::getCleanName() const +{ + return regex_replace(mBaseName, boost::regex("\\((.*)\\)|\\[(.*)\\]"), ""); +} + void GameData::incTimesPlayed() { int timesPlayed = metadata()->getInt("playcount"); diff --git a/src/GameData.h b/src/GameData.h index e67ec2e1d..6f40c798b 100644 --- a/src/GameData.h +++ b/src/GameData.h @@ -21,6 +21,7 @@ public: std::string getBashPath() const; std::string getBaseName() const; + std::string getCleanName() const; bool isFolder() const override; diff --git a/src/components/GuiGameScraper.cpp b/src/components/GuiGameScraper.cpp index e15eab576..701cafb95 100644 --- a/src/components/GuiGameScraper.cpp +++ b/src/components/GuiGameScraper.cpp @@ -69,7 +69,7 @@ GuiGameScraper::GuiGameScraper(Window* window, ScraperSearchParams params, std:: //y = 3 is a spacer row mList.setEntry(Vector2i(0, 4), Vector2i(1, 1), &mSearchLabel, false, ComponentListComponent::AlignLeft); - mSearchText.setValue(!params.nameOverride.empty() ? params.nameOverride : params.game->getBaseName()); + mSearchText.setValue(!params.nameOverride.empty() ? params.nameOverride : params.game->getCleanName()); mSearchText.setSize(colWidth * 2 - mSearchLabel.getSize().x() - 20, mSearchText.getSize().y()); mList.setEntry(Vector2i(1, 4), Vector2i(1, 1), &mSearchText, true, ComponentListComponent::AlignRight); diff --git a/src/scrapers/GamesDBScraper.cpp b/src/scrapers/GamesDBScraper.cpp index a0cb66915..fc018493c 100644 --- a/src/scrapers/GamesDBScraper.cpp +++ b/src/scrapers/GamesDBScraper.cpp @@ -18,7 +18,7 @@ std::shared_ptr GamesDBScraper::makeHttpReq(ScraperSearchParams params) std::string cleanName = params.nameOverride; if(cleanName.empty()) - cleanName = params.game->getBaseName(); + cleanName = params.game->getCleanName(); path += "name=" + cleanName; //platform TODO, should use some params.system get method From 2999a8068a882b32374d74cf725510fe12c23a41 Mon Sep 17 00:00:00 2001 From: Juan Pablo Date: Tue, 24 Sep 2013 04:02:14 -0300 Subject: [PATCH 3/4] Properly encoded parameters. Otherwise the query gets truncated. --- src/HttpReq.cpp | 22 ++++++++++++++++++++++ src/HttpReq.h | 2 ++ src/scrapers/GamesDBScraper.cpp | 2 +- 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/HttpReq.cpp b/src/HttpReq.cpp index 106192109..4cd4a46ec 100644 --- a/src/HttpReq.cpp +++ b/src/HttpReq.cpp @@ -5,6 +5,28 @@ boost::asio::io_service HttpReq::io_service; +std::string HttpReq::urlEncode(const std::string &s) +{ + const std::string unreserved = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_.~"; + + std::string escaped=""; + for(size_t i=0; i GamesDBScraper::makeHttpReq(ScraperSearchParams params) if(cleanName.empty()) cleanName = params.game->getCleanName(); - path += "name=" + cleanName; + path += "name=" + HttpReq::urlEncode(cleanName); //platform TODO, should use some params.system get method return std::make_shared("thegamesdb.net", path); From a8427d33a6461f855bbf1051eb2644d97ad0895d Mon Sep 17 00:00:00 2001 From: Juan Pablo Date: Tue, 24 Sep 2013 04:26:33 -0300 Subject: [PATCH 4/4] Partial implementation for The Archive scraper. --- CMakeLists.txt | 2 + src/Settings.cpp | 1 + src/scrapers/TheArchiveScraper.cpp | 84 ++++++++++++++++++++++++++++++ src/scrapers/TheArchiveScraper.h | 16 ++++++ 4 files changed, 103 insertions(+) create mode 100644 src/scrapers/TheArchiveScraper.cpp create mode 100644 src/scrapers/TheArchiveScraper.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 9661f97f9..3d74b2386 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -179,6 +179,7 @@ set(ES_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/src/components/GuiSettingsMenu.h ${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/Scraper.h ${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/GamesDBScraper.h + ${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/TheArchiveScraper.h ${CMAKE_CURRENT_SOURCE_DIR}/src/pugiXML/pugiconfig.hpp ${CMAKE_CURRENT_SOURCE_DIR}/src/pugiXML/pugixml.hpp ${CMAKE_CURRENT_SOURCE_DIR}/src/resources/ResourceManager.h @@ -230,6 +231,7 @@ set(ES_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/src/components/GuiMenu.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/components/GuiSettingsMenu.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/GamesDBScraper.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/scrapers/TheArchiveScraper.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/pugiXML/pugixml.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/resources/ResourceManager.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/resources/TextureResource.cpp diff --git a/src/Settings.cpp b/src/Settings.cpp index 211679d12..75f0277af 100644 --- a/src/Settings.cpp +++ b/src/Settings.cpp @@ -4,6 +4,7 @@ #include "platform.h" #include #include "scrapers/GamesDBScraper.h" +#include "scrapers/TheArchiveScraper.h" Settings* Settings::sInstance = NULL; diff --git a/src/scrapers/TheArchiveScraper.cpp b/src/scrapers/TheArchiveScraper.cpp new file mode 100644 index 000000000..fadbfeba3 --- /dev/null +++ b/src/scrapers/TheArchiveScraper.cpp @@ -0,0 +1,84 @@ +#include "TheArchiveScraper.h" +#include "../components/GuiGameScraper.h" +#include "../components/AsyncReqComponent.h" +#include "../Log.h" +#include "../pugiXML/pugixml.hpp" + +std::vector TheArchiveScraper::getResults(ScraperSearchParams params) +{ + std::shared_ptr req = makeHttpReq(params); + while(req->status() == HttpReq::REQ_IN_PROGRESS); + + return parseReq(params, req); +} + +std::shared_ptr TheArchiveScraper::makeHttpReq(ScraperSearchParams params) +{ + std::string path = "/2.0/Archive.search/xml/7TTRM4MNTIKR2NNAGASURHJOZJ3QXQC5/"; + + std::string cleanName = params.nameOverride; + if(cleanName.empty()) + cleanName = params.game->getCleanName(); + + path += HttpReq::urlEncode(cleanName); + //platform TODO, should use some params.system get method + + return std::make_shared("api.archive.vg", path); +} + +std::vector TheArchiveScraper::parseReq(ScraperSearchParams params, std::shared_ptr req) +{ + std::vector mdl; + + if(req->status() != HttpReq::REQ_SUCCESS) + { + LOG(LogError) << "HttpReq error"; + return mdl; + } + + pugi::xml_document doc; + pugi::xml_parse_result parseResult = doc.load(req->getContent().c_str()); + if(!parseResult) + { + LOG(LogError) << "Error parsing XML"; + return mdl; + } + + pugi::xml_node data = doc.child("OpenSearchDescription").child("games"); + + unsigned int resultNum = 0; + pugi::xml_node game = data.child("game"); + while(game && resultNum < MAX_SCRAPER_RESULTS) + { + mdl.push_back(MetaDataList(params.system->getGameMDD())); + mdl.back().set("name", game.child("title").text().get()); + mdl.back().set("desc", game.child("description").text().get()); + + pugi::xml_node image = game.child("box_front"); + pugi::xml_node thumbnail = game.child("box_front_small"); + + if (image) + mdl.back().set("image",image.text().get()); + if (thumbnail) + mdl.back().set("thumbnail", thumbnail.text().get()); + + resultNum++; + game = game.next_sibling("game"); + } + + return mdl; +} + +void TheArchiveScraper::getResultsAsync(ScraperSearchParams params, Window* window, std::function)> returnFunc) +{ + std::shared_ptr httpreq = makeHttpReq(params); + AsyncReqComponent* req = new AsyncReqComponent(window, httpreq, + [this, params, returnFunc] (std::shared_ptr r) + { + returnFunc(parseReq(params, r)); + }, [] () + { + }); + + window->pushGui(req); +} diff --git a/src/scrapers/TheArchiveScraper.h b/src/scrapers/TheArchiveScraper.h new file mode 100644 index 000000000..e88c23e98 --- /dev/null +++ b/src/scrapers/TheArchiveScraper.h @@ -0,0 +1,16 @@ +#pragma once + +#include "Scraper.h" +#include "../HttpReq.h" + +class TheArchiveScraper : public IScraper +{ +public: + std::vector getResults(ScraperSearchParams params) override; + void getResultsAsync(ScraperSearchParams params, Window* window, std::function)> returnFunc) override; + +private: + std::shared_ptr makeHttpReq(ScraperSearchParams params); + std::vector parseReq(ScraperSearchParams params, std::shared_ptr); +}; +