From 4e2b57c0013ba7426d891a7d11d2f1d7c9b82161 Mon Sep 17 00:00:00 2001 From: Aloshi Date: Wed, 9 Oct 2013 19:50:42 -0500 Subject: [PATCH] Image downloading is now async for GuiMetaDataEd. GamesDBScraper now uses system->getPlatformId() if set. --- src/HttpReq.cpp | 8 +++++ src/HttpReq.h | 1 + src/ScraperCmdLine.cpp | 34 ++++++++++++++++++++ src/components/GuiMetaDataEd.cpp | 35 ++++++++++++++++----- src/scrapers/GamesDBScraper.cpp | 54 +++++++++++++++++++++++++++++++- 5 files changed, 124 insertions(+), 8 deletions(-) diff --git a/src/HttpReq.cpp b/src/HttpReq.cpp index 77f0a146d..7d1316913 100644 --- a/src/HttpReq.cpp +++ b/src/HttpReq.cpp @@ -2,6 +2,7 @@ #include "HttpReq.h" #include #include "Log.h" +#include boost::asio::io_service HttpReq::io_service; @@ -27,6 +28,13 @@ std::string HttpReq::urlEncode(const std::string &s) return escaped; } +bool HttpReq::isUrl(const std::string& str) +{ + //the worst guess + return (!str.empty() && !boost::filesystem::exists(str) && + (str.find("http://") != std::string::npos || str.find("https://") != std::string::npos || str.find("www.") != std::string::npos)); +} + HttpReq::HttpReq(const std::string& server, const std::string& path) : mResolver(io_service), mSocket(io_service), mStatus(REQ_IN_PROGRESS) { diff --git a/src/HttpReq.h b/src/HttpReq.h index 14181ace4..1fcc9976e 100644 --- a/src/HttpReq.h +++ b/src/HttpReq.h @@ -48,6 +48,7 @@ public: std::string getContent(); static std::string urlEncode(const std::string &s); + static bool isUrl(const std::string& s); private: static boost::asio::io_service io_service; diff --git a/src/ScraperCmdLine.cpp b/src/ScraperCmdLine.cpp index 31f79c438..1daac6b23 100644 --- a/src/ScraperCmdLine.cpp +++ b/src/ScraperCmdLine.cpp @@ -233,6 +233,40 @@ int run_scraper_cmdline() } } + out << "\n\n"; + out << "Downloading boxart...\n"; + + for(auto sysIt = systems.begin(); sysIt != systems.end(); sysIt++) + { + std::vector files = (*sysIt)->getRootFolder()->getFilesRecursive(true); + + for(auto gameIt = files.begin(); gameIt != files.end(); gameIt++) + { + GameData* game = (GameData*)(*gameIt); + std::vector mdd = (*sysIt)->getGameMDD(); + for(auto i = mdd.begin(); i != mdd.end(); i++) + { + std::string key = i->key; + std::string url = game->metadata()->get(key); + + if(i->type == MD_IMAGE_PATH && HttpReq::isUrl(url)) + { + std::string urlShort = url.substr(0, url.length() > 35 ? 35 : url.length()); + if(url.length() != urlShort.length()) urlShort += "..."; + + out << " " << game->metadata()->get("name") << " [from: " << urlShort << "]...\n"; + game->metadata()->set(key, downloadImage(url, getSaveAsPath((*sysIt)->getName(), game->getCleanName(), url))); + if(game->metadata()->get(key).empty()) + { + out << " FAILED! Skipping.\n"; + game->metadata()->set(key, url); //result URL to what it was if download failed, retry some other time + } + } + } + } + } + + out << "\n\n"; out << "==============================\n"; out << "SCRAPE COMPLETE!\n"; diff --git a/src/components/GuiMetaDataEd.cpp b/src/components/GuiMetaDataEd.cpp index ffab7c9ff..13831c943 100644 --- a/src/components/GuiMetaDataEd.cpp +++ b/src/components/GuiMetaDataEd.cpp @@ -4,6 +4,7 @@ #include "AsyncReqComponent.h" #include "../Settings.h" #include "GuiGameScraper.h" +#include #define MDED_RESERVED_ROWS 3 @@ -125,19 +126,39 @@ void GuiMetaDataEd::fetch() void GuiMetaDataEd::fetchDone(MetaDataList result) { + //this is a little tricky: + //go through the list of returned results, if anything is an image and the path looks like a URL: + // (1) start an async download + resize (will create an AsyncReq that blocks further user input) + // (when this is finished, call result.set(key, newly_downloaded_file_path) and call fetchDone() again) + // (2) return from this function immediately + for(auto it = mMetaDataDecl.begin(); it != mMetaDataDecl.end(); it++) + { + std::string key = it->key; + std::string val = result.get(it->key); + + //val is /probably/ a URL + if(it->type == MD_IMAGE_PATH && HttpReq::isUrl(val)) + { + downloadImageAsync(mWindow, val, getSaveAsPath(mScraperParams.system->getName(), mScraperParams.game->getCleanName() + "-" + key, val), + [this, result, key] (std::string filePath) mutable -> void { + //skip it + if(filePath.empty()) + LOG(LogError) << "Error resolving boxart"; + + result.set(key, filePath); + this->fetchDone(result); + }); + return; + } + } + for(unsigned int i = 0; i < mEditors.size(); i++) { //don't overwrite statistics if(mMetaDataDecl.at(i).isStatistic) continue; - const std::string key = mMetaDataDecl.at(i).key; - if(mMetaDataDecl.at(i).type == MD_IMAGE_PATH) - { - std::string url = result.get(key); - result.set(key, downloadImage(url, getSaveAsPath(mScraperParams.system->getName(), mScraperParams.game->getCleanName() + "-" + key, url))); - } - + const std::string& key = mMetaDataDecl.at(i).key; mEditors.at(i)->setValue(result.get(key)); } } diff --git a/src/scrapers/GamesDBScraper.cpp b/src/scrapers/GamesDBScraper.cpp index 1d8d7e39d..86f2a675f 100644 --- a/src/scrapers/GamesDBScraper.cpp +++ b/src/scrapers/GamesDBScraper.cpp @@ -4,9 +4,56 @@ #include "../Log.h" #include "../pugiXML/pugixml.hpp" #include "../MetaData.h" +#include const char* GamesDBScraper::getName() { return "TheGamesDB"; } +using namespace PlatformIds; +const std::map gamesdb_platformid_map = boost::assign::map_list_of + (THREEDO, "3DO") + (AMIGA, "Amiga") + (ARCADE, "Arcade") + (ATARI_2600, "Atari 2600") + (ATARI_5200, "Atari 5200") + (ATARI_7800, "Atari 7800") + (ATARI_JAGUAR, "Atari Jaguar") + (ATARI_JAGUAR_CD, "Atari Jaguar CD") + (ATARI_XE, "Atari XE") + (COLECOVISION, "Colecovision") + (COMMODORE_64, "Commodore 64") + (INTELLIVISION, "Intellivision") + (MAC_OS, "Mac OS") + (XBOX, "Microsoft Xbox") + (XBOX_360, "Microsoft Xbox 360") + (NEOGEO, "NeoGeo") + (NINTENDO_3DS, "Nintendo 3DS") + (NINTENDO_64, "Nintendo 64") + (NINTENDO_DS, "Nintendo DS") + (NINTENDO_ENTERTAINMENT_SYSTEM, "Nintendo Entertainment System (NES)") + (GAME_BOY, "Nintendo Game Boy") + (GAME_BOY_ADVANCE, "Nintendo Game Boy Advance") + (GAME_BOY_COLOR, "Nintendo Game Boy Color") + (NINTENDO_GAMECUBE, "Nintendo GameCube") + (NINTENDO_WII, "Nintendo Wii") + (NINTENDO_WII_U, "Nintendo Wii U") + (PC, "PC") + (SEGA_32X, "Sega 32X") + (SEGA_CD, "Sega CD") + (SEGA_DREAMCAST, "Sega Dreamcast") + (SEGA_GAME_GEAR, "Sega Game Gear") + (SEGA_GENESIS, "Sega Genesis") + (SEGA_MASTER_SYSTEM, "Sega Master System") + (SEGA_MEGA_DRIVE, "Sega Mega Drive") + (SEGA_SATURN, "Sega Saturn") + (PLAYSTATION, "Sony Playstation") + (PLAYSTATION_2, "Sony Playstation 2") + (PLAYSTATION_3, "Sony Playstation 3") + (PLAYSTATION_VITA, "Sony Playstation Vita") + (PLAYSTATION_PORTABLE, "Sony PSP") + (SUPER_NINTENDO, "Super Nintendo (SNES)") + (TURBOGRAFX_16, "TurboGrafx 16"); + + std::shared_ptr GamesDBScraper::makeHttpReq(ScraperSearchParams params) { std::string path = "/api/GetGame.php?"; @@ -16,7 +63,12 @@ std::shared_ptr GamesDBScraper::makeHttpReq(ScraperSearchParams params) cleanName = params.game->getCleanName(); path += "name=" + HttpReq::urlEncode(cleanName); - //platform TODO, should use some params.system get method + + if(params.system->getPlatformId() != PLATFORM_UNKNOWN) + { + path += "&platform="; + path += HttpReq::urlEncode(gamesdb_platformid_map.at(params.system->getPlatformId())); + } return std::make_shared("thegamesdb.net", path); }