From c3de18dd96b249c707d7a6ded3c24c02b462b412 Mon Sep 17 00:00:00 2001 From: Leon Styhre Date: Wed, 5 Aug 2020 22:38:44 +0200 Subject: [PATCH] Added support for scraping videos using ScreenScraper. --- NEWS.md | 2 +- es-app/src/guis/GuiMetaDataEd.cpp | 2 +- es-app/src/guis/GuiScraperMenu.cpp | 16 +++++ es-app/src/guis/GuiScraperSearch.cpp | 1 + es-app/src/scrapers/Scraper.cpp | 85 ++++++++++++++++++--------- es-app/src/scrapers/Scraper.h | 41 ++++++++----- es-app/src/scrapers/ScreenScraper.cpp | 32 ++++++---- es-app/src/scrapers/ScreenScraper.h | 8 ++- es-core/src/Settings.cpp | 3 +- 9 files changed, 130 insertions(+), 60 deletions(-) diff --git a/NEWS.md b/NEWS.md index b401dbc44..3fa10b0d7 100644 --- a/NEWS.md +++ b/NEWS.md @@ -5,7 +5,7 @@ EmulationStation Desktop Edition v1.0.0 ### Release overview -First release, a major update to the application compared to the RetroPie version on which it is based. This includes new gamelist sorting logic, new game media handling and a completely updated Windows port (which now works about as well as the Unix version). The menu system has also been completely overhauled and the scraper has been expanded to support multiple media types as well as providing detailed scraping configuration options. +First release, a major update to the application compared to the RetroPie version on which it is based. This includes new gamelist sorting logic, new game media handling and a completely updated Windows port (which now works about as well as the Unix version). The menu system has also been completely overhauled and the scraper has been expanded to support multiple media types (including videos) as well as providing detailed scraping configuration options. Full navigation sound support has been implemented, and the metadata editor has seen a lot of updates including color coding of all changes done by the user and by the scraper. Favorite games can now also be sorted on top of the gamelists and game collections. diff --git a/es-app/src/guis/GuiMetaDataEd.cpp b/es-app/src/guis/GuiMetaDataEd.cpp index ec57205f0..d961f665c 100644 --- a/es-app/src/guis/GuiMetaDataEd.cpp +++ b/es-app/src/guis/GuiMetaDataEd.cpp @@ -392,7 +392,7 @@ void GuiMetaDataEd::fetchDone(const ScraperSearchResult& result) MetaDataList* metadata = nullptr; metadata = new MetaDataList(*mMetaData); - mMediaFilesUpdated = result.savedNewImages; + mMediaFilesUpdated = result.savedNewMedia; // Check if any values were manually changed before starting the scraping. // If so, it's these values we should compare against when scraping, not diff --git a/es-app/src/guis/GuiScraperMenu.cpp b/es-app/src/guis/GuiScraperMenu.cpp index 223caa1e3..6ae2fdf45 100644 --- a/es-app/src/guis/GuiScraperMenu.cpp +++ b/es-app/src/guis/GuiScraperMenu.cpp @@ -135,6 +135,22 @@ void GuiScraperMenu::openContentSettings() s->addSaveFunc([scrape_metadata] { Settings::getInstance()->setBool("ScrapeMetadata", scrape_metadata->getState()); }); + // Scrape videos. + auto scrape_videos = std::make_shared(mWindow); + scrape_videos->setState(Settings::getInstance()->getBool("ScrapeVideos")); + s->addWithLabel("SCRAPE VIDEOS", scrape_videos); + s->addSaveFunc([scrape_videos] { Settings::getInstance()->setBool("ScrapeVideos", + scrape_videos->getState()); }); + + // Videos are not supported by TheGamesDB, so disable the option if this scraper is selected. + if (Settings::getInstance()->getString("Scraper") == "thegamesdb") { + scrape_videos->setDisabled(); + scrape_videos->setOpacity(DISABLED_OPACITY); + // I'm sure there is a better way to find the text component... + scrape_videos->getParent()->getChild( + scrape_videos->getParent()->getChildCount()-2)->setOpacity(DISABLED_OPACITY); + } + // Scrape screenshots images. auto scrape_screenshots = std::make_shared(mWindow); scrape_screenshots->setState(Settings::getInstance()->getBool("ScrapeScreenshots")); diff --git a/es-app/src/guis/GuiScraperSearch.cpp b/es-app/src/guis/GuiScraperSearch.cpp index fb0d873a7..13a6517b2 100644 --- a/es-app/src/guis/GuiScraperSearch.cpp +++ b/es-app/src/guis/GuiScraperSearch.cpp @@ -540,6 +540,7 @@ void GuiScraperSearch::update(int deltaTime) results_scrape[i].coverUrl = it->coverUrl; results_scrape[i].marqueeUrl = it->marqueeUrl; results_scrape[i].screenshotUrl = it->screenshotUrl; + results_scrape[i].videoUrl = it->videoUrl; results_scrape[i].scraperRequestAllowance = it->scraperRequestAllowance; results_scrape[i].mediaURLFetch = COMPLETED; } diff --git a/es-app/src/scrapers/Scraper.cpp b/es-app/src/scrapers/Scraper.cpp index 06d94fcba..d7253c58e 100644 --- a/es-app/src/scrapers/Scraper.cpp +++ b/es-app/src/scrapers/Scraper.cpp @@ -163,17 +163,19 @@ MDResolveHandle::MDResolveHandle(const ScraperSearchResult& result, std::string fileFormat; std::string subDirectory; std::string existingMediaFile; + bool resizeFile; } mediaFileInfo; std::vector scrapeFiles; - mResult.savedNewImages = false; + mResult.savedNewMedia = false; if (Settings::getInstance()->getBool("Scrape3DBoxes") && result.box3dUrl != "") { mediaFileInfo.fileURL = result.box3dUrl; mediaFileInfo.fileFormat = result.box3dFormat; mediaFileInfo.subDirectory = "3dboxes"; mediaFileInfo.existingMediaFile = search.game->get3DBoxPath(); + mediaFileInfo.resizeFile = true; scrapeFiles.push_back(mediaFileInfo); } if (Settings::getInstance()->getBool("ScrapeCovers") && result.coverUrl != "") { @@ -181,6 +183,7 @@ MDResolveHandle::MDResolveHandle(const ScraperSearchResult& result, mediaFileInfo.fileFormat = result.coverFormat; mediaFileInfo.subDirectory = "covers"; mediaFileInfo.existingMediaFile = search.game->getCoverPath(); + mediaFileInfo.resizeFile = true; scrapeFiles.push_back(mediaFileInfo); } if (Settings::getInstance()->getBool("ScrapeMarquees") && result.marqueeUrl != "") { @@ -188,6 +191,7 @@ MDResolveHandle::MDResolveHandle(const ScraperSearchResult& result, mediaFileInfo.fileFormat = result.marqueeFormat; mediaFileInfo.subDirectory = "marquees"; mediaFileInfo.existingMediaFile = search.game->getMarqueePath(); + mediaFileInfo.resizeFile = true; scrapeFiles.push_back(mediaFileInfo); } if (Settings::getInstance()->getBool("ScrapeScreenshots") && result.screenshotUrl != "") { @@ -195,6 +199,15 @@ MDResolveHandle::MDResolveHandle(const ScraperSearchResult& result, mediaFileInfo.fileFormat = result.screenshotFormat; mediaFileInfo.subDirectory = "screenshots"; mediaFileInfo.existingMediaFile = search.game->getScreenshotPath(); + mediaFileInfo.resizeFile = true; + scrapeFiles.push_back(mediaFileInfo); + } + if (Settings::getInstance()->getBool("ScrapeVideos") && result.videoUrl != "") { + mediaFileInfo.fileURL = result.videoUrl; + mediaFileInfo.fileFormat = result.videoFormat; + mediaFileInfo.subDirectory = "videos"; + mediaFileInfo.existingMediaFile = search.game->getVideoPath(); + mediaFileInfo.resizeFile = false; scrapeFiles.push_back(mediaFileInfo); } @@ -203,7 +216,7 @@ MDResolveHandle::MDResolveHandle(const ScraperSearchResult& result, std::string ext; // If we have a file extension returned by the scraper, then use it. - // Otherwise, try to guess it by the name of the URL, which point to an image. + // Otherwise, try to guess it by the name of the URL, which points to a media file. if (!it->fileFormat.empty()) { ext = it->fileFormat; } @@ -249,7 +262,7 @@ MDResolveHandle::MDResolveHandle(const ScraperSearchResult& result, std::ofstream stream(filePath, std::ios_base::out | std::ios_base::binary); #endif if (!stream || stream.bad()) { - setError("Failed to open path for writing image.\nPermission error?"); + setError("Failed to open path for writing media file.\nPermission error?"); return; } @@ -257,24 +270,26 @@ MDResolveHandle::MDResolveHandle(const ScraperSearchResult& result, stream.write(content.data(), content.length()); stream.close(); if (stream.bad()) { - setError("Failed to save image.\nDisk full?"); + setError("Failed to save media file.\nDisk full?"); return; } // Resize it. - if (!resizeImage(filePath, Settings::getInstance()->getInt("ScraperResizeMaxWidth"), - Settings::getInstance()->getInt("ScraperResizeMaxHeight"))) { - setError("Error saving resized image.\nOut of memory? Disk full?"); - return; + if (it->resizeFile) { + if (!resizeImage(filePath, Settings::getInstance()->getInt("ScraperResizeMaxWidth"), + Settings::getInstance()->getInt("ScraperResizeMaxHeight"))) { + setError("Error saving resized image.\nOut of memory? Disk full?"); + return; + } } - mResult.savedNewImages = true; + mResult.savedNewMedia = true; } // If it's not cached, then initiate the download. else { - mFuncs.push_back(ResolvePair(downloadImageAsync(it->fileURL, filePath, - it->existingMediaFile, mResult.savedNewImages), [this, filePath] { - })); + mFuncs.push_back(ResolvePair(downloadMediaAsync(it->fileURL, filePath, + it->existingMediaFile, it->resizeFile, mResult.savedNewMedia), + [this, filePath] {})); } } } @@ -303,35 +318,42 @@ void MDResolveHandle::update() setStatus(ASYNC_DONE); } -std::unique_ptr downloadImageAsync(const std::string& url, - const std::string& saveAs, const std::string& existingMediaFile, bool& savedNewImage) +std::unique_ptr downloadMediaAsync( + const std::string& url, + const std::string& saveAs, + const std::string& existingMediaPath, + const bool resizeFile, + bool& savedNewMedia) { - return std::unique_ptr(new ImageDownloadHandle( + return std::unique_ptr(new MediaDownloadHandle( url, saveAs, - existingMediaFile, - savedNewImage, + existingMediaPath, + resizeFile, + savedNewMedia, Settings::getInstance()->getInt("ScraperResizeMaxWidth"), Settings::getInstance()->getInt("ScraperResizeMaxHeight"))); } -ImageDownloadHandle::ImageDownloadHandle( +MediaDownloadHandle::MediaDownloadHandle( const std::string& url, const std::string& path, const std::string& existingMediaPath, - bool& savedNewImage, + const bool resizeFile, + bool& savedNewMedia, int maxWidth, int maxHeight) : mSavePath(path), mExistingMediaFile(existingMediaPath), + mResizeFile(resizeFile), mMaxWidth(maxWidth), mMaxHeight(maxHeight), mReq(new HttpReq(url)) { - mSavedNewImagePtr = &savedNewImage; + mSavedNewMediaPtr = &savedNewMedia; } -void ImageDownloadHandle::update() +void MediaDownloadHandle::update() { if (mReq->status() == HttpReq::REQ_IN_PROGRESS) return; @@ -343,6 +365,11 @@ void ImageDownloadHandle::update() return; } + // This seems to take care of a strange race condition where the media saving and + // resizing would sometimes take place twice. + if (mStatus == ASYNC_DONE) + return; + // Download is done, save it to disk. // Remove any existing media file before attempting to write a new one. @@ -366,7 +393,7 @@ void ImageDownloadHandle::update() std::ofstream stream(mSavePath, std::ios_base::out | std::ios_base::binary); #endif if (!stream || stream.bad()) { - setError("Failed to open path for writing image.\nPermission error?"); + setError("Failed to open path for writing media file.\nPermission error?"); return; } @@ -374,18 +401,20 @@ void ImageDownloadHandle::update() stream.write(content.data(), content.length()); stream.close(); if (stream.bad()) { - setError("Failed to save image.\nDisk full?"); + setError("Failed to save media file.\nDisk full?"); return; } // Resize it. - if (!resizeImage(mSavePath, mMaxWidth, mMaxHeight)) { - setError("Error saving resized image.\nOut of memory? Disk full?"); - return; + if (mResizeFile) { + if (!resizeImage(mSavePath, mMaxWidth, mMaxHeight)) { + setError("Error saving resized image.\nOut of memory? Disk full?"); + return; + } } - // If this image was successfully saved, update savedNewImages in ScraperSearchResult. - *mSavedNewImagePtr = true; + // If this media file was successfully saved, update savedNewMedia in ScraperSearchResult. + *mSavedNewMediaPtr = true; setStatus(ASYNC_DONE); } diff --git a/es-app/src/scrapers/Scraper.h b/es-app/src/scrapers/Scraper.h index 4206bf419..afda1547d 100644 --- a/es-app/src/scrapers/Scraper.h +++ b/es-app/src/scrapers/Scraper.h @@ -24,7 +24,7 @@ class FileData; class SystemData; -enum eDownloadStatus { +enum downloadStatus { NOT_STARTED, IN_PROGRESS, COMPLETED @@ -47,26 +47,28 @@ struct ScraperSearchResult { // within a given time period. unsigned int scraperRequestAllowance; - enum eDownloadStatus mediaURLFetch = NOT_STARTED; - enum eDownloadStatus thumbnailDownloadStatus = NOT_STARTED; - enum eDownloadStatus mediaFilesDownloadStatus = NOT_STARTED; + enum downloadStatus mediaURLFetch = NOT_STARTED; + enum downloadStatus thumbnailDownloadStatus = NOT_STARTED; + enum downloadStatus mediaFilesDownloadStatus = NOT_STARTED; - std::string ThumbnailImageData; // Thumbnail cache, will containe entire image. + std::string ThumbnailImageData; // Thumbnail cache, will contain entire image. std::string ThumbnailImageUrl; std::string box3dUrl; std::string coverUrl; std::string marqueeUrl; std::string screenshotUrl; + std::string videoUrl; // Needed to pre-set the image type. std::string box3dFormat; std::string coverFormat; std::string marqueeFormat; std::string screenshotFormat; + std::string videoFormat; - // Indicate whether any new images were downloaded and saved. - bool savedNewImages; + // Indicates whether any new media files were downloaded and saved. + bool savedNewMedia; }; // So let me explain why I've abstracted this so heavily. @@ -132,8 +134,11 @@ public: ScraperSearchHandle(); void update(); - inline const std::vector& getResults() const { - assert(mStatus != ASYNC_IN_PROGRESS); return mResults; } + inline const std::vector& getResults() const + { + assert(mStatus != ASYNC_IN_PROGRESS); + return mResults; + } protected: friend std::unique_ptr @@ -180,14 +185,15 @@ private: std::vector mFuncs; }; -class ImageDownloadHandle : public AsyncHandle +class MediaDownloadHandle : public AsyncHandle { public: - ImageDownloadHandle( + MediaDownloadHandle( const std::string& url, const std::string& path, const std::string& existingMediaPath, - bool& savedNewImage, + const bool resizeFile, + bool& savedNewMedia, int maxWidth, int maxHeight); @@ -197,7 +203,8 @@ private: std::unique_ptr mReq; std::string mSavePath; std::string mExistingMediaFile; - bool *mSavedNewImagePtr; + bool mResizeFile; + bool *mSavedNewMediaPtr; int mMaxWidth; int mMaxHeight; }; @@ -210,8 +217,12 @@ std::string getSaveAsPath(const ScraperSearchParams& params, // Will resize according to Settings::getInt("ScraperResizeMaxWidth") and // Settings::getInt("ScraperResizeMaxHeight"). -std::unique_ptr downloadImageAsync(const std::string& url, - const std::string& saveAs, const std::string& existingMediaPath, bool& savedNewImage); +std::unique_ptr downloadMediaAsync( + const std::string& url, + const std::string& saveAs, + const std::string& existingMediaPath, + const bool resizeFile, + bool& savedNewMedia); // Resolves all metadata assets that need to be downloaded. std::unique_ptr resolveMetaDataAssets(const ScraperSearchResult& result, diff --git a/es-app/src/scrapers/ScreenScraper.cpp b/es-app/src/scrapers/ScreenScraper.cpp index be7666523..32030990a 100644 --- a/es-app/src/scrapers/ScreenScraper.cpp +++ b/es-app/src/scrapers/ScreenScraper.cpp @@ -248,7 +248,7 @@ void ScreenScraperRequest::processGame(const pugi::xml_document& xmldoc, std::string language = Utils::String::toLower(Settings::getInstance()->getString("ScraperLanguage")); - // Name fallback: US, WOR(LD). ( Xpath: Data/jeu[0]/noms/nom[*] ). + // Name fallback: US, WOR(LD). (Xpath: Data/jeu[0]/noms/nom[*]). result.mdl.set("name", find_child_by_attribute_list(game.child("noms"), "nom", "region", { region, "wor", "us" , "ss", "eu", "jp" }).text().get()); LOG(LogDebug) << "ScreenScraperRequest::processGame(): Name: " << @@ -347,6 +347,9 @@ void ScreenScraperRequest::processGame(const pugi::xml_document& xmldoc, // Screenshot processMedia(result, media_list, ssConfig.media_screenshot, result.screenshotUrl, result.screenshotFormat, region); + // Video + processMedia(result, media_list, ssConfig.media_video, + result.videoUrl, result.videoFormat, region); } result.mediaURLFetch = COMPLETED; out_results.push_back(result); @@ -371,23 +374,30 @@ void ScreenScraperRequest::processMedia( ("media[@type='") + mediaType + "']").c_str()); if (results.size()) { - // Region fallback: WOR(LD), US, CUS(TOM?), JP, EU. - for (auto _region : std::vector{ - region, "wor", "us", "cus", "jp", "eu" }) { - if (art) - break; - - for (auto node : results) { - if (node.node().attribute("region").value() == _region) { - art = node.node(); + // Videos don't have any region attributes, so just take the first entry + // (which should be the only entry as well). + if (mediaType == "video" || mediaType == "video-normalized") { + art = results.first().node(); + } + else { + // Region fallback: WOR(LD), US, CUS(TOM?), JP, EU. + for (auto _region : std::vector{ + region, "wor", "us", "cus", "jp", "eu" }) { + if (art) break; + + for (auto node : results) { + if (node.node().attribute("region").value() == _region) { + art = node.node(); + break; + } } } } } if (art) { - // Sending a 'softname' containing space will make the image URLs returned + // Sending a 'softname' containing space will make the media URLs returned // by the API also contain the space. Escape any spaces in the URL here. fileURL = Utils::String::replace(art.text().get(), " ", "%20"); diff --git a/es-app/src/scrapers/ScreenScraper.h b/es-app/src/scrapers/ScreenScraper.h index 99675e052..ffd4ff475 100644 --- a/es-app/src/scrapers/ScreenScraper.h +++ b/es-app/src/scrapers/ScreenScraper.h @@ -45,7 +45,7 @@ public: const std::string API_DEV_KEY = { 54, 73, 115, 100, 101, 67, 111, 107, 79, 66, 68, 66, 67, 56, 118, 77, 54, 88, 101, 54 }; const std::string API_URL_BASE = "https://www.screenscraper.fr/api2"; - const std::string API_SOFT_NAME = "Emulationstation-DE " + + const std::string API_SOFT_NAME = "EmulationStation-DE " + static_cast(PROGRAM_VERSION_STRING); // Which type of image artwork we need. Possible values (not a comprehensive list): @@ -58,15 +58,17 @@ public: // - wheel: spine // - support-2D: media showing the 2d boxart on the cart // - support-3D: media showing the 3d boxart on the cart + // - video: gameplay videos + // - video-normalized: gameplay videos in smaller file sizes with lower audio quality // - // Note that not all games contain values for all these, so we default to "box-2D" - // since it's the most common. + // Note that not all games contain values for all these, so we default to "ss". // std::string media_3dbox = "box-3D"; std::string media_cover = "box-2D"; std::string media_marquee = "wheel"; std::string media_screenshot = "ss"; + std::string media_video = "video"; // Which Region to use when selecting the artwork. // Applies to: artwork, name of the game, date of release. diff --git a/es-core/src/Settings.cpp b/es-core/src/Settings.cpp index c1098f7cb..e5fd53069 100644 --- a/es-core/src/Settings.cpp +++ b/es-core/src/Settings.cpp @@ -98,7 +98,7 @@ void Settings::setDefaults() mStringMap["ScreenSaverBehavior"] = "dim"; // UI settings -> screensaver settings -> video screensaver settings. - mIntMap["ScreenSaverSwapVideoTimeout"] = 20000; + mIntMap["ScreenSaverSwapVideoTimeout"] = 25000; mBoolMap["ScreenSaverStretchVideos"] = false; #ifdef _RPI_ mStringMap["ScreenSaverGameInfo"] = "never"; @@ -161,6 +161,7 @@ void Settings::setDefaults() mBoolMap["ScrapeCovers"] = true; mBoolMap["ScrapeMarquees"] = true; mBoolMap["ScrapeScreenshots"] = true; + mBoolMap["ScrapeVideos"] = false; // Other settings. #ifdef _RPI_