Xargon/ES-DE
1
0
Fork 0
mirror of https://github.com/RetroDECK/ES-DE.git synced 2024-11-22 06:05:38 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Added support for scraping videos using ScreenScraper.

Browse source
This commit is contained in:
Leon Styhre 2020-08-05 22:38:44 +02:00
parent 2c6bc918d6
commit c3de18dd96
9 changed files with 130 additions and 60 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
NEWS.md
View file

@ -5,7 +5,7 @@ EmulationStation Desktop Edition v1.0.0
### Release overview
First release, a major update to the application compared to the RetroPie version on which it is based. This includes new gamelist sorting logic, new game media handling and a completely updated Windows port (which now works about as well as the Unix version). The menu system has also been completely overhauled and the scraper has been expanded to support multiple media types as well as providing detailed scraping configuration options.
First release, a major update to the application compared to the RetroPie version on which it is based. This includes new gamelist sorting logic, new game media handling and a completely updated Windows port (which now works about as well as the Unix version). The menu system has also been completely overhauled and the scraper has been expanded to support multiple media types (including videos) as well as providing detailed scraping configuration options.
Full navigation sound support has been implemented, and the metadata editor has seen a lot of updates including color coding of all changes done by the user and by the scraper. Favorite games can now also be sorted on top of the gamelists and game collections.

2
es-app/src/guis/GuiMetaDataEd.cpp
View file

@ -392,7 +392,7 @@ void GuiMetaDataEd::fetchDone(const ScraperSearchResult& result)
MetaDataList* metadata = nullptr;
metadata = new MetaDataList(*mMetaData);
mMediaFilesUpdated = result.savedNewImages;
mMediaFilesUpdated = result.savedNewMedia;
// Check if any values were manually changed before starting the scraping.
// If so, it's these values we should compare against when scraping, not

16
es-app/src/guis/GuiScraperMenu.cpp
View file

@ -135,6 +135,22 @@ void GuiScraperMenu::openContentSettings()
s->addSaveFunc([scrape_metadata] { Settings::getInstance()->setBool("ScrapeMetadata",
scrape_metadata->getState()); });
// Scrape videos.
auto scrape_videos = std::make_shared<SwitchComponent>(mWindow);
scrape_videos->setState(Settings::getInstance()->getBool("ScrapeVideos"));
s->addWithLabel("SCRAPE VIDEOS", scrape_videos);
s->addSaveFunc([scrape_videos] { Settings::getInstance()->setBool("ScrapeVideos",
scrape_videos->getState()); });
// Videos are not supported by TheGamesDB, so disable the option if this scraper is selected.
if (Settings::getInstance()->getString("Scraper") == "thegamesdb") {
scrape_videos->setDisabled();
scrape_videos->setOpacity(DISABLED_OPACITY);
// I'm sure there is a better way to find the text component...
scrape_videos->getParent()->getChild(
scrape_videos->getParent()->getChildCount()-2)->setOpacity(DISABLED_OPACITY);
}
// Scrape screenshots images.
auto scrape_screenshots = std::make_shared<SwitchComponent>(mWindow);
scrape_screenshots->setState(Settings::getInstance()->getBool("ScrapeScreenshots"));

1
es-app/src/guis/GuiScraperSearch.cpp
View file

@ -540,6 +540,7 @@ void GuiScraperSearch::update(int deltaTime)
results_scrape[i].coverUrl = it->coverUrl;
results_scrape[i].marqueeUrl = it->marqueeUrl;
results_scrape[i].screenshotUrl = it->screenshotUrl;
results_scrape[i].videoUrl = it->videoUrl;
results_scrape[i].scraperRequestAllowance = it->scraperRequestAllowance;
results_scrape[i].mediaURLFetch = COMPLETED;
}

85
es-app/src/scrapers/Scraper.cpp
View file

@ -163,17 +163,19 @@ MDResolveHandle::MDResolveHandle(const ScraperSearchResult& result,
std::string fileFormat;
std::string subDirectory;
std::string existingMediaFile;
bool resizeFile;
} mediaFileInfo;
std::vector<struct mediaFileInfoStruct> scrapeFiles;
mResult.savedNewImages = false;
mResult.savedNewMedia = false;
if (Settings::getInstance()->getBool("Scrape3DBoxes") && result.box3dUrl != "") {
mediaFileInfo.fileURL = result.box3dUrl;
mediaFileInfo.fileFormat = result.box3dFormat;
mediaFileInfo.subDirectory = "3dboxes";
mediaFileInfo.existingMediaFile = search.game->get3DBoxPath();
mediaFileInfo.resizeFile = true;
scrapeFiles.push_back(mediaFileInfo);
}
if (Settings::getInstance()->getBool("ScrapeCovers") && result.coverUrl != "") {
@ -181,6 +183,7 @@ MDResolveHandle::MDResolveHandle(const ScraperSearchResult& result,
mediaFileInfo.fileFormat = result.coverFormat;
mediaFileInfo.subDirectory = "covers";
mediaFileInfo.existingMediaFile = search.game->getCoverPath();
mediaFileInfo.resizeFile = true;
scrapeFiles.push_back(mediaFileInfo);
}
if (Settings::getInstance()->getBool("ScrapeMarquees") && result.marqueeUrl != "") {
@ -188,6 +191,7 @@ MDResolveHandle::MDResolveHandle(const ScraperSearchResult& result,
mediaFileInfo.fileFormat = result.marqueeFormat;
mediaFileInfo.subDirectory = "marquees";
mediaFileInfo.existingMediaFile = search.game->getMarqueePath();
mediaFileInfo.resizeFile = true;
scrapeFiles.push_back(mediaFileInfo);
}
if (Settings::getInstance()->getBool("ScrapeScreenshots") && result.screenshotUrl != "") {
@ -195,6 +199,15 @@ MDResolveHandle::MDResolveHandle(const ScraperSearchResult& result,
mediaFileInfo.fileFormat = result.screenshotFormat;
mediaFileInfo.subDirectory = "screenshots";
mediaFileInfo.existingMediaFile = search.game->getScreenshotPath();
mediaFileInfo.resizeFile = true;
scrapeFiles.push_back(mediaFileInfo);
}
if (Settings::getInstance()->getBool("ScrapeVideos") && result.videoUrl != "") {
mediaFileInfo.fileURL = result.videoUrl;
mediaFileInfo.fileFormat = result.videoFormat;
mediaFileInfo.subDirectory = "videos";
mediaFileInfo.existingMediaFile = search.game->getVideoPath();
mediaFileInfo.resizeFile = false;
scrapeFiles.push_back(mediaFileInfo);
}
@ -203,7 +216,7 @@ MDResolveHandle::MDResolveHandle(const ScraperSearchResult& result,
std::string ext;
// If we have a file extension returned by the scraper, then use it.
// Otherwise, try to guess it by the name of the URL, which point to an image.
// Otherwise, try to guess it by the name of the URL, which points to a media file.
if (!it->fileFormat.empty()) {
ext = it->fileFormat;
}
@ -249,7 +262,7 @@ MDResolveHandle::MDResolveHandle(const ScraperSearchResult& result,
std::ofstream stream(filePath, std::ios_base::out | std::ios_base::binary);
#endif
if (!stream || stream.bad()) {
setError("Failed to open path for writing image.\nPermission error?");
setError("Failed to open path for writing media file.\nPermission error?");
return;
}
@ -257,24 +270,26 @@ MDResolveHandle::MDResolveHandle(const ScraperSearchResult& result,
stream.write(content.data(), content.length());
stream.close();
if (stream.bad()) {
setError("Failed to save image.\nDisk full?");
setError("Failed to save media file.\nDisk full?");
return;
}
// Resize it.
if (!resizeImage(filePath, Settings::getInstance()->getInt("ScraperResizeMaxWidth"),
Settings::getInstance()->getInt("ScraperResizeMaxHeight"))) {
setError("Error saving resized image.\nOut of memory? Disk full?");
return;
if (it->resizeFile) {
if (!resizeImage(filePath, Settings::getInstance()->getInt("ScraperResizeMaxWidth"),
Settings::getInstance()->getInt("ScraperResizeMaxHeight"))) {
setError("Error saving resized image.\nOut of memory? Disk full?");
return;
}
}
mResult.savedNewImages = true;
mResult.savedNewMedia = true;
}
// If it's not cached, then initiate the download.
else {
mFuncs.push_back(ResolvePair(downloadImageAsync(it->fileURL, filePath,
it->existingMediaFile, mResult.savedNewImages), [this, filePath] {
}));
mFuncs.push_back(ResolvePair(downloadMediaAsync(it->fileURL, filePath,
it->existingMediaFile, it->resizeFile, mResult.savedNewMedia),
[this, filePath] {}));
}
}
}
@ -303,35 +318,42 @@ void MDResolveHandle::update()
setStatus(ASYNC_DONE);
}
std::unique_ptr<ImageDownloadHandle> downloadImageAsync(const std::string& url,
const std::string& saveAs, const std::string& existingMediaFile, bool& savedNewImage)
std::unique_ptr<MediaDownloadHandle> downloadMediaAsync(
const std::string& url,
const std::string& saveAs,
const std::string& existingMediaPath,
const bool resizeFile,
bool& savedNewMedia)
{
return std::unique_ptr<ImageDownloadHandle>(new ImageDownloadHandle(
return std::unique_ptr<MediaDownloadHandle>(new MediaDownloadHandle(
url,
saveAs,
existingMediaFile,
savedNewImage,
existingMediaPath,
resizeFile,
savedNewMedia,
Settings::getInstance()->getInt("ScraperResizeMaxWidth"),
Settings::getInstance()->getInt("ScraperResizeMaxHeight")));
}
ImageDownloadHandle::ImageDownloadHandle(
MediaDownloadHandle::MediaDownloadHandle(
const std::string& url,
const std::string& path,
const std::string& existingMediaPath,
bool& savedNewImage,
const bool resizeFile,
bool& savedNewMedia,
int maxWidth,
int maxHeight)
: mSavePath(path),
mExistingMediaFile(existingMediaPath),
mResizeFile(resizeFile),
mMaxWidth(maxWidth),
mMaxHeight(maxHeight),
mReq(new HttpReq(url))
{
mSavedNewImagePtr = &savedNewImage;
mSavedNewMediaPtr = &savedNewMedia;
}
void ImageDownloadHandle::update()
void MediaDownloadHandle::update()
{
if (mReq->status() == HttpReq::REQ_IN_PROGRESS)
return;
@ -343,6 +365,11 @@ void ImageDownloadHandle::update()
return;
}
// This seems to take care of a strange race condition where the media saving and
// resizing would sometimes take place twice.
if (mStatus == ASYNC_DONE)
return;
// Download is done, save it to disk.
// Remove any existing media file before attempting to write a new one.
@ -366,7 +393,7 @@ void ImageDownloadHandle::update()
std::ofstream stream(mSavePath, std::ios_base::out | std::ios_base::binary);
#endif
if (!stream || stream.bad()) {
setError("Failed to open path for writing image.\nPermission error?");
setError("Failed to open path for writing media file.\nPermission error?");
return;
}
@ -374,18 +401,20 @@ void ImageDownloadHandle::update()
stream.write(content.data(), content.length());
stream.close();
if (stream.bad()) {
setError("Failed to save image.\nDisk full?");
setError("Failed to save media file.\nDisk full?");
return;
}
// Resize it.
if (!resizeImage(mSavePath, mMaxWidth, mMaxHeight)) {
setError("Error saving resized image.\nOut of memory? Disk full?");
return;
if (mResizeFile) {
if (!resizeImage(mSavePath, mMaxWidth, mMaxHeight)) {
setError("Error saving resized image.\nOut of memory? Disk full?");
return;
}
}
// If this image was successfully saved, update savedNewImages in ScraperSearchResult.
*mSavedNewImagePtr = true;
// If this media file was successfully saved, update savedNewMedia in ScraperSearchResult.
*mSavedNewMediaPtr = true;
setStatus(ASYNC_DONE);
}

41
es-app/src/scrapers/Scraper.h
View file

@ -24,7 +24,7 @@
class FileData;
class SystemData;
enum eDownloadStatus {
enum downloadStatus {
NOT_STARTED,
IN_PROGRESS,
COMPLETED
@ -47,26 +47,28 @@ struct ScraperSearchResult {
// within a given time period.
unsigned int scraperRequestAllowance;
enum eDownloadStatus mediaURLFetch = NOT_STARTED;
enum eDownloadStatus thumbnailDownloadStatus = NOT_STARTED;
enum eDownloadStatus mediaFilesDownloadStatus = NOT_STARTED;
enum downloadStatus mediaURLFetch = NOT_STARTED;
enum downloadStatus thumbnailDownloadStatus = NOT_STARTED;
enum downloadStatus mediaFilesDownloadStatus = NOT_STARTED;
std::string ThumbnailImageData; // Thumbnail cache, will containe entire image.
std::string ThumbnailImageData; // Thumbnail cache, will contain entire image.
std::string ThumbnailImageUrl;
std::string box3dUrl;
std::string coverUrl;
std::string marqueeUrl;
std::string screenshotUrl;
std::string videoUrl;
// Needed to pre-set the image type.
std::string box3dFormat;
std::string coverFormat;
std::string marqueeFormat;
std::string screenshotFormat;
std::string videoFormat;
// Indicate whether any new images were downloaded and saved.
bool savedNewImages;
// Indicates whether any new media files were downloaded and saved.
bool savedNewMedia;
};
// So let me explain why I've abstracted this so heavily.
@ -132,8 +134,11 @@ public:
ScraperSearchHandle();
void update();
inline const std::vector<ScraperSearchResult>& getResults() const {
assert(mStatus != ASYNC_IN_PROGRESS); return mResults; }
inline const std::vector<ScraperSearchResult>& getResults() const
{
assert(mStatus != ASYNC_IN_PROGRESS);
return mResults;
}
protected:
friend std::unique_ptr<ScraperSearchHandle>
@ -180,14 +185,15 @@ private:
std::vector<ResolvePair> mFuncs;
};
class ImageDownloadHandle : public AsyncHandle
class MediaDownloadHandle : public AsyncHandle
{
public:
ImageDownloadHandle(
MediaDownloadHandle(
const std::string& url,
const std::string& path,
const std::string& existingMediaPath,
bool& savedNewImage,
const bool resizeFile,
bool& savedNewMedia,
int maxWidth,
int maxHeight);
@ -197,7 +203,8 @@ private:
std::unique_ptr<HttpReq> mReq;
std::string mSavePath;
std::string mExistingMediaFile;
bool *mSavedNewImagePtr;
bool mResizeFile;
bool *mSavedNewMediaPtr;
int mMaxWidth;
int mMaxHeight;
};
@ -210,8 +217,12 @@ std::string getSaveAsPath(const ScraperSearchParams& params,
// Will resize according to Settings::getInt("ScraperResizeMaxWidth") and
// Settings::getInt("ScraperResizeMaxHeight").
std::unique_ptr<ImageDownloadHandle> downloadImageAsync(const std::string& url,
const std::string& saveAs, const std::string& existingMediaPath, bool& savedNewImage);
std::unique_ptr<MediaDownloadHandle> downloadMediaAsync(
const std::string& url,
const std::string& saveAs,
const std::string& existingMediaPath,
const bool resizeFile,
bool& savedNewMedia);
// Resolves all metadata assets that need to be downloaded.
std::unique_ptr<MDResolveHandle> resolveMetaDataAssets(const ScraperSearchResult& result,

32
es-app/src/scrapers/ScreenScraper.cpp
View file

@ -248,7 +248,7 @@ void ScreenScraperRequest::processGame(const pugi::xml_document& xmldoc,
std::string language =
Utils::String::toLower(Settings::getInstance()->getString("ScraperLanguage"));
// Name fallback: US, WOR(LD). ( Xpath: Data/jeu[0]/noms/nom[*] ).
// Name fallback: US, WOR(LD). (Xpath: Data/jeu[0]/noms/nom[*]).
result.mdl.set("name", find_child_by_attribute_list(game.child("noms"),
"nom", "region", { region, "wor", "us" , "ss", "eu", "jp" }).text().get());
LOG(LogDebug) << "ScreenScraperRequest::processGame(): Name: " <<
@ -347,6 +347,9 @@ void ScreenScraperRequest::processGame(const pugi::xml_document& xmldoc,
// Screenshot
processMedia(result, media_list, ssConfig.media_screenshot,
result.screenshotUrl, result.screenshotFormat, region);
// Video
processMedia(result, media_list, ssConfig.media_video,
result.videoUrl, result.videoFormat, region);
}
result.mediaURLFetch = COMPLETED;
out_results.push_back(result);
@ -371,23 +374,30 @@ void ScreenScraperRequest::processMedia(
("media[@type='") + mediaType + "']").c_str());
if (results.size()) {
// Region fallback: WOR(LD), US, CUS(TOM?), JP, EU.
for (auto _region : std::vector<std::string>{
region, "wor", "us", "cus", "jp", "eu" }) {
if (art)
break;
for (auto node : results) {
if (node.node().attribute("region").value() == _region) {
art = node.node();
// Videos don't have any region attributes, so just take the first entry
// (which should be the only entry as well).
if (mediaType == "video" || mediaType == "video-normalized") {
art = results.first().node();
}
else {
// Region fallback: WOR(LD), US, CUS(TOM?), JP, EU.
for (auto _region : std::vector<std::string>{
region, "wor", "us", "cus", "jp", "eu" }) {
if (art)
break;
for (auto node : results) {
if (node.node().attribute("region").value() == _region) {
art = node.node();
break;
}
}
}
}
}
if (art) {
// Sending a 'softname' containing space will make the image URLs returned
// Sending a 'softname' containing space will make the media URLs returned
// by the API also contain the space. Escape any spaces in the URL here.
fileURL = Utils::String::replace(art.text().get(), " ", "%20");

8
es-app/src/scrapers/ScreenScraper.h
View file

@ -45,7 +45,7 @@ public:
const std::string API_DEV_KEY =
{ 54, 73, 115, 100, 101, 67, 111, 107, 79, 66, 68, 66, 67, 56, 118, 77, 54, 88, 101, 54 };
const std::string API_URL_BASE = "https://www.screenscraper.fr/api2";
const std::string API_SOFT_NAME = "Emulationstation-DE " +
const std::string API_SOFT_NAME = "EmulationStation-DE " +
static_cast<std::string>(PROGRAM_VERSION_STRING);
// Which type of image artwork we need. Possible values (not a comprehensive list):
@ -58,15 +58,17 @@ public:
// - wheel: spine
// - support-2D: media showing the 2d boxart on the cart
// - support-3D: media showing the 3d boxart on the cart
// - video: gameplay videos
// - video-normalized: gameplay videos in smaller file sizes with lower audio quality
//
// Note that not all games contain values for all these, so we default to "box-2D"
// since it's the most common.
// Note that not all games contain values for all these, so we default to "ss".
//
std::string media_3dbox = "box-3D";
std::string media_cover = "box-2D";
std::string media_marquee = "wheel";
std::string media_screenshot = "ss";
std::string media_video = "video";
// Which Region to use when selecting the artwork.
// Applies to: artwork, name of the game, date of release.

3
es-core/src/Settings.cpp
View file

@ -98,7 +98,7 @@ void Settings::setDefaults()
mStringMap["ScreenSaverBehavior"] = "dim";
// UI settings -> screensaver settings -> video screensaver settings.
mIntMap["ScreenSaverSwapVideoTimeout"] = 20000;
mIntMap["ScreenSaverSwapVideoTimeout"] = 25000;
mBoolMap["ScreenSaverStretchVideos"] = false;
#ifdef _RPI_
mStringMap["ScreenSaverGameInfo"] = "never";
@ -161,6 +161,7 @@ void Settings::setDefaults()
mBoolMap["ScrapeCovers"] = true;
mBoolMap["ScrapeMarquees"] = true;
mBoolMap["ScrapeScreenshots"] = true;
mBoolMap["ScrapeVideos"] = false;
// Other settings.
#ifdef _RPI_