Xargon/ES-DE
1
0
Fork 0
mirror of https://github.com/RetroDECK/ES-DE.git synced 2024-11-22 06:05:38 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Added an option to scrape game manuals using ScreenScraper

Browse source 
Also changed the scraper auto-retry functionality to not run on non-recoverable errors or duing manual scraping
This commit is contained in:
Leon Styhre 2023-05-08 17:14:52 +02:00
parent 2da37eb896
commit d83374b38f
13 changed files with 177 additions and 76 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
es-app/src/FileData.cpp
View file

@ -322,6 +322,29 @@ const std::string FileData::getVideoPath() const
return "";
}
const std::string FileData::getManualPath() const
{
const std::vector<std::string> extList {".pdf"};
std::string subFolders;
// Extract possible subfolders from the path.
if (mEnvData->mStartPath != "")
subFolders =
Utils::String::replace(Utils::FileSystem::getParent(mPath), mEnvData->mStartPath, "");
const std::string tempPath {getMediaDirectory() + mSystemName + "/manuals" + subFolders + "/" +
getDisplayName()};
// Look for manuals in the media directory.
for (size_t i {0}; i < extList.size(); ++i) {
std::string mediaPath {tempPath + extList[i]};
if (Utils::FileSystem::exists(mediaPath))
return mediaPath;
}
return "";
}
const std::vector<FileData*>& FileData::getChildrenListToDisplay()
{
FileFilterIndex* idx {mSystem->getIndex()};

1
es-app/src/FileData.h
View file

@ -96,6 +96,7 @@ public:
const std::string getScreenshotPath() const;
const std::string getTitleScreenPath() const;
const std::string getVideoPath() const;
const std::string getManualPath() const;
const bool getDeletionFlag() const { return mDeletionFlag; }
void setDeletionFlag(bool setting) { mDeletionFlag = setting; }

26
es-app/src/guis/GuiScraperMenu.cpp
View file

@ -419,6 +419,27 @@ void GuiScraperMenu::openContentOptions()
}
});
// Scrape game manuals.
auto scrapeManuals = std::make_shared<SwitchComponent>();
scrapeManuals->setState(Settings::getInstance()->getBool("ScrapeManuals"));
s->addWithLabel("GAME MANUALS", scrapeManuals);
s->addSaveFunc([scrapeManuals, s] {
if (scrapeManuals->getState() != Settings::getInstance()->getBool("ScrapeManuals")) {
Settings::getInstance()->setBool("ScrapeManuals", scrapeManuals->getState());
s->setNeedsSaving();
}
});
// Game manuals are not supported by TheGamesDB, so gray out the option if this scraper
// is selected.
if (Settings::getInstance()->getString("Scraper") == "thegamesdb") {
scrapeManuals->setEnabled(false);
scrapeManuals->setOpacity(DISABLED_OPACITY);
scrapeManuals->getParent()
->getChild(scrapeManuals->getChildIndex() - 1)
->setOpacity(DISABLED_OPACITY);
}
mWindow->pushGui(s);
}
@ -1126,6 +1147,11 @@ void GuiScraperMenu::start()
contentToScrape = true;
break;
}
if (scraperService == "screenscraper" &&
Settings::getInstance()->getBool("ScrapeManuals")) {
contentToScrape = true;
break;
}
if (Settings::getInstance()->getBool("ScrapeMarquees")) {
contentToScrape = true;
break;

30
es-app/src/guis/GuiScraperSearch.cpp
View file

@ -255,7 +255,7 @@ void GuiScraperSearch::resizeMetadata()
mRenderer->getScreenWidthModifier()));
}
for (unsigned int i = 0; i < mMD_Pairs.size(); ++i)
for (unsigned int i {0}; i < mMD_Pairs.size(); ++i)
mMD_Grid->setRowHeightPerc(
i * 2,
(fontLbl->getLetterHeight() + (2.0f * (mRenderer->getIsVerticalOrientation() ?
@ -406,7 +406,7 @@ void GuiScraperSearch::onSearchDone(std::vector<ScraperSearchResult>& results)
mFoundGame = true;
ComponentListRow row;
for (size_t i = 0; i < results.size(); ++i) {
for (size_t i {0}; i < results.size(); ++i) {
// If the platform IDs returned by the scraper do not match the platform IDs of the
// scraped game, then add the additional platform information to the end of the game
// name (within square brackets).
@ -425,7 +425,7 @@ void GuiScraperSearch::onSearchDone(std::vector<ScraperSearchResult>& results)
}
}
bool hasOtherPlatforms = false;
bool hasOtherPlatforms {false};
for (auto& platformID : mLastSearch.system->getSystemEnvData()->mPlatformIds) {
if (!results.at(i).platformIDs.empty() &&
@ -495,11 +495,13 @@ void GuiScraperSearch::onSearchDone(std::vector<ScraperSearchResult>& results)
}
}
void GuiScraperSearch::onSearchError(const std::string& error, HttpReq::Status status)
void GuiScraperSearch::onSearchError(const std::string& error,
const bool retry,
HttpReq::Status status)
{
const int retries {
glm::clamp(Settings::getInstance()->getInt("ScraperRetryOnErrorCount"), 0, 10)};
if (retries > 0 && mRetryCount < retries) {
if (retry && mSearchType != NEVER_AUTO_ACCEPT && retries > 0 && mRetryCount < retries) {
LOG(LogError) << "GuiScraperSearch: " << Utils::String::replace(error, "\n", "");
mRetrySearch = true;
++mRetryCount;
@ -535,7 +537,7 @@ int GuiScraperSearch::getSelectedIndex()
void GuiScraperSearch::updateInfoPane()
{
int i = getSelectedIndex();
int i {getSelectedIndex()};
if (mSearchType == ALWAYS_ACCEPT_FIRST_RESULT && mScraperResults.size())
i = 0;
@ -670,6 +672,8 @@ void GuiScraperSearch::returnResult(ScraperSearchResult result)
// Resolve metadata image before returning.
if (result.mediaFilesDownloadStatus != COMPLETED) {
result.mediaFilesDownloadStatus = IN_PROGRESS;
LOG(LogDebug) << "GuiScraperSearch::returnResult(): Selected game \""
<< result.mdl.get("name") << "\"";
mMDResolveHandle = resolveMetaDataAssets(result, mLastSearch);
return;
}
@ -710,7 +714,8 @@ void GuiScraperSearch::update(int deltaTime)
if (mSearchHandle && mSearchHandle->status() != ASYNC_IN_PROGRESS) {
auto status = mSearchHandle->status();
mScraperResults = mSearchHandle->getResults();
auto statusString = mSearchHandle->getStatusString();
const std::string statusString {mSearchHandle->getStatusString()};
const bool retryFlag {mSearchHandle->getRetry()};
// We reset here because onSearchDone in auto mode can call mSkipCallback() which
// can call another search() which will set our mSearchHandle to something important.
@ -734,7 +739,7 @@ void GuiScraperSearch::update(int deltaTime)
}
}
else if (status == ASYNC_ERROR) {
onSearchError(statusString);
onSearchError(statusString, retryFlag);
}
}
@ -767,7 +772,8 @@ void GuiScraperSearch::update(int deltaTime)
onSearchDone(results_scrape);
}
else if (mMDRetrieveURLsHandle->status() == ASYNC_ERROR) {
onSearchError(mMDRetrieveURLsHandle->getStatusString());
onSearchError(mMDRetrieveURLsHandle->getStatusString(),
mMDRetrieveURLsHandle->getRetry());
mMDRetrieveURLsHandle.reset();
}
}
@ -823,7 +829,7 @@ void GuiScraperSearch::update(int deltaTime)
}
}
else if (mMDResolveHandle->status() == ASYNC_ERROR) {
onSearchError(mMDResolveHandle->getStatusString());
onSearchError(mMDResolveHandle->getStatusString(), mMDResolveHandle->getRetry());
mMDResolveHandle.reset();
}
}
@ -850,7 +856,7 @@ void GuiScraperSearch::updateThumbnail()
}
else {
mResultThumbnail->setImage("");
onSearchError("Error downloading thumbnail:\n " + it->second->getErrorMsg(),
onSearchError("Error downloading thumbnail:\n " + it->second->getErrorMsg(), true,
it->second->status());
}
@ -954,7 +960,7 @@ bool GuiScraperSearch::saveMetadata(const ScraperSearchResult& result,
if (defaultName == metadata.get("name"))
hasDefaultName = true;
for (unsigned int i = 0; i < mMetaDataDecl.size(); ++i) {
for (unsigned int i {0}; i < mMetaDataDecl.size(); ++i) {
// Skip elements that are tagged not to be scraped.
if (!mMetaDataDecl.at(i).shouldScrape)

1
es-app/src/guis/GuiScraperSearch.h
View file

@ -108,6 +108,7 @@ private:
void resizeMetadata();
void onSearchError(const std::string& error,
const bool retry,
HttpReq::Status status = HttpReq::REQ_UNDEFINED_ERROR);
void onSearchDone(std::vector<ScraperSearchResult>& results);

29
es-app/src/scrapers/GamesDBJSONScraper.cpp
View file

@ -200,10 +200,11 @@ void thegamesdb_generate_json_scraper_requests(
if (Settings::getInstance()->getBool("ScraperConvertUnderscores"))
cleanName = Utils::String::replace(cleanName, "_", " ");
path += "/Games/ByGameName?" + apiKey +
"&fields=players,publishers,genres,overview,last_updated,rating,"
"platform,coop,youtube,os,processor,ram,hdd,video,sound,alternates&name=" +
HttpReq::urlEncode(cleanName);
path.append("/Games/ByGameName?")
.append(apiKey)
.append("&fields=players,publishers,genres,overview,last_updated,rating,"
"platform,coop,youtube,os,processor,ram,hdd,video,sound,alternates&name=")
.append(HttpReq::urlEncode(cleanName));
}
if (usingGameID) {
@ -248,10 +249,10 @@ void thegamesdb_generate_json_scraper_requests(
std::vector<ScraperSearchResult>& results)
{
resources.prepare();
std::string path = "https://api.thegamesdb.net/v1";
std::string path {"https://api.thegamesdb.net/v1"};
const std::string apiKey {std::string("apikey=") + resources.getApiKey()};
path += "/Games/Images/GamesImages?" + apiKey + "&games_id=" + gameIDs;
path.append("/Games/Images/GamesImages?").append(apiKey).append("&games_id=").append(gameIDs);
requests.push(
std::unique_ptr<ScraperRequest>(new TheGamesDBJSONRequest(requests, results, path)));
@ -290,9 +291,9 @@ namespace
if (!v.IsArray())
return "";
std::string out = "";
std::string out;
bool first {true};
for (int i = 0; i < static_cast<int>(v.Size()); ++i) {
for (int i {0}; i < static_cast<int>(v.Size()); ++i) {
auto mapIt = resources.gamesdb_new_developers_map.find(getIntOrThrow(v[i]));
if (mapIt == resources.gamesdb_new_developers_map.cend())
@ -314,7 +315,7 @@ namespace
std::string out;
bool first {true};
for (int i = 0; i < static_cast<int>(v.Size()); ++i) {
for (int i {0}; i < static_cast<int>(v.Size()); ++i) {
auto mapIt = resources.gamesdb_new_publishers_map.find(getIntOrThrow(v[i]));
if (mapIt == resources.gamesdb_new_publishers_map.cend())
@ -336,7 +337,7 @@ namespace
std::string out;
bool first {true};
for (int i = 0; i < static_cast<int>(v.Size()); ++i) {
for (int i {0}; i < static_cast<int>(v.Size()); ++i) {
auto mapIt = resources.gamesdb_new_genres_map.find(getIntOrThrow(v[i]));
if (mapIt == resources.gamesdb_new_genres_map.cend())
@ -432,7 +433,7 @@ void processMediaURLs(const Value& images,
// Quite excessive testing for valid values, but you never know what the server has
// returned and we don't want to crash the program due to malformed data.
if (gameMedia.IsArray()) {
for (SizeType i = 0; i < gameMedia.Size(); ++i) {
for (SizeType i {0}; i < gameMedia.Size(); ++i) {
std::string mediatype;
std::string mediaside;
if (gameMedia[i]["type"].IsString())
@ -477,7 +478,7 @@ void TheGamesDBJSONRequest::process(const std::unique_ptr<HttpReq>& req,
if (doc.HasParseError()) {
std::string err {std::string("TheGamesDBJSONRequest - Error parsing JSON \n\t") +
GetParseError_En(doc.GetParseError())};
setError(err);
setError(err, true);
LOG(LogError) << err;
return;
}
@ -508,7 +509,7 @@ void TheGamesDBJSONRequest::process(const std::unique_ptr<HttpReq>& req,
// Find how many more requests we can make before the scraper
// request allowance counter is reset.
if (doc.HasMember("remaining_monthly_allowance") && doc.HasMember("extra_allowance")) {
for (size_t i = 0; i < results.size(); ++i) {
for (size_t i {0}; i < results.size(); ++i) {
results[i].scraperRequestAllowance =
doc["remaining_monthly_allowance"].GetInt() + doc["extra_allowance"].GetInt();
}
@ -529,7 +530,7 @@ void TheGamesDBJSONRequest::process(const std::unique_ptr<HttpReq>& req,
const Value& games {doc["data"]["games"]};
resources.ensureResources();
for (int i = 0; i < static_cast<int>(games.Size()); ++i) {
for (int i {0}; i < static_cast<int>(games.Size()); ++i) {
auto& v = games[i];
try {
processGame(v, results);

119
es-app/src/scrapers/Scraper.cpp
View file

@ -62,7 +62,7 @@ std::unique_ptr<ScraperSearchHandle> startScraperSearch(const ScraperSearchParam
std::unique_ptr<ScraperSearchHandle> startMediaURLsFetch(const std::string& gameIDs)
{
const std::string& name = Settings::getInstance()->getString("Scraper");
const std::string& name {Settings::getInstance()->getString("Scraper")};
std::unique_ptr<ScraperSearchHandle> handle(new ScraperSearchHandle());
ScraperSearchParams params;
@ -90,7 +90,7 @@ std::vector<std::string> getScraperList()
bool isValidConfiguredScraper()
{
const std::string& name = Settings::getInstance()->getString("Scraper");
const std::string& name {Settings::getInstance()->getString("Scraper")};
return scraper_request_funcs.find(name) != scraper_request_funcs.end();
}
@ -107,7 +107,7 @@ void ScraperSearchHandle::update()
if (status == ASYNC_ERROR) {
// Propagate error.
setError(req.getStatusString());
setError(req.getStatusString(), req.getRetry());
// Empty our queue.
while (!mRequestQueue.empty())
@ -147,7 +147,7 @@ ScraperHttpRequest::ScraperHttpRequest(std::vector<ScraperSearchResult>& results
void ScraperHttpRequest::update()
{
HttpReq::Status status = mReq->status();
HttpReq::Status status {mReq->status()};
if (status == HttpReq::REQ_SUCCESS) {
// If process() has an error, status will be changed to ASYNC_ERROR.
setStatus(ASYNC_DONE);
@ -162,7 +162,7 @@ void ScraperHttpRequest::update()
// Everything else is some sort of error.
LOG(LogError) << "ScraperHttpRequest network error (status: " << status << ") - "
<< mReq->getErrorMsg();
setError("Network error: " + mReq->getErrorMsg());
setError("Network error: " + mReq->getErrorMsg(), true);
}
// Download and write the media files to disk.
@ -264,9 +264,16 @@ MDResolveHandle::MDResolveHandle(const ScraperSearchResult& result,
ViewController::getInstance()->stopViewVideos();
#endif
}
if (Settings::getInstance()->getBool("ScrapeManuals") && result.manualUrl != "") {
mediaFileInfo.fileURL = result.manualUrl;
mediaFileInfo.fileFormat = result.manualFormat;
mediaFileInfo.subDirectory = "manuals";
mediaFileInfo.existingMediaFile = search.game->getManualPath();
mediaFileInfo.resizeFile = false;
scrapeFiles.push_back(mediaFileInfo);
}
for (auto it = scrapeFiles.cbegin(); it != scrapeFiles.cend(); ++it) {
std::string ext;
// If we have a file extension returned by the scraper, then use it.
@ -275,13 +282,13 @@ MDResolveHandle::MDResolveHandle(const ScraperSearchResult& result,
ext = it->fileFormat;
}
else {
size_t dot = it->fileURL.find_last_of('.');
size_t dot {it->fileURL.find_last_of('.')};
if (dot != std::string::npos)
ext = it->fileURL.substr(dot, std::string::npos);
}
std::string filePath = getSaveAsPath(search, it->subDirectory, ext);
std::string filePath {getSaveAsPath(search, it->subDirectory, ext)};
// If there is an existing media file on disk and the setting to overwrite data
// has been set to no, then don't proceed with downloading or saving a new file.
@ -304,17 +311,19 @@ MDResolveHandle::MDResolveHandle(const ScraperSearchResult& result,
if (Settings::getInstance()->getBool("ScraperHaltOnInvalidMedia") &&
mResult.thumbnailImageData.size() < 350) {
FIMEMORY* memoryStream =
FIMEMORY* memoryStream {
FreeImage_OpenMemory(reinterpret_cast<BYTE*>(&mResult.thumbnailImageData.at(0)),
static_cast<DWORD>(mResult.thumbnailImageData.size()));
static_cast<DWORD>(mResult.thumbnailImageData.size()))};
FREE_IMAGE_FORMAT imageFormat = FreeImage_GetFileTypeFromMemory(memoryStream, 0);
FREE_IMAGE_FORMAT imageFormat {FreeImage_GetFileTypeFromMemory(memoryStream, 0)};
FreeImage_CloseMemory(memoryStream);
if (imageFormat == FIF_UNKNOWN) {
setError("The file \"" + Utils::FileSystem::getFileName(filePath) +
"\" returned by the scraper seems to be invalid as it's less than " +
"350 bytes in size");
setError(
"The file \"" + Utils::FileSystem::getFileName(filePath) +
"\" returned by the scraper seems to be invalid as it's less than " +
"350 bytes in size",
true);
return;
}
}
@ -330,7 +339,8 @@ MDResolveHandle::MDResolveHandle(const ScraperSearchResult& result,
// problems or the MediaDirectory setting points to a file instead of a directory.
if (!Utils::FileSystem::isDirectory(Utils::FileSystem::getParent(filePath))) {
setError("Media directory does not exist and can't be created. "
"Permission problems?");
"Permission problems?",
false);
LOG(LogError) << "Couldn't create media directory: \""
<< Utils::FileSystem::getParent(filePath) << "\"";
return;
@ -343,22 +353,22 @@ MDResolveHandle::MDResolveHandle(const ScraperSearchResult& result,
std::ofstream stream(filePath, std::ios_base::out | std::ios_base::binary);
#endif
if (!stream || stream.bad()) {
setError("Failed to open path for writing media file.\nPermission error?");
setError("Failed to open path for writing media file.\nPermission error?", false);
return;
}
const std::string& content = mResult.thumbnailImageData;
const std::string& content {mResult.thumbnailImageData};
stream.write(content.data(), content.length());
stream.close();
if (stream.bad()) {
setError("Failed to save media file.\nDisk full?");
setError("Failed to save media file.\nDisk full?", false);
return;
}
// Resize it.
if (it->resizeFile) {
if (!resizeImage(filePath, it->subDirectory)) {
setError("Error saving resized image.\nOut of memory? Disk full?");
setError("Error saving resized image.\nOut of memory? Disk full?", false);
return;
}
}
@ -384,7 +394,7 @@ void MDResolveHandle::update()
while (it != mFuncs.cend()) {
if (it->first->status() == ASYNC_ERROR) {
setError(it->first->getStatusString());
setError(it->first->getStatusString(), it->first->getRetry());
return;
}
else if (it->first->status() == ASYNC_DONE) {
@ -433,7 +443,7 @@ void MediaDownloadHandle::update()
if (mReq->status() != HttpReq::REQ_SUCCESS) {
std::stringstream ss;
ss << "Network error: " << mReq->getErrorMsg();
setError(ss.str());
setError(ss.str(), true);
return;
}
@ -450,22 +460,22 @@ void MediaDownloadHandle::update()
// and skip them so they're not saved to disk.
if (Settings::getInstance()->getString("Scraper") == "screenscraper" &&
mMediaType == "backcovers") {
bool emptyImage = false;
FREE_IMAGE_FORMAT imageFormat = FIF_UNKNOWN;
std::string imageData = mReq->getContent();
FIMEMORY* memoryStream = FreeImage_OpenMemory(reinterpret_cast<BYTE*>(&imageData.at(0)),
static_cast<DWORD>(imageData.size()));
bool emptyImage {false};
FREE_IMAGE_FORMAT imageFormat {FIF_UNKNOWN};
std::string imageData {mReq->getContent()};
FIMEMORY* memoryStream {FreeImage_OpenMemory(reinterpret_cast<BYTE*>(&imageData.at(0)),
static_cast<DWORD>(imageData.size()))};
imageFormat = FreeImage_GetFileTypeFromMemory(memoryStream, 0);
if (imageFormat != FIF_UNKNOWN) {
emptyImage = true;
FIBITMAP* tempImage = FreeImage_LoadFromMemory(imageFormat, memoryStream);
FIBITMAP* tempImage {FreeImage_LoadFromMemory(imageFormat, memoryStream)};
RGBQUAD firstPixel;
RGBQUAD currPixel;
unsigned int width = FreeImage_GetWidth(tempImage);
unsigned int height = FreeImage_GetHeight(tempImage);
unsigned int width {FreeImage_GetWidth(tempImage)};
unsigned int height {FreeImage_GetHeight(tempImage)};
// Skip really small images as they're obviously not valid.
if (width < 50) {
@ -477,7 +487,7 @@ void MediaDownloadHandle::update()
else {
// Remove the alpha channel which will convert fully transparent pixels to black.
if (FreeImage_GetBPP(tempImage) != 24) {
FIBITMAP* convertImage = FreeImage_ConvertTo24Bits(tempImage);
FIBITMAP* convertImage {FreeImage_ConvertTo24Bits(tempImage)};
FreeImage_Unload(tempImage);
tempImage = convertImage;
}
@ -485,11 +495,11 @@ void MediaDownloadHandle::update()
// Skip the first line as this can apparently lead to false positives.
FreeImage_GetPixelColor(tempImage, 0, 1, &firstPixel);
for (unsigned int x = 0; x < width; ++x) {
for (unsigned int x {0}; x < width; ++x) {
if (!emptyImage)
break;
// Skip the last line as well.
for (unsigned int y = 1; y < height - 1; ++y) {
for (unsigned int y {1}; y < height - 1; ++y) {
FreeImage_GetPixelColor(tempImage, x, y, &currPixel);
if (currPixel.rgbBlue != firstPixel.rgbBlue ||
currPixel.rgbGreen != firstPixel.rgbGreen ||
@ -527,20 +537,21 @@ void MediaDownloadHandle::update()
if (Settings::getInstance()->getBool("ScraperHaltOnInvalidMedia") &&
mReq->getContent().size() < 350) {
FREE_IMAGE_FORMAT imageFormat = FIF_UNKNOWN;
FREE_IMAGE_FORMAT imageFormat {FIF_UNKNOWN};
if (mMediaType != "videos") {
std::string imageData = mReq->getContent();
FIMEMORY* memoryStream = FreeImage_OpenMemory(reinterpret_cast<BYTE*>(&imageData.at(0)),
static_cast<DWORD>(imageData.size()));
std::string imageData {mReq->getContent()};
FIMEMORY* memoryStream {FreeImage_OpenMemory(reinterpret_cast<BYTE*>(&imageData.at(0)),
static_cast<DWORD>(imageData.size()))};
imageFormat = FreeImage_GetFileTypeFromMemory(memoryStream, 0);
FreeImage_CloseMemory(memoryStream);
}
if (imageFormat == FIF_UNKNOWN) {
setError("The file \"" + Utils::FileSystem::getFileName(mSavePath) +
"\" returned by the scraper seems to be invalid as it's less than " +
"350 bytes in size");
"\" returned by the scraper seems to be invalid as it's less than " +
"350 bytes in size",
true);
return;
}
}
@ -555,7 +566,8 @@ void MediaDownloadHandle::update()
// If the media directory does not exist, something is wrong, possibly permission
// problems or the MediaDirectory setting points to a file instead of a directory.
if (!Utils::FileSystem::isDirectory(Utils::FileSystem::getParent(mSavePath))) {
setError("Media directory does not exist and can't be created. Permission problems?");
setError("Media directory does not exist and can't be created. Permission problems?",
false);
LOG(LogError) << "Couldn't create media directory: \""
<< Utils::FileSystem::getParent(mSavePath) << "\"";
return;
@ -568,22 +580,29 @@ void MediaDownloadHandle::update()
std::ofstream stream(mSavePath, std::ios_base::out | std::ios_base::binary);
#endif
if (!stream || stream.bad()) {
setError("Failed to open path for writing media file.\nPermission error?");
setError("Failed to open path for writing media file.\nPermission error?", false);
return;
}
const std::string& content = mReq->getContent();
const std::string& content {mReq->getContent()};
stream.write(content.data(), content.length());
stream.close();
if (stream.bad()) {
setError("Failed to save media file.\nDisk full?");
setError("Failed to save media file.\nDisk full?", false);
return;
}
if (mMediaType == "manuals") {
LOG(LogDebug) << "Scraper::update(): Saving game manual \"" << mSavePath << "\"";
}
else if (mMediaType == "videos") {
LOG(LogDebug) << "Scraper::update(): Saving video \"" << mSavePath << "\"";
}
// Resize it.
if (mResizeFile) {
if (!resizeImage(mSavePath, mMediaType)) {
setError("Error saving resized image.\nOut of memory? Disk full?");
setError("Error saving resized image.\nOut of memory? Disk full?", false);
return;
}
}
@ -713,8 +732,8 @@ std::string getSaveAsPath(const ScraperSearchParams& params,
const std::string& filetypeSubdirectory,
const std::string& extension)
{
const std::string systemsubdirectory = params.system->getName();
const std::string name = Utils::FileSystem::getStem(params.game->getPath());
const std::string systemsubdirectory {params.system->getName()};
const std::string name {Utils::FileSystem::getStem(params.game->getPath())};
std::string subFolders;
// Extract possible subfolders from the path.
@ -722,16 +741,20 @@ std::string getSaveAsPath(const ScraperSearchParams& params,
subFolders = Utils::String::replace(Utils::FileSystem::getParent(params.game->getPath()),
params.system->getSystemEnvData()->mStartPath, "");
std::string path = FileData::getMediaDirectory();
std::string path {FileData::getMediaDirectory()};
if (!Utils::FileSystem::exists(path))
Utils::FileSystem::createDirectory(path);
path += systemsubdirectory + "/" + filetypeSubdirectory + subFolders + "/";
path.append(systemsubdirectory)
.append("/")
.append(filetypeSubdirectory)
.append(subFolders)
.append("/");
if (!Utils::FileSystem::exists(path))
Utils::FileSystem::createDirectory(path);
path += name + extension;
path.append(name).append(extension);
return path;
}

2
es-app/src/scrapers/Scraper.h
View file

@ -81,6 +81,7 @@ struct ScraperSearchResult {
std::string screenshotUrl;
std::string titlescreenUrl;
std::string videoUrl;
std::string manualUrl;
// Needed to pre-set the image type.
std::string box3DFormat;
@ -92,6 +93,7 @@ struct ScraperSearchResult {
std::string screenshotFormat;
std::string titlescreenFormat;
std::string videoFormat;
std::string manualFormat;
// Indicates whether any new media files were downloaded and saved.
bool savedNewMedia;

5
es-app/src/scrapers/ScreenScraper.cpp
View file

@ -266,7 +266,7 @@ void ScreenScraperRequest::process(const std::unique_ptr<HttpReq>& req,
std::string err = ss.str();
LOG(LogError) << err;
setError("ScreenScraper error: \n" + req->getContent());
setError("ScreenScraper error: \n" + req->getContent(), true);
return;
}
@ -606,6 +606,9 @@ void ScreenScraperRequest::processGame(const pugi::xml_document& xmldoc,
if (result.videoUrl == "")
processMedia(result, media_list, ssConfig.media_video_normalized, result.videoUrl,
result.videoFormat, region);
// Game manuals.
processMedia(result, media_list, ssConfig.media_manual, result.manualUrl,
result.manualFormat, region);
}
result.mediaURLFetch = COMPLETED;
out_results.emplace_back(result);

1
es-app/src/scrapers/ScreenScraper.h
View file

@ -97,6 +97,7 @@ public:
std::string media_titlescreen = "sstitle";
std::string media_video = "video";
std::string media_video_normalized = "video-normalized";
std::string media_manual = "manuel";
bool isArcadeSystem;
bool automaticMode;

8
es-app/src/views/GamelistBase.cpp
View file

@ -668,6 +668,14 @@ void GamelistBase::removeMedia(FileData* game)
removeEmptyDirFunc(systemMediaDir, mediaType, path);
}
while (Utils::FileSystem::exists(game->getManualPath())) {
mediaType = "manuals";
path = game->getManualPath();
if (Utils::FileSystem::removeFile(path))
break;
removeEmptyDirFunc(systemMediaDir, mediaType, path);
}
while (Utils::FileSystem::exists(game->getMiximagePath())) {
mediaType = "miximages";
path = game->getMiximagePath();

7
es-core/src/AsyncHandle.h
View file

@ -23,6 +23,7 @@ class AsyncHandle
public:
AsyncHandle()
: mStatus(ASYNC_IN_PROGRESS)
, mRetry {true}
{
}
virtual ~AsyncHandle() {}
@ -36,6 +37,8 @@ public:
return mStatus;
}
const bool getRetry() { return mRetry; }
// User-friendly string of our current status.
// Will return error message if status() == SEARCH_ERROR.
std::string getStatusString()
@ -54,14 +57,16 @@ public:
protected:
void setStatus(AsyncHandleStatus status) { mStatus = status; }
void setError(const std::string& error)
void setError(const std::string& error, bool retry)
{
setStatus(ASYNC_ERROR);
mError = error;
mRetry = retry;
}
std::string mError;
AsyncHandleStatus mStatus;
bool mRetry;
};
#endif // ES_CORE_ASYNC_HANDLE_H

1
es-core/src/Settings.cpp
View file

@ -118,6 +118,7 @@ void Settings::setDefaults()
mBoolMap["Scrape3DBoxes"] = {true, true};
mBoolMap["ScrapePhysicalMedia"] = {true, true};
mBoolMap["ScrapeFanArt"] = {true, true};
mBoolMap["ScrapeManuals"] = {false, false};
mStringMap["MiximageResolution"] = {"1280x960", "1280x960"};
mStringMap["MiximageScreenshotScaling"] = {"sharp", "sharp"};