Xargon/ES-DE
1
0
Fork 0
mirror of https://github.com/RetroDECK/ES-DE.git synced 2024-11-22 06:05:38 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Boost.Asio -> libcurl to hopefully fix the Linux bugs.

Browse source
This commit is contained in:
Aloshi 2013-10-10 13:11:01 -05:00
parent 4e2b57c001
commit dca5467f7b
6 changed files with 130 additions and 216 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
CMakeLists.txt
View file

@ -37,6 +37,7 @@ find_package(FreeImage REQUIRED)
find_package(SDL2 REQUIRED)
find_package(Boost REQUIRED COMPONENTS system filesystem regex date_time)
find_package(Eigen3 REQUIRED)
find_package(CURL REQUIRED)
#add ALSA for Linux
if(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
@ -102,6 +103,7 @@ set(ES_INCLUDE_DIRS
${SDL2_INCLUDE_DIR}
${Boost_INCLUDE_DIRS}
${EIGEN3_INCLUDE_DIR}
${CURL_INCLUDE_DIR}
)
#add ALSA for Linux
@ -294,6 +296,7 @@ set(ES_LIBRARIES
${FreeImage_LIBRARIES}
${SDL2_LIBRARY}
${SDL2MAIN_LIBRARY}
${CURL_LIBRARIES}
)
#add ALSA for Linux

6
README.md
View file

@ -27,12 +27,12 @@ Building
EmulationStation uses some C++11 code, which means you'll need to install at least g++-4.7 on Linux, or VS2010 on Windows.
For installing and switching to g++-4.7 see [here](http://lektiondestages.blogspot.de/2013/05/installing-and-switching-gccg-versions.html). You can also just use `export CXX=g++-4.7` to explicitly specify the compiler for CMake (make sure you delete your CMake cache files if it's not working).
EmulationStation has a few dependencies. For building, you'll need SDL2, Boost.System, Boost.Filesystem, Boost.Asio, Boost.Regex, Boost.DateTime, FreeImage, FreeType, and Eigen3. You'll also need the DejaVu TrueType font on Linux to run ES.
EmulationStation has a few dependencies. For building, you'll need SDL2, Boost (System, Filesystem, Regex, DateTime), FreeImage, FreeType, Eigen3, and cURL. You'll also need the DejaVu TrueType font on Linux to run ES.
**On Linux:**
All of this be easily installed with apt-get:
```
sudo apt-get install libsdl2-dev libboost-dev libboost-system-dev libboost-filesystem-dev libboost-regex-dev libboost-date-time-dev libfreeimage-dev libfreetype6-dev libeigen3-dev ttf-dejavu libasound2-dev
sudo apt-get install libsdl2-dev libboost-dev libboost-system-dev libboost-filesystem-dev libboost-regex-dev libboost-date-time-dev libfreeimage-dev libfreetype6-dev libeigen3-dev libcurl-dev ttf-dejavu libasound2-dev
```
Unless you're on the Raspberry Pi, you'll also need OpenGL:
@ -62,6 +62,8 @@ make
[SDL2](http://www.libsdl.org/release/SDL2-devel-2.0.0-VC.zip)
[CURL](http://curl.haxx.se/download.html) (you'll need to compile or get the pre-compiled (DLL version))
(remember to copy necessary .DLLs into the same folder as the executable: FreeImage.dll, freetype6.dll, SDL2.dll, and zlib1.dll)
[CMake](http://www.cmake.org/cmake/resources/software.html) (this is used for generating the Visual Studio project)

296
src/HttpReq.cpp
View file

@ -4,7 +4,9 @@
#include "Log.h"
#include <boost/filesystem.hpp>
boost::asio::io_service HttpReq::io_service;
CURLM* HttpReq::s_multi_handle = curl_multi_init();
std::map<CURL*, HttpReq*> HttpReq::s_requests;
std::string HttpReq::urlEncode(const std::string &s)
{
@ -35,195 +37,108 @@ bool HttpReq::isUrl(const std::string& str)
(str.find("http://") != std::string::npos || str.find("https://") != std::string::npos || str.find("www.") != std::string::npos));
}
HttpReq::HttpReq(const std::string& server, const std::string& path)
: mResolver(io_service), mSocket(io_service), mStatus(REQ_IN_PROGRESS)
{
start(server, path);
}
HttpReq::HttpReq(const std::string& url)
: mResolver(io_service), mSocket(io_service), mStatus(REQ_IN_PROGRESS)
: mStatus(REQ_IN_PROGRESS), mHandle(NULL)
{
size_t startpos = 0;
mHandle = curl_easy_init();
if(url.substr(startpos, 7) == "http://")
startpos = 7;
else if(url.substr(0, 8) == "https://")
startpos = 8;
size_t pathStart = url.find('/', startpos);
std::string server, path;
if(pathStart == std::string::npos)
if(mHandle == NULL)
{
server = url;
path = "/";
}else{
server = url.substr(startpos, pathStart - startpos);
path = url.substr(pathStart, std::string::npos);
mStatus = REQ_IO_ERROR;
onError("curl_easy_init failed");
return;
}
start(server, path);
//set the url
CURLcode err = curl_easy_setopt(mHandle, CURLOPT_URL, url.c_str());
if(err != CURLE_OK)
{
mStatus = REQ_IO_ERROR;
onError(curl_easy_strerror(err));
return;
}
//tell curl how to write the data
err = curl_easy_setopt(mHandle, CURLOPT_WRITEFUNCTION, &HttpReq::write_content);
if(err != CURLE_OK)
{
mStatus = REQ_IO_ERROR;
onError(curl_easy_strerror(err));
return;
}
//give curl a pointer to this HttpReq so we know where to write the data *to* in our write function
err = curl_easy_setopt(mHandle, CURLOPT_WRITEDATA, this);
if(err != CURLE_OK)
{
mStatus = REQ_IO_ERROR;
onError(curl_easy_strerror(err));
return;
}
//add the handle to our multi
CURLMcode merr = curl_multi_add_handle(s_multi_handle, mHandle);
if(merr != CURLM_OK)
{
mStatus = REQ_IO_ERROR;
onError(curl_multi_strerror(merr));
return;
}
s_requests[mHandle] = this;
}
HttpReq::~HttpReq()
{
mResolver.cancel();
mSocket.close();
//status(); //poll once
while(status() == REQ_IN_PROGRESS); //otherwise you get really weird heap-allocation-related crashes
}
void HttpReq::start(const std::string& server, const std::string& path)
{
std::ostream req_str(&mRequest);
req_str << "GET " << path << " HTTP/1.0\r\n";
req_str << "Host: " << server << "\r\n";
req_str << "Accept: */*\r\n";
req_str << "Connection: close\r\n\r\n";
tcp::resolver::query query(server, "http");
mResolver.async_resolve(query,
boost::bind(&HttpReq::handleResolve, this,
boost::asio::placeholders::error,
boost::asio::placeholders::iterator));
}
void HttpReq::handleResolve(const boost::system::error_code& err, tcp::resolver::iterator endpoint_iterator)
{
if (!err)
if(mHandle)
{
// Attempt a connection to each endpoint in the list until we
// successfully establish a connection.
boost::asio::async_connect(mSocket, endpoint_iterator,
boost::bind(&HttpReq::handleConnect, this,
boost::asio::placeholders::error));
}
else
{
onError(err);
}
}
s_requests.erase(mHandle);
void HttpReq::handleConnect(const boost::system::error_code& err)
{
if (!err)
{
// The connection was successful. Send the request.
boost::asio::async_write(mSocket, mRequest,
boost::bind(&HttpReq::handleWriteRequest, this,
boost::asio::placeholders::error));
}
else
{
onError(err);
}
}
CURLMcode merr = curl_multi_remove_handle(s_multi_handle, mHandle);
void HttpReq::handleWriteRequest(const boost::system::error_code& err)
{
if (!err)
{
// Read the response status line. The response_ streambuf will
// automatically grow to accommodate the entire line. The growth may be
// limited by passing a maximum size to the streambuf constructor.
boost::asio::async_read_until(mSocket, mResponse, "\r\n",
boost::bind(&HttpReq::handleReadStatusLine, this,
boost::asio::placeholders::error));
}
else
{
onError(err);
}
}
if(merr != CURLM_OK)
LOG(LogError) << "Error removing curl_easy handle from curl_multi: " << curl_multi_strerror(merr);
void HttpReq::handleReadStatusLine(const boost::system::error_code& err)
{
if (!err)
{
// Check that response is OK.
std::istream response_stream(&mResponse);
std::string http_version;
response_stream >> http_version;
response_stream >> mResponseStatusCode;
std::string status_message;
std::getline(response_stream, status_message);
if(!response_stream || http_version.substr(0, 5) != "HTTP/")
{
mStatus = REQ_INVALID_RESPONSE;
return;
}
if(mResponseStatusCode != 200)
{
mStatus = REQ_BAD_STATUS_CODE;
return;
}
// Read the response headers, which are terminated by a blank line.
boost::asio::async_read_until(mSocket, mResponse, "\r\n\r\n",
boost::bind(&HttpReq::handleReadHeaders, this,
boost::asio::placeholders::error));
}
else
{
onError(err);
}
}
void HttpReq::handleReadHeaders(const boost::system::error_code& err)
{
if (!err)
{
// Process the response headers.
std::istream response_stream(&mResponse);
std::string header;
while (std::getline(response_stream, header) && header != "\r"); //and by process we mean ignore
// Write whatever content we already have to output.
if (mResponse.size() > 0)
mContent << &mResponse;
// Start reading remaining data until EOF.
boost::asio::async_read(mSocket, mResponse,
boost::asio::transfer_at_least(1),
boost::bind(&HttpReq::handleReadContent, this,
boost::asio::placeholders::error));
}
else
{
onError(err);
}
}
void HttpReq::handleReadContent(const boost::system::error_code& err)
{
if (!err)
{
// Write all of the data that has been read so far.
mContent << &mResponse;
// Continue reading remaining data until EOF.
boost::asio::async_read(mSocket, mResponse,
boost::asio::transfer_at_least(1),
boost::bind(&HttpReq::handleReadContent, this,
boost::asio::placeholders::error));
}else{
if (err != boost::asio::error::eof)
{
onError(err);
}else{
mStatus = REQ_SUCCESS;
}
curl_easy_cleanup(mHandle);
}
}
HttpReq::Status HttpReq::status()
{
try {
io_service.poll();
} catch(boost::system::system_error& e)
if(mStatus == REQ_IN_PROGRESS)
{
LOG(LogError) << "Boost.ASIO system error! " << e.what();
int handle_count;
CURLMcode merr = curl_multi_perform(s_multi_handle, &handle_count);
if(merr != CURLM_OK && merr != CURLM_CALL_MULTI_PERFORM)
{
mStatus = REQ_IO_ERROR;
onError(curl_multi_strerror(merr));
return mStatus;
}
int msgs_left;
CURLMsg* msg;
while(msg = curl_multi_info_read(s_multi_handle, &msgs_left))
{
if(msg->msg == CURLMSG_DONE)
{
HttpReq* req = s_requests[msg->easy_handle];
if(req == NULL)
{
LOG(LogError) << "Cannot find easy handle!";
continue;
}
if(msg->data.result == CURLE_OK)
{
req->mStatus = REQ_SUCCESS;
}else{
req->mStatus = REQ_IO_ERROR;
req->onError(curl_easy_strerror(msg->data.result));
}
}
}
}
return mStatus;
@ -233,35 +148,36 @@ std::string HttpReq::getContent()
{
if(mStatus != REQ_SUCCESS)
{
LOG(LogError) << "Called getContent() on an unsuccessful HttpReq!";
LOG(LogError) << "Called getContent() on an incomplete HttpReq!";
return "";
}
return mContent.str();
}
//only called for boost-level errors (REQ_IO_ERROR)
void HttpReq::onError(const boost::system::error_code& err)
void HttpReq::onError(const char* msg)
{
mError = err;
mStatus = REQ_IO_ERROR;
mErrorMsg = msg;
}
std::string HttpReq::getErrorMsg()
{
switch(mStatus)
{
case REQ_BAD_STATUS_CODE:
return "Bad status code";
case REQ_INVALID_RESPONSE:
return "Invalid response from server";
case REQ_IO_ERROR:
return mError.message();
case REQ_IN_PROGRESS:
return "Not done yet";
case REQ_SUCCESS:
return "No error";
default:
return "???";
}
return mErrorMsg;
}
//used as a curl callback
//size = size of an element, nmemb = number of elements
//return value is number of elements successfully read
size_t HttpReq::write_content(void* buff, size_t size, size_t nmemb, void* req_ptr)
{
std::stringstream& ss = ((HttpReq*)req_ptr)->mContent;
ss.write((char*)buff, size * nmemb);
return nmemb;
}
//used as a curl callback
/*int HttpReq::update_progress(void* req_ptr, double dlTotal, double dlNow, double ulTotal, double ulNow)
{
}*/

35
src/HttpReq.h
View file

@ -1,10 +1,8 @@
#pragma once
#include <boost/asio.hpp>
using boost::asio::ip::tcp;
//Based on: http://www.boost.org/doc/libs/1_51_0/doc/html/boost_asio/example/http/client/async_client.cpp
#include <curl/curl.h>
#include <sstream>
#include <map>
/* Usage:
* HttpReq myRequest("www.google.com", "/index.html");
@ -26,7 +24,6 @@ using boost::asio::ip::tcp;
class HttpReq
{
public:
HttpReq(const std::string& server, const std::string& path);
HttpReq(const std::string& url);
~HttpReq();
@ -51,25 +48,21 @@ public:
static bool isUrl(const std::string& s);
private:
static boost::asio::io_service io_service;
static size_t write_content(void* buff, size_t size, size_t nmemb, void* req_ptr);
//static int update_progress(void* req_ptr, double dlTotal, double dlNow, double ulTotal, double ulNow);
void start(const std::string& server, const std::string& path);
void handleResolve(const boost::system::error_code& err, tcp::resolver::iterator endpoint_iterator);
void handleConnect(const boost::system::error_code& err);
void handleWriteRequest(const boost::system::error_code& err);
void handleReadStatusLine(const boost::system::error_code& err);
void handleReadHeaders(const boost::system::error_code& err);
void handleReadContent(const boost::system::error_code& err);
//god dammit libcurl why can't you have some way to check the status of an individual handle
//why do I have to handle ALL messages at once
static std::map<CURL*, HttpReq*> s_requests;
void onError(const boost::system::error_code& error);
static CURLM* s_multi_handle;
tcp::resolver mResolver;
tcp::socket mSocket;
boost::asio::streambuf mRequest;
boost::asio::streambuf mResponse;
void onError(const char* msg);
CURL* mHandle;
Status mStatus;
std::stringstream mContent;
unsigned int mResponseStatusCode;
boost::system::error_code mError;
std::string mErrorMsg;
};

2
src/scrapers/GamesDBScraper.cpp
View file

@ -70,7 +70,7 @@ std::shared_ptr<HttpReq> GamesDBScraper::makeHttpReq(ScraperSearchParams params)
path += HttpReq::urlEncode(gamesdb_platformid_map.at(params.system->getPlatformId()));
}
return std::make_shared<HttpReq>("thegamesdb.net", path);
return std::make_shared<HttpReq>("thegamesdb.net" + path);
}
std::vector<MetaDataList> GamesDBScraper::parseReq(ScraperSearchParams params, std::shared_ptr<HttpReq> req)

2
src/scrapers/TheArchiveScraper.cpp
View file

@ -17,7 +17,7 @@ std::shared_ptr<HttpReq> TheArchiveScraper::makeHttpReq(ScraperSearchParams para
path += HttpReq::urlEncode(cleanName);
//platform TODO, should use some params.system get method
return std::make_shared<HttpReq>("api.archive.vg", path);
return std::make_shared<HttpReq>("api.archive.vg" + path);
}
std::vector<MetaDataList> TheArchiveScraper::parseReq(ScraperSearchParams params, std::shared_ptr<HttpReq> req)