Boost.Asio -> libcurl to hopefully fix the Linux bugs.

CMakeLists.txt

@@ -37,6 +37,7 @@ find_package(FreeImage REQUIRED)
 find_package(SDL2 REQUIRED)
 find_package(Boost REQUIRED COMPONENTS system filesystem regex date_time)
 find_package(Eigen3 REQUIRED)
+find_package(CURL REQUIRED)
 
 #add ALSA for Linux
 if(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
@@ -102,6 +103,7 @@ set(ES_INCLUDE_DIRS
     ${SDL2_INCLUDE_DIR}
     ${Boost_INCLUDE_DIRS}
     ${EIGEN3_INCLUDE_DIR}
+    ${CURL_INCLUDE_DIR}
 )
 
 #add ALSA for Linux
@@ -294,6 +296,7 @@ set(ES_LIBRARIES
     ${FreeImage_LIBRARIES}
 	${SDL2_LIBRARY}
     ${SDL2MAIN_LIBRARY}
+    ${CURL_LIBRARIES}
 )
 
 #add ALSA for Linux

README.md

@@ -27,12 +27,12 @@ Building
 EmulationStation uses some C++11 code, which means you'll need to install at least g++-4.7 on Linux, or VS2010 on Windows. 
 For installing and switching to g++-4.7 see [here](http://lektiondestages.blogspot.de/2013/05/installing-and-switching-gccg-versions.html).  You can also just use `export CXX=g++-4.7` to explicitly specify the compiler for CMake (make sure you delete your CMake cache files if it's not working).
 
-EmulationStation has a few dependencies. For building, you'll need SDL2, Boost.System, Boost.Filesystem, Boost.Asio, Boost.Regex, Boost.DateTime, FreeImage, FreeType, and Eigen3.  You'll also need the DejaVu TrueType font on Linux to run ES.
+EmulationStation has a few dependencies. For building, you'll need SDL2, Boost (System, Filesystem, Regex, DateTime), FreeImage, FreeType, Eigen3, and cURL.  You'll also need the DejaVu TrueType font on Linux to run ES.
 
 **On Linux:**
 All of this be easily installed with apt-get:
 ```
-sudo apt-get install libsdl2-dev libboost-dev libboost-system-dev libboost-filesystem-dev libboost-regex-dev libboost-date-time-dev libfreeimage-dev libfreetype6-dev libeigen3-dev ttf-dejavu libasound2-dev
+sudo apt-get install libsdl2-dev libboost-dev libboost-system-dev libboost-filesystem-dev libboost-regex-dev libboost-date-time-dev libfreeimage-dev libfreetype6-dev libeigen3-dev libcurl-dev ttf-dejavu libasound2-dev
 ```
 
 Unless you're on the Raspberry Pi, you'll also need OpenGL:
@@ -62,6 +62,8 @@ make
 
 [SDL2](http://www.libsdl.org/release/SDL2-devel-2.0.0-VC.zip)
 
+[CURL](http://curl.haxx.se/download.html) (you'll need to compile or get the pre-compiled (DLL version))
+
 (remember to copy necessary .DLLs into the same folder as the executable: FreeImage.dll, freetype6.dll, SDL2.dll, and zlib1.dll)
 
 [CMake](http://www.cmake.org/cmake/resources/software.html) (this is used for generating the Visual Studio project)

src/HttpReq.cpp

@@ -4,7 +4,9 @@
 #include "Log.h"
 #include <boost/filesystem.hpp>
 
-boost::asio::io_service HttpReq::io_service;
+CURLM* HttpReq::s_multi_handle = curl_multi_init();
+
+std::map<CURL*, HttpReq*> HttpReq::s_requests;
 
 std::string HttpReq::urlEncode(const std::string &s)
 {
@@ -35,195 +37,108 @@ bool HttpReq::isUrl(const std::string& str)
 		(str.find("http://") != std::string::npos || str.find("https://") != std::string::npos || str.find("www.") != std::string::npos));
 }
 
-HttpReq::HttpReq(const std::string& server, const std::string& path)
-	: mResolver(io_service), mSocket(io_service), mStatus(REQ_IN_PROGRESS)
-{
-	start(server, path);
-}
-
 HttpReq::HttpReq(const std::string& url)
-	: mResolver(io_service), mSocket(io_service), mStatus(REQ_IN_PROGRESS)
+	: mStatus(REQ_IN_PROGRESS), mHandle(NULL)
 {
-	size_t startpos = 0;
+	mHandle = curl_easy_init();
 
-	if(url.substr(startpos, 7) == "http://")
+	if(mHandle == NULL)
-		startpos = 7;
-	else if(url.substr(0, 8) == "https://")
-		startpos = 8;
-
-	size_t pathStart = url.find('/', startpos);
-
-	std::string server, path;
-	if(pathStart == std::string::npos)
 	{
-		server = url;
+		mStatus = REQ_IO_ERROR;
-		path = "/";
+		onError("curl_easy_init failed");
-	}else{
+		return;
-		server = url.substr(startpos, pathStart - startpos);
-		path = url.substr(pathStart, std::string::npos);
 	}
-	
+
-	start(server, path);
+	//set the url
+	CURLcode err = curl_easy_setopt(mHandle, CURLOPT_URL, url.c_str());
+	if(err != CURLE_OK)
+	{
+		mStatus = REQ_IO_ERROR;
+		onError(curl_easy_strerror(err));
+		return;
+	}
+
+	//tell curl how to write the data
+	err = curl_easy_setopt(mHandle, CURLOPT_WRITEFUNCTION, &HttpReq::write_content);
+	if(err != CURLE_OK)
+	{
+		mStatus = REQ_IO_ERROR;
+		onError(curl_easy_strerror(err));
+		return;
+	}
+
+	//give curl a pointer to this HttpReq so we know where to write the data *to* in our write function
+	err = curl_easy_setopt(mHandle, CURLOPT_WRITEDATA, this);
+	if(err != CURLE_OK)
+	{
+		mStatus = REQ_IO_ERROR;
+		onError(curl_easy_strerror(err));
+		return;
+	}
+
+	//add the handle to our multi
+	CURLMcode merr = curl_multi_add_handle(s_multi_handle, mHandle);
+	if(merr != CURLM_OK)
+	{
+		mStatus = REQ_IO_ERROR;
+		onError(curl_multi_strerror(merr));
+		return;
+	}
+
+	s_requests[mHandle] = this;
 }
 
 HttpReq::~HttpReq()
 {
-	mResolver.cancel();
+	if(mHandle)
-	mSocket.close();
-	//status(); //poll once
-	while(status() == REQ_IN_PROGRESS); //otherwise you get really weird heap-allocation-related crashes
-}
-
-void HttpReq::start(const std::string& server, const std::string& path)
-{
-	std::ostream req_str(&mRequest);
-	req_str << "GET " << path << " HTTP/1.0\r\n";
-	req_str << "Host: " << server << "\r\n";
-	req_str << "Accept: */*\r\n";
-	req_str << "Connection: close\r\n\r\n";
-
-	tcp::resolver::query query(server, "http");
-	mResolver.async_resolve(query,
-		boost::bind(&HttpReq::handleResolve, this, 
-			boost::asio::placeholders::error,
-			boost::asio::placeholders::iterator));
-}
-
-void HttpReq::handleResolve(const boost::system::error_code& err, tcp::resolver::iterator endpoint_iterator)
-{
-	if (!err)
 	{
-		// Attempt a connection to each endpoint in the list until we
+		s_requests.erase(mHandle);
-		// successfully establish a connection.
-		boost::asio::async_connect(mSocket, endpoint_iterator,
-			boost::bind(&HttpReq::handleConnect, this,
-				boost::asio::placeholders::error));
-	}
-	else
-	{
-		onError(err);
-	}
-}
 
-void HttpReq::handleConnect(const boost::system::error_code& err)
+		CURLMcode merr = curl_multi_remove_handle(s_multi_handle, mHandle);
-{
-	if (!err)
-	{
-		// The connection was successful. Send the request.
-		boost::asio::async_write(mSocket, mRequest,
-			boost::bind(&HttpReq::handleWriteRequest, this,
-			boost::asio::placeholders::error));
-	}
-	else
-	{
-		onError(err);
-	}
-}
 
-void HttpReq::handleWriteRequest(const boost::system::error_code& err)
+		if(merr != CURLM_OK)
-{
+			LOG(LogError) << "Error removing curl_easy handle from curl_multi: " << curl_multi_strerror(merr);
-	if (!err)
-	{
-		// Read the response status line. The response_ streambuf will
-		// automatically grow to accommodate the entire line. The growth may be
-		// limited by passing a maximum size to the streambuf constructor.
-		boost::asio::async_read_until(mSocket, mResponse, "\r\n",
-			boost::bind(&HttpReq::handleReadStatusLine, this,
-			boost::asio::placeholders::error));
-	}
-	else
-	{
-		onError(err);
-	}
-}
 
-void HttpReq::handleReadStatusLine(const boost::system::error_code& err)
+		curl_easy_cleanup(mHandle);
-{
-	if (!err)
-	{
-		// Check that response is OK.
-		std::istream response_stream(&mResponse);
-		std::string http_version;
-		response_stream >> http_version;
-		response_stream >> mResponseStatusCode;
-		std::string status_message;
-		std::getline(response_stream, status_message);
-		if(!response_stream || http_version.substr(0, 5) != "HTTP/")
-		{
-			mStatus = REQ_INVALID_RESPONSE;
-			return;
-		}
-		if(mResponseStatusCode != 200)
-		{
-			mStatus = REQ_BAD_STATUS_CODE;
-			return;
-		}
-
-		// Read the response headers, which are terminated by a blank line.
-		boost::asio::async_read_until(mSocket, mResponse, "\r\n\r\n",
-			boost::bind(&HttpReq::handleReadHeaders, this,
-				boost::asio::placeholders::error));
-	}
-	else
-	{
-		onError(err);
-	}
-}
-
-void HttpReq::handleReadHeaders(const boost::system::error_code& err)
-{
-	if (!err)
-	{
-		// Process the response headers.
-		std::istream response_stream(&mResponse);
-		std::string header;
-		while (std::getline(response_stream, header) && header != "\r"); //and by process we mean ignore
-
-		// Write whatever content we already have to output.
-		if (mResponse.size() > 0)
-			mContent << &mResponse;
-		
-		// Start reading remaining data until EOF.
-		boost::asio::async_read(mSocket, mResponse,
-			boost::asio::transfer_at_least(1),
-			boost::bind(&HttpReq::handleReadContent, this,
-				boost::asio::placeholders::error));
-	}
-	else
-	{
-		onError(err);
-	}
-}
-
-void HttpReq::handleReadContent(const boost::system::error_code& err)
-{
-	if (!err)
-	{
-		// Write all of the data that has been read so far.
-		mContent << &mResponse;
-
-		// Continue reading remaining data until EOF.
-		boost::asio::async_read(mSocket, mResponse,
-			boost::asio::transfer_at_least(1),
-			boost::bind(&HttpReq::handleReadContent, this,
-				boost::asio::placeholders::error));
-	}else{
-		if (err != boost::asio::error::eof)
-		{
-			onError(err);
-		}else{
-			mStatus = REQ_SUCCESS;
-		}
 	}
 }
 
 HttpReq::Status HttpReq::status()
 {
-	try {
+	if(mStatus == REQ_IN_PROGRESS)
-		io_service.poll();
-	} catch(boost::system::system_error& e)
 	{
-		LOG(LogError) << "Boost.ASIO system error!  " << e.what();
+		int handle_count;
+		CURLMcode merr = curl_multi_perform(s_multi_handle, &handle_count);
+		if(merr != CURLM_OK && merr != CURLM_CALL_MULTI_PERFORM)
+		{
+			mStatus = REQ_IO_ERROR;
+			onError(curl_multi_strerror(merr));
+			return mStatus;
+		}
+
+		int msgs_left;
+		CURLMsg* msg;
+		while(msg = curl_multi_info_read(s_multi_handle, &msgs_left))
+		{
+			if(msg->msg == CURLMSG_DONE)
+			{
+				HttpReq* req = s_requests[msg->easy_handle];
+				
+				if(req == NULL)
+				{
+					LOG(LogError) << "Cannot find easy handle!";
+					continue;
+				}
+
+				if(msg->data.result == CURLE_OK)
+				{
+					req->mStatus = REQ_SUCCESS;
+				}else{
+					req->mStatus = REQ_IO_ERROR;
+					req->onError(curl_easy_strerror(msg->data.result));
+				}
+			}
+		}
 	}
 
 	return mStatus;
@@ -233,35 +148,36 @@ std::string HttpReq::getContent()
 {
 	if(mStatus != REQ_SUCCESS)
 	{
-		LOG(LogError) << "Called getContent() on an unsuccessful HttpReq!";
+		LOG(LogError) << "Called getContent() on an incomplete HttpReq!";
 		return "";
 	}
 
 	return mContent.str();
 }
 
-//only called for boost-level errors (REQ_IO_ERROR)
+void HttpReq::onError(const char* msg)
-void HttpReq::onError(const boost::system::error_code& err)
 {
-	mError = err;
+	mErrorMsg = msg;
-	mStatus = REQ_IO_ERROR;
 }
 
 std::string HttpReq::getErrorMsg()
 {
-	switch(mStatus)
+	return mErrorMsg;
-	{
-	case REQ_BAD_STATUS_CODE:
-		return "Bad status code";
-	case REQ_INVALID_RESPONSE:
-		return "Invalid response from server";
-	case REQ_IO_ERROR:
-		return mError.message();
-	case REQ_IN_PROGRESS:
-		return "Not done yet";
-	case REQ_SUCCESS:
-		return "No error";
-	default:
-		return "???";
-	}
 }
+
+//used as a curl callback
+//size = size of an element, nmemb = number of elements
+//return value is number of elements successfully read
+size_t HttpReq::write_content(void* buff, size_t size, size_t nmemb, void* req_ptr)
+{
+	std::stringstream& ss = ((HttpReq*)req_ptr)->mContent;
+	ss.write((char*)buff, size * nmemb);
+
+	return nmemb;
+}
+
+//used as a curl callback
+/*int HttpReq::update_progress(void* req_ptr, double dlTotal, double dlNow, double ulTotal, double ulNow)
+{
+	
+}*/

src/HttpReq.h

@@ -1,10 +1,8 @@
 #pragma once
 
-#include <boost/asio.hpp>
+#include <curl/curl.h>
-
+#include <sstream>
-using boost::asio::ip::tcp;
+#include <map>
-
-//Based on: http://www.boost.org/doc/libs/1_51_0/doc/html/boost_asio/example/http/client/async_client.cpp
 
 /* Usage:
  * HttpReq myRequest("www.google.com", "/index.html");
@@ -26,7 +24,6 @@ using boost::asio::ip::tcp;
 class HttpReq
 {
 public:
-	HttpReq(const std::string& server, const std::string& path);
 	HttpReq(const std::string& url);
 
 	~HttpReq();
@@ -51,25 +48,21 @@ public:
 	static bool isUrl(const std::string& s);
 
 private:
-	static boost::asio::io_service io_service;
+	static size_t write_content(void* buff, size_t size, size_t nmemb, void* req_ptr);
+	//static int update_progress(void* req_ptr, double dlTotal, double dlNow, double ulTotal, double ulNow);
 
-	void start(const std::string& server, const std::string& path);
+	//god dammit libcurl why can't you have some way to check the status of an individual handle
-	void handleResolve(const boost::system::error_code& err, tcp::resolver::iterator endpoint_iterator);
+	//why do I have to handle ALL messages at once
-	void handleConnect(const boost::system::error_code& err);
+	static std::map<CURL*, HttpReq*> s_requests;
-	void handleWriteRequest(const boost::system::error_code& err);
-	void handleReadStatusLine(const boost::system::error_code& err);
-	void handleReadHeaders(const boost::system::error_code& err);
-	void handleReadContent(const boost::system::error_code& err);
 
-	void onError(const boost::system::error_code& error);
+	static CURLM* s_multi_handle;
 
-	tcp::resolver mResolver;
+	void onError(const char* msg);
-	tcp::socket mSocket;
+
-	boost::asio::streambuf mRequest;
+	CURL* mHandle;
-	boost::asio::streambuf mResponse;
 
 	Status mStatus;
+
 	std::stringstream mContent;
-	unsigned int mResponseStatusCode;
+	std::string mErrorMsg;
-	boost::system::error_code mError;
 };

src/scrapers/GamesDBScraper.cpp

@@ -70,7 +70,7 @@ std::shared_ptr<HttpReq> GamesDBScraper::makeHttpReq(ScraperSearchParams params)
 		path += HttpReq::urlEncode(gamesdb_platformid_map.at(params.system->getPlatformId()));
 	}
 
-	return std::make_shared<HttpReq>("thegamesdb.net", path);
+	return std::make_shared<HttpReq>("thegamesdb.net" + path);
 }
 
 std::vector<MetaDataList> GamesDBScraper::parseReq(ScraperSearchParams params, std::shared_ptr<HttpReq> req)

src/scrapers/TheArchiveScraper.cpp

@@ -17,7 +17,7 @@ std::shared_ptr<HttpReq> TheArchiveScraper::makeHttpReq(ScraperSearchParams para
 	path += HttpReq::urlEncode(cleanName);
 	//platform TODO, should use some params.system get method
 
-	return std::make_shared<HttpReq>("api.archive.vg", path);
+	return std::make_shared<HttpReq>("api.archive.vg" + path);
 }
 
 std::vector<MetaDataList> TheArchiveScraper::parseReq(ScraperSearchParams params, std::shared_ptr<HttpReq> req)