Simplify writing to the buffers during download

src/core/hle/service/http/http.cpp

@@ -2,6 +2,7 @@
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
+#include <chrono>
 #include <curl/curl.h>
 
 #include "common/string_util.h"
@@ -16,79 +17,104 @@ namespace HTTP {
 std::unordered_map<ContextHandle, std::unique_ptr<HttpContext>> context_map;
 ContextHandle next_handle;
 
-static int WriteBuf(u8 *data, size_t size, size_t nmemb, std::mutex *mutex, std::vector<u8> *buffer) {
+static int BufWriter(u8 *data, size_t size, size_t nmemb, std::vector<u8>* out_buffer) {
-    std::lock_guard<std::mutex> lock(*mutex);
+    if (out_buffer == nullptr)
-    buffer->reserve(buffer->size() + size * nmemb);
+        return -1;
-    buffer->insert(buffer->end(), data, data + size * nmemb);
+    out_buffer->insert(out_buffer->end(), data, data + size * nmemb);
     return static_cast<int>(size * nmemb);
 }
 
-static int HdrWriter(u8 *data, size_t size, size_t nmemb, HttpContext *context) {
-    if (context == nullptr || context->should_quit)
-        return -1;
-    return WriteBuf(data, size, nmemb, &context->mutex, &context->response_hdrs);
-}
-
-static int DataWriter(u8 *data, size_t size, size_t nmemb, HttpContext *context) {
-    if (context == nullptr || context->should_quit)
-        return -1;
-    return WriteBuf(data, size, nmemb, &context->mutex, &context->response_data);
-}
-
 HttpContext::~HttpContext() {
     curl_slist_free_all(request_hdrs);
 }
 
+static CURLcode SetConnectionType(CURL* connection, RequestType type) {
+    switch (type) {
+    case REQUEST_TYPE_GET:
+        return curl_easy_setopt(connection, CURLOPT_HTTPGET, 1);
+    case REQUEST_TYPE_POST:
+    case REQUEST_TYPE_POST_:
+        return curl_easy_setopt(connection, CURLOPT_POST, 1);
+    case REQUEST_TYPE_PUT:
+    case REQUEST_TYPE_PUT_:
+        return curl_easy_setopt(connection, CURLOPT_UPLOAD, 1);
+    case REQUEST_TYPE_DELETE:
+        // TODO
+        break;
+    case REQUEST_TYPE_HEAD:
+        return curl_easy_setopt(connection, CURLOPT_NOBODY, 1);
+    }
+}
+
 void MakeRequest(HttpContext* context) {
+    CURLM* manager = curl_multi_init();
     CURL* connection = curl_easy_init();
+
     CURLcode res;
+    CURLMcode mres;
+
     {
         std::lock_guard<std::mutex> lock(context->mutex);
 
-        context->state = REQUEST_STATE_IN_PROGRESS;
         res = curl_easy_setopt(connection, CURLOPT_URL, context->url.c_str());
-        switch (context->req_type) {
+        res = SetConnectionType(connection, context->req_type);
-            case REQUEST_TYPE_GET:
-                res = curl_easy_setopt(connection, CURLOPT_HTTPGET, 1);
-                break;
-            case REQUEST_TYPE_POST:
-            case REQUEST_TYPE_POST_:
-                res = curl_easy_setopt(connection, CURLOPT_POST, 1);
-                break;
-            case REQUEST_TYPE_PUT:
-            case REQUEST_TYPE_PUT_:
-                res = curl_easy_setopt(connection, CURLOPT_UPLOAD, 1);
-                break;
-            case REQUEST_TYPE_DELETE:
-                // TODO
-                break;
-            case REQUEST_TYPE_HEAD:
-                res = curl_easy_setopt(connection, CURLOPT_NOBODY, 1);
-                break;
-        }
-
-        res = curl_easy_setopt(connection, CURLOPT_HEADERFUNCTION, HdrWriter);
-        res = curl_easy_setopt(connection, CURLOPT_WRITEFUNCTION, DataWriter);
-        res = curl_easy_setopt(connection, CURLOPT_HEADERDATA, context);
-        res = curl_easy_setopt(connection, CURLOPT_WRITEDATA, context);
         res = curl_easy_setopt(connection, CURLOPT_HTTPHEADER, context->request_hdrs);
     }
 
-    res = curl_easy_perform(connection);
+    std::vector<u8> response_hdrs, response_data;
+    res = curl_easy_setopt(connection, CURLOPT_HEADERFUNCTION, BufWriter);
+    res = curl_easy_setopt(connection, CURLOPT_WRITEFUNCTION, BufWriter);
+    res = curl_easy_setopt(connection, CURLOPT_HEADERDATA, &response_hdrs);
+    res = curl_easy_setopt(connection, CURLOPT_WRITEDATA, &response_data);
+
+    mres = curl_multi_add_handle(manager, connection);
+
+    int still_running;
+    int repeats;
+    mres = curl_multi_perform(manager, &still_running);
+
+    do {
+        // Number of file descriptors. If this is zero, nothing happened to
+        // the active connection.
+        int numfds;
+
+        if (curl_multi_wait(manager, NULL, 0, 1000, &numfds) != CURLM_OK)
+            break;
+
+        if (numfds == 0) {
+            if (repeats++ > 1)
+                std::this_thread::sleep_for(std::chrono::milliseconds(25));
+        } else {
+            repeats = 0;
+        }
+
+        {
+            std::lock_guard<std::mutex> lock(context->mutex);
+            if (context->should_quit)
+                break;
+        }
+
+        mres = curl_multi_perform(manager, &still_running);
+    } while (still_running != 0);
+
     {
         std::lock_guard<std::mutex> lock(context->mutex);
 
+        context->response_hdrs = std::move(response_hdrs);
+        context->response_data = std::move(response_data);
         res = curl_easy_getinfo(connection, CURLINFO_RESPONSE_CODE, &context->response_code);
         res = curl_easy_getinfo(connection, CURLINFO_SIZE_DOWNLOAD, &context->downloaded_size);
-        res = curl_easy_getinfo(connection, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &context->download_size);
+        res = curl_easy_getinfo(connection, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &context->content_length);
 
         context->state = REQUEST_STATE_READY;
     }
 
+    curl_multi_remove_handle(manager, connection);
     curl_easy_cleanup(connection);
+    curl_multi_cleanup(manager);
 }
 
-void AddRequestHeader(std::string name, std::string value, curl_slist** hdr_list) {
+void AddRequestHeader(const std::string& name, const std::string& value, curl_slist** hdr_list) {
     std::string header = Common::StringFromFormat("%s: %s", name.c_str(), value.c_str());
     *hdr_list = curl_slist_append(*hdr_list, header.c_str());
 }
@@ -115,5 +141,5 @@ void Shutdown() {
     ClearInstance();
 }
 
-} // namespace HTTP
+}; // namespace HTTP
 } // namespace Service

src/core/hle/service/http/http.h

@@ -50,7 +50,7 @@ struct HttpContext {
     std::vector<u8> response_hdrs;
     std::vector<u8> response_data;
     long response_code;
-    double download_size;
+    double content_length;
     double downloaded_size;
 
     ~HttpContext();
@@ -60,7 +60,7 @@ extern std::unordered_map<ContextHandle, std::unique_ptr<HttpContext>> context_m
 extern ContextHandle next_handle;
 
 void MakeRequest(HttpContext* context);
-void AddRequestHeader(std::string name, std::string value, curl_slist** hdr_list);
+void AddRequestHeader(const std::string& name, const std::string& value, curl_slist** hdr_list);
 
 /// Initialize the HTTP service
 void Init();

src/core/hle/service/http/http_c.cpp

@@ -2,7 +2,6 @@
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
-#include <stdexcept>
 #include <thread>
 
 #include "core/hle/hle.h"
@@ -27,18 +26,15 @@ static void CreateContext(Service::Interface* self) {
 
     u32 url_len = cmd_buff[1];
     RequestType req_type = static_cast<RequestType>(cmd_buff[2]);
-    u32 url_ptr = cmd_buff[4];
+    char* url_ptr = reinterpret_cast<char*>(Memory::GetPointer(cmd_buff[4]));
 
-    std::string url(reinterpret_cast<const char*>(Memory::GetPointer(url_ptr)), url_len);
+    LOG_DEBUG(Service, "request url=%s req_type=%u", url_ptr, req_type);
-
-    LOG_DEBUG(Service, "request url=%s req_type=%u",
-              url.c_str(), req_type);
 
     // TODO: give HttpContext a proper constructor
     std::unique_ptr<HttpContext> context(new HttpContext{});
 
     context->req_type = req_type;
-    context->url = url;
+    context->url = std::string(url_ptr, url_len);
     context_map.emplace(next_handle, std::move(context));
 
     cmd_buff[1] = RESULT_SUCCESS.raw;
@@ -117,8 +113,7 @@ static void GetDownloadSizeState(Service::Interface* self) {
     cmd_buff[1] = RESULT_SUCCESS.raw;
     cmd_buff[2] = (u32) response->downloaded_size;
     // TODO: invalid content-length?
-    cmd_buff[3] = (u32) response->download_size;
+    cmd_buff[3] = (u32) response->content_length;
-    return;
 }
 
 static void BeginRequest(Service::Interface* self) {
@@ -135,6 +130,7 @@ static void BeginRequest(Service::Interface* self) {
     HttpContext* context = map_it->second.get();
     std::lock_guard<std::mutex> lock(context->mutex);
     context->req_thread = Common::make_unique<std::thread>(MakeRequest, context);
+    context->state = REQUEST_STATE_IN_PROGRESS;
 
     cmd_buff[1] = RESULT_SUCCESS.raw;
 }
@@ -169,8 +165,8 @@ static void AddRequestHeader(Service::Interface* self) {
     ContextHandle handle = static_cast<ContextHandle>(cmd_buff[1]);
     u32 hdr_name_len = cmd_buff[2];
     u32 hdr_val_len  = cmd_buff[3];
-    const char* hdr_name_buf = reinterpret_cast<const char*>(Memory::GetPointer(cmd_buff[5]));
+    char* hdr_name_buf = reinterpret_cast<char*>(Memory::GetPointer(cmd_buff[5]));
-    const char* hdr_val_buf = reinterpret_cast<const char*>(Memory::GetPointer(cmd_buff[7]));
+    char* hdr_val_buf = reinterpret_cast<char*>(Memory::GetPointer(cmd_buff[7]));
 
     // TODO: something is NULL
     // TODO: header value is empty