/src/libs/HttpService/HTTPClient/UploadQueue.cpp
C++ | 380 lines | 270 code | 60 blank | 50 comment | 69 complexity | 90fb609d33c556401700d5be05432657 MD5 | raw file
- /**********************************************************\
- Original Author: Dan Weatherford
-
- Imported into FireBreath: Oct 4, 2010
- License: Dual license model; choose one of two:
- New BSD License
- http://www.opensource.org/licenses/bsd-license.php
- - or -
- GNU Lesser General Public License, version 2.1
- http://www.gnu.org/licenses/lgpl-2.1.html
-
- Copyright 2010 Dan Weatherford and Facebook, Inc
- \**********************************************************/
-
- #include "UploadQueue.h"
- #include "HTTPRequest.h"
- #include <boost/algorithm/string.hpp>
- #include "../HTTPCommon/Status.h"
- #include "../HTTPCommon/Utils.h"
- #include "logging.h"
-
- using namespace HTTP;
- using namespace boost::algorithm;
-
- UploadQueue::UploadQueue( const std::string& _name )
- : name(_name), status(UPLOAD_IDLE), current_queue_bytes(0), current_batch_bytes(0), total_queue_bytes(0),
- total_queue_files(0), files_waiting(0), current_upload_request(NULL), current_batch_retry(0),
- batch_size(8), max_retries(3)
- {
-
- }
-
- UploadQueue::~UploadQueue()
- {
-
- }
-
- FB::VariantMap UploadQueue::getEmptyProgressDict() {
- FB::VariantMap d;
- d["queue_name"] = std::string();
- d["current_files"] = FB::VariantList();
- d["current_queue_bytes_remaining"] = 0;
- d["current_queue_files_remaining"] = 0;
- d["total_queue_bytes"] = 0;
- d["total_queue_files"] = 0;
- d["batches_remaining"] = 0;
-
- return d;
- }
-
- FB::VariantMap UploadQueue::getStatusDict(bool include_filenames) {
- FB::VariantMap d;
- // whole-queue stats
- d["queue_name"] = name;
- d["current_queue_bytes_remaining"] = current_queue_bytes;
- d["total_queue_bytes"] = total_queue_bytes;
- d["total_queue_files"] = total_queue_files;
- d["current_queue_files_remaining"] = files_waiting;
- d["batches_remaining"] = ceil(static_cast<float>(files_waiting) / static_cast<float>(batch_size));
-
- // this-batch stats
- if (include_filenames) {
- FB::VariantList cf;
- for (std::set<std::wstring>::const_iterator it = current_upload_files.begin();
- it != current_upload_files.end(); ++it) {
- cf.push_back(*it);
- }
- d["current_files"] = cf;
- }
- d["current_batch_bytes_total"] = current_batch_bytes;
-
- double current_batch_pct_complete = 0.0;
-
- if (current_upload_request) {
- HTTP::Status st = current_upload_request->getStatus();
- FB::VariantMap status(st.asDict());
- d.insert(status.begin(), status.end());
- if (st.send_total) {
- current_batch_pct_complete = static_cast<double>(st.bytes_sent)
- / static_cast<double>(st.send_total);
- }
- }
-
- d["current_batch_pct_complete"] = current_batch_pct_complete;
- d["overall_progress_pct"] = static_cast<double>(
- (total_queue_bytes - (current_queue_bytes + current_batch_bytes))
- + (current_batch_pct_complete * current_batch_bytes))
- / static_cast<double>(total_queue_bytes);
-
- return d;
- }
-
- void UploadQueue::dispatch() {
- status = UploadQueue::UPLOAD_IN_PROGRESS;
- start_next_upload();
-
- // Deliver the first status dict to HTTP server. We need it to be there before this
- // function returns, because otherwise there's a race between the HTTP request posting
- // actual progress data and the JS setting up the polling for it, where JS could see
- // an empty progress dict and think the uploads are all done.
- // TODOTODO
- //{
- // boost::shared_ptr<UploadStatusHandler> ph = http_status_handler_weak.lock();
- // if (ph) ph->postUpdate(getStatusDict());
- //}
- sendUpdateEvent();
- }
-
- void UploadQueue::start_next_upload() {
- unsigned int files_started = 0;
-
- boost::shared_ptr<HTTPRequestData> data(new HTTPRequestData);
- data->method = "POST";
- data->cookies = cookies;
-
- current_upload_files.clear();
- current_batch_bytes = 0;
- current_batch_retry = 0;
-
- for (std::list<UploadQueueEntry>::iterator it = queue.begin(); it != queue.end(); ++it) {
- if (it->status != UploadQueueEntry::ENTRY_WAITING) continue;
- try {
- UploadQueueEntry& qe = *it;
-
- // Open and process the image.
- // We do this now so we can catch errors and attribute them to individual images;
- // letting the HTTPRequest do it would fail the whole request if a single image
- // was unreadable.
- qe.datablock->resolve();
-
- // Pull in post params
- for (std::map<std::string, std::string>::iterator pvit = post_vars.begin();
- pvit != post_vars.end(); ++pvit) {
- qe.target.query_data[pvit->first] = pvit->second;
- }
-
- if (files_started == 0) {
- // First match sets the target uri
- data->uri = qe.target;
- } else {
- // Subsequently, we can only batch up more uploads to the same endpoint
- if (!(data->uri == qe.target)) continue;
- }
-
- it->setStatus(UploadQueueEntry::ENTRY_IN_PROGRESS);
- --files_waiting;
- current_queue_bytes -= qe.filesize;
-
- std::string s = "file" + files_started;
- qe.post_field = s;
- data->addFile(s, FB::wstring_to_utf8(qe.filename), "application/octet-stream", qe.datablock);
-
- current_upload_files.insert(qe.source_path);
- current_batch_bytes += qe.filesize;
- ++files_started;
- if (files_started >= batch_size) break;
-
- } catch (const std::exception& e) {
- it->result = e.what();
- it->setStatus(UploadQueueEntry::ENTRY_ERROR);
- }
- }
-
- if (files_started) {
- // We had enough images left in the queue for another request, so kick that off now.
- current_upload_request = HTTPRequest::create();
- current_upload_request->onStatusChanged(
- bind(&UploadQueue::upload_request_status_changed, this, _1)
- );
- current_upload_request->startRequest(data);
-
- // As long as we're doing uploads, we want to keep the HTTP server
- // up to provide progress to the chat bar widget -- so enable deferred shutdown.
- // TODOTODO
- //try {
- // boost::shared_ptr<HTTPService> h = http_srv_inst_weak.lock();
- // if (h) h->setDeferShutdown(true);
- //} catch (...) {}
-
- } else {
- #ifndef NDEBUG
- FBLOG_DEBUG("UploadQueue",
- "start_next_upload() found no waiting files, running completion handlers\n");
- #endif
- // All done, post upload finished callback to all instances
- FB::VariantMap d = getEmptyProgressDict();
- FB::VariantMap failures;
-
- for (std::list<UploadQueueEntry>::iterator it = queue.begin(); it != queue.end(); ++it) {
- if (it->status == UploadQueueEntry::ENTRY_ERROR) {
- failures[FB::wstring_to_utf8(it->source_path)] = it->result;
- #ifndef NDEBUG
- FBLOG_WARN("UploadQueue", "Reporting file \"" << it->source_path.c_str()
- << "\" as failed: " << it->result.c_str() << std::endl);
- #endif
- }
- }
- queue.clear();
-
- if (! failures.empty()) d["failed_files"] = failures;
-
- // TODOTODO
- {
- StatusUpdateEvent evt(d);
- SendEvent(&evt);
- }
-
- status = UploadQueue::UPLOAD_COMPLETE;
- // fire completion handlers, if available
- for (std::list<FB::URI>::iterator it = completion_handlers.begin();
- it != completion_handlers.end(); ++it) {
- boost::shared_ptr<HTTPRequestData> reqdata(new HTTPRequestData(*it));
- reqdata->cookies = cookies;
- HTTPRequest::asyncStartRequest(reqdata);
- }
-
- if (queue_finished_callback) queue_finished_callback(shared_ptr());
- }
- }
-
- void UploadQueue::upload_request_status_changed(const HTTP::Status &status) {
- // Deliver status dict to interested observers
- sendUpdateEvent();
-
- uint64_t this_batch_size = 0;
- if (status.state == HTTP::Status::HTTP_ERROR) {
- if ((++current_batch_retry) < max_retries) {
- // Retry on networking error.
- #ifndef NDEBUG
- FBLOG_WARN("UploadQueue", "Retrying current batch on networking error ("
- << status.last_error.c_str() << ")" << std::endl);
- #endif
- // Retake ownership of the request data, dispose this request, and start a new one.
- // The request data doesn't get mangled by the HTTPRequest object, it should be safe
- // to reuse.
- boost::shared_ptr<HTTPRequestData> req_data = current_upload_request->getRequest();
- current_upload_request->threadSafeDestroy();
-
- current_upload_request = HTTPRequest::create();
- current_upload_request->onStatusChanged(
- bind(&UploadQueue::upload_request_status_changed, this, _1)
- );
- current_upload_request->startRequest(req_data);
- return;
- }
- for (std::list<UploadQueueEntry>::iterator it = queue.begin(); it != queue.end(); ++it) {
- if (it->status == UploadQueueEntry::ENTRY_IN_PROGRESS) {
- this_batch_size += it->filesize;
- it->onFailure(status.last_error);
- }
- }
- } else if (status.state == HTTP::Status::COMPLETE) {
- HTTPDatablock* db = current_upload_request->getResponse()->coalesceBlocks();
-
- std::multimap<std::string, std::string> file_statuses;
-
- try {
- std::vector<std::string> status_lines;
- std::string db_str(db->data(), db->size());
- std::istringstream data_stream(db_str);
-
- while (!data_stream.eof()) {
- std::string tmp;
- std::getline(data_stream, tmp);
- trim(tmp);
- if (tmp.size() && tmp.at(tmp.size() - 1) == '\r') tmp.resize(tmp.size() - 1);
- if (tmp.empty()) break;
- status_lines.push_back(tmp);
- }
- file_statuses = parse_http_headers(status_lines.begin(), status_lines.end());
- } catch (const std::exception& e) {
- #ifndef NDEBUG
- printf("Parse error while reading file statuses back from endpoint: %s\n", e.what());
- #endif
- }
-
- for (std::list<UploadQueueEntry>::iterator it = queue.begin(); it != queue.end(); ++it) {
- if (it->status == UploadQueueEntry::ENTRY_IN_PROGRESS) {
- this_batch_size += it->filesize;
-
- bool had_error = false;
-
- // See if we got an error condition back from the endpoint
- std::multimap<std::string, std::string>::iterator stat_it
- = file_statuses.find(it->post_field);
- if (stat_it != file_statuses.end()) {
- std::string lcase_status = to_lower_copy(stat_it->second);
- if (lcase_status.size() >= 5 && (lcase_status.substr(0,5) == "error")) {
- #ifndef NDEBUG
- FBLOG_WARN("UploadQueue", "Upload endpoint signaled an error for field "
- << it->post_field.c_str() << ":" << stat_it->second.c_str() << std::endl);
- #endif
- it->onFailure(stat_it->second);
- had_error = true;
- }
- } else {
- #ifndef NDEBUG
- FBLOG_WARN("UploadQueue", "Upload endpoint didn't return a status for field "
- << it->post_field.c_str() << std::endl);
- #endif
- }
-
- if (!had_error) it->setStatus(UploadQueueEntry::ENTRY_COMPLETE);
- }
- }
- } else return;
-
- boost::shared_ptr<HTTPResponseData> response = current_upload_request->getResponse();
- if (response) {
- for (std::map<std::string, std::string>::iterator it = response->cookies.begin();
- it != response->cookies.end(); ++it) {
- cookies[it->first] = it->second;
- }
- }
-
-
- // using threadSafeDestroy since our caller is probably the i/o service of that request...
- current_upload_request->threadSafeDestroy();
- current_upload_request = NULL;
-
- // Since this is done (error or not), find another queue entry that's WAITING and start that.
- start_next_upload();
- }
-
- void UploadQueue::cancel() {
- if (current_upload_request) {
- HTTPRequest* r = current_upload_request;
- current_upload_request = NULL;
-
- r->cancel();
- delete r;
- }
-
- queue.clear();
- status = UploadQueue::UPLOAD_COMPLETE;
- if (queue_finished_callback) queue_finished_callback(shared_ptr());
- }
-
- void UploadQueue::sendUpdateEvent()
- {
- StatusUpdateEvent evt(getStatusDict());
- SendEvent(&evt);
- }
-
- void HTTP::UploadQueue::addCompletionHandler( const FB::URI& uri )
- {
- if (status != UPLOAD_IDLE) throw std::runtime_error("UploadQueue::addCompletionHandler(): queue has been dispatched, can't be modified");
- completion_handlers.push_back(uri);
- }
-
- void HTTP::UploadQueue::addFile( const UploadQueueEntry& qe )
- {
- if (status != UPLOAD_IDLE) throw std::runtime_error("UploadQueue::addFile(): queue has been dispatched, can't be modified");
-
- current_queue_bytes += qe.filesize;
- total_queue_bytes += qe.filesize;
- ++total_queue_files;
- ++files_waiting;
- queue.push_back(qe);
- }
-
- bool HTTP::UploadQueue::removeFile( const std::wstring& source_path )
- {
- if (status != UPLOAD_IDLE) throw std::runtime_error("UploadQueue::addFile(): queue has been dispatched, can't be modified");
-
- for (std::list<UploadQueueEntry>::iterator it = queue.begin(); it != queue.end(); ++it) {
- if (it->source_path == source_path) {
- current_queue_bytes -= it->filesize;
- total_queue_bytes -= it->filesize;
- --total_queue_files;
- --files_waiting;
- queue.erase(it);
- return true;
- }
- }
-
- return false; // not found
- }
-
- void sendUpdateEvent();