/mordor/streams/http.cpp
C++ | 609 lines | 553 code | 31 blank | 25 comment | 138 complexity | 3e4f93e3e22c22d0e11dca1d727f8b4d MD5 | raw file
Possible License(s): BSD-3-Clause
- // Copyright (c) 2009 - Mozy, Inc.
- #include "http.h"
- #include "mordor/fiber.h"
- #include "mordor/http/client.h"
- #include "mordor/socket.h"
- #include "null.h"
- #include "transfer.h"
- using namespace Mordor::HTTP;
- namespace Mordor {
- HTTPStream::HTTPStream(const URI &uri, RequestBroker::ptr requestBroker,
- boost::function<bool (size_t)> delayDg)
- : FilterStream(Stream::ptr(), false),
- m_hasResponse(false),
- m_requestBroker(requestBroker),
- m_pos(0),
- m_size(-1),
- m_sizeAdvice(-1),
- m_readAdvice(~0ull),
- m_writeAdvice(~0ull),
- m_readRequested(0ull),
- m_writeRequested(0ull),
- m_delayDg(delayDg),
- mp_retries(NULL),
- m_writeInProgress(false),
- m_abortWrite(false)
- {
- m_requestHeaders.requestLine.uri = uri;
- }
- HTTPStream::HTTPStream(const Request &requestHeaders, RequestBroker::ptr requestBroker,
- boost::function<bool (size_t)> delayDg)
- : FilterStream(Stream::ptr(), false),
- m_requestHeaders(requestHeaders),
- m_hasResponse(false),
- m_requestBroker(requestBroker),
- m_pos(0),
- m_size(-1),
- m_sizeAdvice(-1),
- m_readAdvice(~0ull),
- m_writeAdvice(~0ull),
- m_readRequested(0ull),
- m_writeRequested(0ull),
- m_delayDg(delayDg),
- mp_retries(NULL),
- m_writeInProgress(false),
- m_abortWrite(false)
- {}
- HTTPStream::~HTTPStream()
- {
- try {
- clearParent();
- } catch (...) {
- // ignore clean up errors
- }
- }
- ETag
- HTTPStream::eTag()
- {
- stat();
- return m_eTag;
- }
- void
- HTTPStream::start()
- {
- start(m_readAdvice == 0ull ? (size_t)-1 : 0u);
- }
- void
- HTTPStream::start(size_t length)
- {
- if (parent() && !parent()->supportsRead())
- clearParent();
- if (!parent() || m_readRequested == 0) {
- if (parent()) {
- // We've read everything, but haven't triggered EOF yet
- MORDOR_ASSERT(m_readRequested == 0);
- try {
- char byte;
- MORDOR_VERIFY(parent()->read(&byte, 1) == 0);
- } catch (...) {
- // Ignore errors finishing the request
- }
- }
- // Don't bother doing a request that will never do anything
- if (m_size >= 0 && m_pos >= m_size) {
- parent(NullStream::get_ptr());
- return;
- }
- m_requestHeaders.requestLine.method = GET;
- m_requestHeaders.request.ifNoneMatch.clear();
- m_requestHeaders.general.transferEncoding.clear();
- m_requestHeaders.entity.contentLength = ~0ull;
- m_requestHeaders.entity.contentRange = ContentRange();
- if (!m_eTag.unspecified) {
- if (m_pos != 0) {
- m_requestHeaders.request.ifRange = m_eTag;
- m_requestHeaders.request.ifMatch.clear();
- } else {
- m_requestHeaders.request.ifMatch.insert(m_eTag);
- m_requestHeaders.request.ifRange = ETag();
- }
- } else {
- m_requestHeaders.request.ifRange = ETag();
- m_requestHeaders.request.ifMatch.clear();
- }
- m_requestHeaders.request.range.clear();
- if (m_pos != 0 || (m_readAdvice != ~0ull && length != (size_t)-1)) {
- if (length == (size_t)-1)
- m_readRequested = ~0ull;
- else
- m_readRequested = (std::max)((unsigned long long)length,
- m_readAdvice);
- m_requestHeaders.request.range.push_back(
- std::make_pair((unsigned long long)m_pos,
- m_readRequested == ~0ull ? ~0ull : m_pos + m_readRequested - 1));
- } else {
- m_readRequested = ~0ull;
- }
- ClientRequest::ptr request = m_requestBroker->request(m_requestHeaders);
- const Response &response = request->response();
- Stream::ptr responseStream;
- switch (response.status.status) {
- case OK:
- case PARTIAL_CONTENT:
- m_response = response;
- m_hasResponse = true;
- if (response.entity.contentRange.instance != ~0ull)
- m_size = (long long)response.entity.contentRange.instance;
- else if (response.entity.contentLength != ~0ull)
- m_size = (long long)response.entity.contentLength;
- else
- m_size = -2;
- if (m_eTag.unspecified &&
- !response.response.eTag.unspecified) {
- m_eTag = response.response.eTag;
- } else if (!m_eTag.unspecified &&
- !response.response.eTag.unspecified &&
- response.response.eTag != m_eTag) {
- m_eTag = response.response.eTag;
- if (m_pos != 0 && response.status.status ==
- PARTIAL_CONTENT) {
- // Server doesn't support If-Range
- request->cancel(true);
- clearParent();
- } else {
- m_readRequest = request;
- parent(request->responseStream());
- }
- m_pos = 0;
- MORDOR_THROW_EXCEPTION(EntityChangedException());
- }
- responseStream = request->responseStream();
- // Server doesn't support Range
- if (m_pos != 0 && response.status.status == OK) {
- m_readRequested = ~0ull;
- transferStream(responseStream, NullStream::get(), m_pos);
- }
- m_readRequest = request;
- parent(responseStream);
- break;
- default:
- // The bad ClientRequest `request' is propagated along with the
- // InvalidResponseException, the uppper level should handle the
- // error and finish the request, fail to do so will cause the
- // corresponding ClientConnection abortion and all the remaining
- // requests on the connection will be canceled.
- MORDOR_THROW_EXCEPTION(InvalidResponseException(request));
- }
- }
- }
- bool
- HTTPStream::checkModified()
- {
- MORDOR_ASSERT(!m_eTag.unspecified);
- if (parent())
- clearParent();
- m_requestHeaders.requestLine.method = GET;
- m_requestHeaders.request.ifMatch.clear();
- m_requestHeaders.request.ifRange = ETag();
- m_requestHeaders.request.range.clear();
- m_requestHeaders.request.ifNoneMatch.insert(m_eTag);
- m_requestHeaders.general.transferEncoding.clear();
- m_requestHeaders.entity.contentLength = ~0ull;
- m_requestHeaders.entity.contentRange = ContentRange();
- if (m_pos != 0 || m_readAdvice != ~0ull) {
- m_readRequested = m_readAdvice == 0ull ? ~0ull : m_readAdvice;
- m_requestHeaders.request.range.push_back(
- std::make_pair((unsigned long long)m_pos,
- m_readRequested == ~0ull ? ~0ull : m_pos + m_readRequested - 1));
- } else {
- m_readRequested = ~0ull;
- }
- ClientRequest::ptr request = m_requestBroker->request(m_requestHeaders);
- const Response &response = request->response();
- switch (response.status.status) {
- case OK:
- case PARTIAL_CONTENT:
- case NOT_MODIFIED:
- m_response = response;
- m_hasResponse = true;
- if (response.entity.contentRange.instance != ~0ull)
- m_size = (long long)response.entity.contentRange.instance;
- else if (response.entity.contentLength != ~0ull)
- m_size = (long long)response.entity.contentLength;
- else
- m_size = -2;
- break;
- default:
- MORDOR_THROW_EXCEPTION(InvalidResponseException(request));
- }
- if (response.status.status != NOT_MODIFIED)
- m_eTag = response.response.eTag;
- else
- return false;
- Stream::ptr responseStream = request->responseStream();
- // Server doesn't support Range
- if (m_pos != 0 && response.status.status == OK) {
- // We don't really care about any data transfer problems here,
- // since that's not what the caller is asking for
- try {
- m_readRequested = ~0ull;
- transferStream(responseStream, NullStream::get(), m_pos);
- } catch (...) {
- return true;
- }
- }
- m_readRequest = request;
- parent(responseStream);
- return true;
- }
- const HTTP::Response &
- HTTPStream::response()
- {
- if (m_hasResponse)
- return m_response;
- // NOTE: if m_writeInProgress == true, the write operation will be aborted
- stat();
- return m_response;
- }
- size_t
- HTTPStream::read(Buffer &buffer, size_t length)
- {
- size_t localRetries = 0;
- size_t *retries = mp_retries ? mp_retries : &localRetries;
- while (true) {
- // remember previous retry count
- size_t prevRetries = *retries;
- bool new_request = !parent();
- start(length);
- MORDOR_ASSERT(parent());
- try {
- try {
- size_t result = parent()->read(buffer, length);
- m_pos += result;
- m_readRequested -= result;
- if (result > 0)
- *retries = 0; // only do this if we've made progress
- return result;
- } catch(...) {
- // if this was a new request, RetryRequestBroker reset the retry count to 0
- // undo this, so we don't get stuck in an infinite retry loop if the server
- // consistently hangs up on us immediately after 200 OK
- if (new_request)
- *retries = prevRetries;
- throw;
- }
- } catch (SocketException &) {
- clearParent(true);
- if (!m_delayDg || !m_delayDg(++*retries))
- throw;
- continue;
- } catch (UnexpectedEofException &) {
- clearParent(true);
- if (!m_delayDg || !m_delayDg(++*retries))
- throw;
- continue;
- }
- }
- }
- void
- HTTPStream::doWrite(ClientRequest::ptr request)
- {
- parent(request->requestStream());
- m_writeFuture2.reset();
- m_writeFuture.signal();
- m_writeFuture2.wait();
- if (m_abortWrite)
- MORDOR_THROW_EXCEPTION(OperationAbortedException());
- MORDOR_ASSERT(!parent());
- }
- void
- HTTPStream::startWrite()
- {
- try {
- m_writeRequest = m_requestBroker->request(m_requestHeaders,
- false, boost::bind(&HTTPStream::doWrite, this, _1));
- m_writeInProgress = false;
- m_writeFuture.signal();
- } catch (OperationAbortedException &) {
- if (!m_abortWrite)
- m_writeException = boost::current_exception();
- m_writeInProgress = false;
- m_writeFuture.signal();
- } catch (...) {
- m_writeException = boost::current_exception();
- m_writeInProgress = false;
- m_writeFuture.signal();
- }
- }
- size_t
- HTTPStream::write(const Buffer &buffer, size_t length)
- {
- MORDOR_ASSERT(m_sizeAdvice == -1 ||
- (unsigned long long)m_pos + length <= (unsigned long long)m_sizeAdvice);
- if (parent() && !parent()->supportsWrite())
- clearParent();
- if (!parent()) {
- m_requestHeaders.requestLine.method = PUT;
- m_requestHeaders.request.ifMatch.clear();
- m_requestHeaders.request.ifRange = ETag();
- m_requestHeaders.request.range.clear();
- m_requestHeaders.request.ifNoneMatch.clear();
- if (m_pos != 0 || m_writeAdvice != ~0ull) {
- m_requestHeaders.entity.contentRange.first = m_pos;
- if (m_sizeAdvice != -1) {
- m_writeRequested = (std::max)((unsigned long long)length,
- m_writeAdvice);
- m_writeRequested = (std::min)(m_pos + m_writeRequested,
- (unsigned long long)m_sizeAdvice) - m_pos;
- m_requestHeaders.entity.contentLength = m_writeRequested;
- m_requestHeaders.entity.contentRange.last = m_pos +
- m_writeRequested - 1;
- } else {
- m_requestHeaders.entity.contentLength = ~0ull;
- m_requestHeaders.entity.contentRange.last = ~0ull;
- m_writeRequested = m_writeAdvice;
- }
- m_requestHeaders.entity.contentRange.instance =
- (unsigned long long)m_sizeAdvice;
- } else {
- m_requestHeaders.entity.contentLength =
- (unsigned long long)m_sizeAdvice;
- m_requestHeaders.entity.contentRange = ContentRange();
- m_writeRequested = m_sizeAdvice;
- }
- if (m_sizeAdvice == -1) {
- if (m_requestHeaders.general.transferEncoding.empty())
- m_requestHeaders.general.transferEncoding.push_back("chunked");
- } else {
- m_requestHeaders.general.transferEncoding.clear();
- }
- m_writeException = boost::exception_ptr();
- m_writeFuture.reset();
- // Have to schedule this because RequestBroker::request doesn't return
- // until the entire request is complete
- m_writeInProgress = true;
- MORDOR_ASSERT(Scheduler::getThis());
- Scheduler::getThis()->schedule(
- boost::bind(&HTTPStream::startWrite, this));
- m_writeFuture.wait();
- if (m_writeException)
- Mordor::rethrow_exception(m_writeException);
- MORDOR_ASSERT(parent());
- }
- size_t result = parent()->write(buffer,
- (size_t)(std::min)((unsigned long long)length, m_writeRequested));
- m_pos += result;
- m_writeRequested -= result;
- if (m_writeRequested == 0)
- close(WRITE);
- return result;
- }
- long long
- HTTPStream::seek(long long offset, Anchor anchor)
- {
- switch (anchor) {
- case CURRENT:
- offset = m_pos + offset;
- break;
- case END:
- stat();
- MORDOR_ASSERT(m_size >= 0);
- offset = m_size + offset;
- break;
- case BEGIN:
- break;
- default:
- MORDOR_NOTREACHED();
- }
- if (offset < 0)
- MORDOR_THROW_EXCEPTION(std::invalid_argument(
- "resulting offset is before the beginning of the file"));
- if (offset == m_pos)
- return m_pos;
- // Attempt to do an optimized forward seek
- if (offset > m_pos && m_readRequested != ~0ull && parent() &&
- parent()->supportsRead() &&
- m_pos + m_readRequested > (unsigned long long)offset) {
- try {
- transferStream(*this, NullStream::get(), offset - m_pos);
- MORDOR_ASSERT(m_pos == offset);
- return m_pos;
- } catch (...) {
- clearParent(true);
- }
- } else {
- clearParent();
- }
- return m_pos = offset;
- }
- bool
- HTTPStream::supportsSize()
- {
- stat();
- MORDOR_ASSERT(m_size != -1);
- MORDOR_ASSERT(m_size >= -2);
- if (m_size == -2)
- return false;
- else
- return true;
- }
- void
- HTTPStream::stat()
- {
- if (m_size == -1) {
- clearParent();
- m_requestHeaders.requestLine.method = HEAD;
- m_requestHeaders.request.ifMatch.clear();
- m_requestHeaders.request.ifRange = ETag();
- m_requestHeaders.request.range.clear();
- m_requestHeaders.request.ifNoneMatch.clear();
- m_requestHeaders.general.transferEncoding.clear();
- m_requestHeaders.entity.contentLength = ~0ull;
- m_requestHeaders.entity.contentRange = ContentRange();
- ClientRequest::ptr request = m_requestBroker->request(m_requestHeaders);
- const Response &response = request->response();
- switch (response.status.status) {
- case OK:
- m_response = response;
- m_hasResponse = true;
- if (response.entity.contentLength != ~0ull)
- m_size = (long long)response.entity.contentLength;
- else
- m_size = -2;
- if (m_eTag.unspecified &&
- !response.response.eTag.unspecified) {
- m_eTag = response.response.eTag;
- } else if (!m_eTag.unspecified &&
- response.response.eTag != m_eTag) {
- m_eTag = response.response.eTag;
- clearParent();
- m_pos = 0;
- MORDOR_THROW_EXCEPTION(EntityChangedException());
- }
- break;
- default:
- MORDOR_THROW_EXCEPTION(InvalidResponseException(request));
- }
- }
- }
- long long
- HTTPStream::size()
- {
- stat();
- MORDOR_ASSERT(supportsSize());
- return m_size;
- }
- void
- HTTPStream::truncate(long long size)
- {
- if (parent())
- clearParent();
- m_requestHeaders.requestLine.method = PUT;
- m_requestHeaders.request.ifMatch.clear();
- m_requestHeaders.request.ifRange = ETag();
- m_requestHeaders.request.range.clear();
- m_requestHeaders.request.ifNoneMatch.clear();
- m_requestHeaders.general.transferEncoding.clear();
- m_requestHeaders.entity.contentLength = 0;
- m_requestHeaders.entity.contentRange = ContentRange(~0ull, ~0ull, size);
- ClientRequest::ptr request = m_requestBroker->request(m_requestHeaders);
- if (request->response().status.status != OK)
- MORDOR_THROW_EXCEPTION(InvalidResponseException(request));
- request->finish();
- }
- void
- HTTPStream::close(CloseType type)
- {
- if ((type & WRITE) && m_sizeAdvice == 0ll) {
- // If we weren't supposed to write anything, we shouldn't have written
- // anything :)
- MORDOR_ASSERT(!parent());
- m_requestHeaders.requestLine.method = PUT;
- m_requestHeaders.request.ifMatch.clear();
- m_requestHeaders.request.ifRange = ETag();
- m_requestHeaders.request.range.clear();
- m_requestHeaders.request.ifNoneMatch.clear();
- m_requestHeaders.general.transferEncoding.clear();
- m_requestHeaders.entity.contentLength = 0;
- m_requestHeaders.entity.contentRange = ContentRange();
- ClientRequest::ptr request = m_requestBroker->request(m_requestHeaders);
- switch (request->response().status.status) {
- case OK:
- case CREATED:
- m_response = request->response();
- m_hasResponse = true;
- break;
- default:
- MORDOR_THROW_EXCEPTION(InvalidResponseException(m_writeRequest));
- }
- request->finish();
- m_sizeAdvice = ~0ull;
- return;
- }
- if (parent()) {
- parent()->close();
- }
- if (parent() && (type & WRITE) && parent()->supportsWrite()) {
- parent(Stream::ptr());
- m_writeFuture.reset();
- m_writeFuture2.signal();
- m_writeFuture.wait();
- if (m_writeException)
- Mordor::rethrow_exception(m_writeException);
- MORDOR_ASSERT(m_writeRequest);
- switch ((int)m_writeRequest->response().status.status) {
- case OK:
- case CREATED:
- case 207: // Partial Update OK, from http://www.hpl.hp.com/personal/ange/archives/archives-97/http-wg-archive/2530.html
- m_response = m_writeRequest->response();
- m_hasResponse = true;
- try {
- m_writeRequest->finish();
- } catch (...) {
- m_writeRequest.reset();
- throw;
- }
- m_writeRequest.reset();
- break;
- default:
- MORDOR_THROW_EXCEPTION(InvalidResponseException(m_writeRequest));
- }
- }
- parent(Stream::ptr());
- }
- void
- HTTPStream::flush(bool flushParent)
- {
- if (parent() && parent()->supportsWrite())
- parent()->flush(flushParent);
- }
- void
- HTTPStream::clearParent(bool error)
- {
- if (m_writeInProgress) {
- m_abortWrite = true;
- m_writeFuture.reset();
- m_writeFuture2.signal();
- m_writeFuture.wait();
- MORDOR_ASSERT(!m_writeInProgress);
- }
- if (parent()) {
- if (parent()->supportsWrite() && m_writeRequest) {
- if (error)
- m_writeRequest->cancel(true);
- else
- m_writeRequest->finish();
- m_writeRequest.reset();
- }
- if (parent()->supportsRead() && m_readRequest) {
- if (error)
- m_readRequest->cancel(true);
- else
- m_readRequest->finish();
- m_readRequest.reset();
- }
- parent(Stream::ptr());
- }
- }
- }