/mordor/streams/http.cpp

  1// Copyright (c) 2009 - Mozy, Inc.
  2
  3#include "http.h"
  4
  5#include "mordor/fiber.h"
  6#include "mordor/http/client.h"
  7#include "mordor/socket.h"
  8#include "null.h"
  9#include "transfer.h"
 10
 11using namespace Mordor::HTTP;
 12
 13namespace Mordor {
 14
 15HTTPStream::HTTPStream(const URI &uri, RequestBroker::ptr requestBroker,
 16    boost::function<bool (size_t)> delayDg)
 17: FilterStream(Stream::ptr(), false),
 18  m_hasResponse(false),
 19  m_requestBroker(requestBroker),
 20  m_pos(0),
 21  m_size(-1),
 22  m_sizeAdvice(-1),
 23  m_readAdvice(~0ull),
 24  m_writeAdvice(~0ull),
 25  m_readRequested(0ull),
 26  m_writeRequested(0ull),
 27  m_delayDg(delayDg),
 28  mp_retries(NULL),
 29  m_writeInProgress(false),
 30  m_abortWrite(false)
 31{
 32    m_requestHeaders.requestLine.uri = uri;
 33}
 34
 35HTTPStream::HTTPStream(const Request &requestHeaders, RequestBroker::ptr requestBroker,
 36    boost::function<bool (size_t)> delayDg)
 37: FilterStream(Stream::ptr(), false),
 38  m_requestHeaders(requestHeaders),
 39  m_hasResponse(false),
 40  m_requestBroker(requestBroker),
 41  m_pos(0),
 42  m_size(-1),
 43  m_sizeAdvice(-1),
 44  m_readAdvice(~0ull),
 45  m_writeAdvice(~0ull),
 46  m_readRequested(0ull),
 47  m_writeRequested(0ull),
 48  m_delayDg(delayDg),
 49  mp_retries(NULL),
 50  m_writeInProgress(false),
 51  m_abortWrite(false)
 52{}
 53
 54HTTPStream::~HTTPStream()
 55{
 56    try {
 57        clearParent();
 58    } catch (...) {
 59        // ignore clean up errors
 60    }
 61}
 62
 63ETag
 64HTTPStream::eTag()
 65{
 66    stat();
 67    return m_eTag;
 68}
 69
 70void
 71HTTPStream::start()
 72{
 73    start(m_readAdvice == 0ull ? (size_t)-1 : 0u);
 74}
 75
 76void
 77HTTPStream::start(size_t length)
 78{
 79    if (parent() && !parent()->supportsRead())
 80        clearParent();
 81    if (!parent() || m_readRequested == 0) {
 82        if (parent()) {
 83            // We've read everything, but haven't triggered EOF yet
 84            MORDOR_ASSERT(m_readRequested == 0);
 85            try {
 86                char byte;
 87                MORDOR_VERIFY(parent()->read(&byte, 1) == 0);
 88            } catch (...) {
 89                // Ignore errors finishing the request
 90            }
 91        }
 92        // Don't bother doing a request that will never do anything
 93        if (m_size >= 0 && m_pos >= m_size) {
 94            parent(NullStream::get_ptr());
 95            return;
 96        }
 97        m_requestHeaders.requestLine.method = GET;
 98        m_requestHeaders.request.ifNoneMatch.clear();
 99        m_requestHeaders.general.transferEncoding.clear();
100        m_requestHeaders.entity.contentLength = ~0ull;
101        m_requestHeaders.entity.contentRange = ContentRange();
102        if (!m_eTag.unspecified) {
103            if (m_pos != 0) {
104                m_requestHeaders.request.ifRange = m_eTag;
105                m_requestHeaders.request.ifMatch.clear();
106            } else {
107                m_requestHeaders.request.ifMatch.insert(m_eTag);
108                m_requestHeaders.request.ifRange = ETag();
109            }
110        } else {
111            m_requestHeaders.request.ifRange = ETag();
112            m_requestHeaders.request.ifMatch.clear();
113        }
114        m_requestHeaders.request.range.clear();
115        if (m_pos != 0 || (m_readAdvice != ~0ull && length != (size_t)-1)) {
116            if (length == (size_t)-1)
117                m_readRequested = ~0ull;
118            else
119                m_readRequested = (std::max)((unsigned long long)length,
120                    m_readAdvice);
121            m_requestHeaders.request.range.push_back(
122                std::make_pair((unsigned long long)m_pos,
123                m_readRequested == ~0ull ? ~0ull : m_pos + m_readRequested - 1));
124        } else {
125            m_readRequested = ~0ull;
126        }
127        ClientRequest::ptr request = m_requestBroker->request(m_requestHeaders);
128        const Response &response = request->response();
129        Stream::ptr responseStream;
130        switch (response.status.status) {
131            case OK:
132            case PARTIAL_CONTENT:
133                m_response = response;
134                m_hasResponse = true;
135                if (response.entity.contentRange.instance != ~0ull)
136                    m_size = (long long)response.entity.contentRange.instance;
137                else if (response.entity.contentLength != ~0ull)
138                    m_size = (long long)response.entity.contentLength;
139                else
140                    m_size = -2;
141                if (m_eTag.unspecified &&
142                    !response.response.eTag.unspecified) {
143                    m_eTag = response.response.eTag;
144                } else if (!m_eTag.unspecified &&
145                    !response.response.eTag.unspecified &&
146                    response.response.eTag != m_eTag) {
147                    m_eTag = response.response.eTag;
148                    if (m_pos != 0 && response.status.status ==
149                        PARTIAL_CONTENT) {
150                        // Server doesn't support If-Range
151                        request->cancel(true);
152                        clearParent();
153                    } else {
154                        m_readRequest = request;
155                        parent(request->responseStream());
156                    }
157                    m_pos = 0;
158                    MORDOR_THROW_EXCEPTION(EntityChangedException());
159                }
160                responseStream = request->responseStream();
161                // Server doesn't support Range
162                if (m_pos != 0 && response.status.status == OK) {
163                    m_readRequested = ~0ull;
164                    transferStream(responseStream, NullStream::get(), m_pos);
165                }
166                m_readRequest = request;
167                parent(responseStream);
168                break;
169            default:
170                // The bad ClientRequest `request' is propagated along with the
171                // InvalidResponseException, the uppper level should handle the
172                // error and finish the request, fail to do so will cause the
173                // corresponding ClientConnection abortion and all the remaining
174                // requests on the connection will be canceled.
175                MORDOR_THROW_EXCEPTION(InvalidResponseException(request));
176        }
177    }
178}
179
180bool
181HTTPStream::checkModified()
182{
183    MORDOR_ASSERT(!m_eTag.unspecified);
184    if (parent())
185        clearParent();
186
187    m_requestHeaders.requestLine.method = GET;
188    m_requestHeaders.request.ifMatch.clear();
189    m_requestHeaders.request.ifRange = ETag();
190    m_requestHeaders.request.range.clear();
191    m_requestHeaders.request.ifNoneMatch.insert(m_eTag);
192    m_requestHeaders.general.transferEncoding.clear();
193    m_requestHeaders.entity.contentLength = ~0ull;
194    m_requestHeaders.entity.contentRange = ContentRange();
195    if (m_pos != 0 || m_readAdvice != ~0ull) {
196        m_readRequested = m_readAdvice == 0ull ? ~0ull : m_readAdvice;
197        m_requestHeaders.request.range.push_back(
198            std::make_pair((unsigned long long)m_pos,
199            m_readRequested == ~0ull ? ~0ull : m_pos + m_readRequested - 1));
200    } else {
201        m_readRequested = ~0ull;
202    }
203    ClientRequest::ptr request = m_requestBroker->request(m_requestHeaders);
204    const Response &response = request->response();
205    switch (response.status.status) {
206        case OK:
207        case PARTIAL_CONTENT:
208        case NOT_MODIFIED:
209            m_response = response;
210            m_hasResponse = true;
211            if (response.entity.contentRange.instance != ~0ull)
212                m_size = (long long)response.entity.contentRange.instance;
213            else if (response.entity.contentLength != ~0ull)
214                m_size = (long long)response.entity.contentLength;
215            else
216                m_size = -2;
217            break;
218        default:
219            MORDOR_THROW_EXCEPTION(InvalidResponseException(request));
220    }
221    if (response.status.status != NOT_MODIFIED)
222        m_eTag = response.response.eTag;
223    else
224        return false;
225    Stream::ptr responseStream = request->responseStream();
226
227    // Server doesn't support Range
228    if (m_pos != 0 && response.status.status == OK) {
229        // We don't really care about any data transfer problems here,
230        // since that's not what the caller is asking for
231        try {
232            m_readRequested = ~0ull;
233            transferStream(responseStream, NullStream::get(), m_pos);
234        } catch (...) {
235            return true;
236        }
237    }
238    m_readRequest = request;
239    parent(responseStream);
240    return true;
241}
242
243const HTTP::Response &
244HTTPStream::response()
245{
246    if (m_hasResponse)
247        return m_response;
248    // NOTE: if m_writeInProgress == true, the write operation will be aborted
249    stat();
250    return m_response;
251}
252
253size_t
254HTTPStream::read(Buffer &buffer, size_t length)
255{
256    size_t localRetries = 0;
257    size_t *retries = mp_retries ? mp_retries : &localRetries;
258    while (true) {
259        // remember previous retry count
260        size_t prevRetries = *retries;
261        bool new_request = !parent();
262        start(length);
263
264        MORDOR_ASSERT(parent());
265        try {
266            try {
267                size_t result = parent()->read(buffer, length);
268                m_pos += result;
269                m_readRequested -= result;
270                if (result > 0)
271                    *retries = 0;   // only do this if we've made progress
272                return result;
273            } catch(...) {
274                // if this was a new request, RetryRequestBroker reset the retry count to 0
275                // undo this, so we don't get stuck in an infinite retry loop if the server
276                // consistently hangs up on us immediately after 200 OK
277                if (new_request)
278                    *retries = prevRetries;
279                throw;
280            }
281        } catch (SocketException &) {
282            clearParent(true);
283            if (!m_delayDg || !m_delayDg(++*retries))
284                throw;
285            continue;
286        } catch (UnexpectedEofException &) {
287            clearParent(true);
288            if (!m_delayDg || !m_delayDg(++*retries))
289                throw;
290            continue;
291        }
292    }
293}
294
295void
296HTTPStream::doWrite(ClientRequest::ptr request)
297{
298    parent(request->requestStream());
299    m_writeFuture2.reset();
300    m_writeFuture.signal();
301    m_writeFuture2.wait();
302    if (m_abortWrite)
303        MORDOR_THROW_EXCEPTION(OperationAbortedException());
304    MORDOR_ASSERT(!parent());
305}
306
307void
308HTTPStream::startWrite()
309{
310    try {
311        m_writeRequest = m_requestBroker->request(m_requestHeaders,
312            false, boost::bind(&HTTPStream::doWrite, this, _1));
313        m_writeInProgress = false;
314        m_writeFuture.signal();
315    } catch (OperationAbortedException &) {
316        if (!m_abortWrite)
317            m_writeException = boost::current_exception();
318        m_writeInProgress = false;
319        m_writeFuture.signal();
320    } catch (...) {
321        m_writeException = boost::current_exception();
322        m_writeInProgress = false;
323        m_writeFuture.signal();
324    }
325}
326
327size_t
328HTTPStream::write(const Buffer &buffer, size_t length)
329{
330    MORDOR_ASSERT(m_sizeAdvice == -1 ||
331        (unsigned long long)m_pos + length <= (unsigned long long)m_sizeAdvice);
332    if (parent() && !parent()->supportsWrite())
333        clearParent();
334    if (!parent()) {
335        m_requestHeaders.requestLine.method = PUT;
336        m_requestHeaders.request.ifMatch.clear();
337        m_requestHeaders.request.ifRange = ETag();
338        m_requestHeaders.request.range.clear();
339        m_requestHeaders.request.ifNoneMatch.clear();
340
341        if (m_pos != 0 || m_writeAdvice != ~0ull) {
342            m_requestHeaders.entity.contentRange.first = m_pos;
343            if (m_sizeAdvice != -1) {
344                m_writeRequested = (std::max)((unsigned long long)length,
345                    m_writeAdvice);
346                m_writeRequested = (std::min)(m_pos + m_writeRequested,
347                    (unsigned long long)m_sizeAdvice) - m_pos;
348                m_requestHeaders.entity.contentLength = m_writeRequested;
349                m_requestHeaders.entity.contentRange.last = m_pos +
350                    m_writeRequested - 1;
351            } else {
352                m_requestHeaders.entity.contentLength = ~0ull;
353                m_requestHeaders.entity.contentRange.last = ~0ull;
354                m_writeRequested = m_writeAdvice;
355            }
356            m_requestHeaders.entity.contentRange.instance =
357                (unsigned long long)m_sizeAdvice;
358        } else {
359            m_requestHeaders.entity.contentLength =
360                (unsigned long long)m_sizeAdvice;
361            m_requestHeaders.entity.contentRange = ContentRange();
362            m_writeRequested = m_sizeAdvice;
363        }
364        if (m_sizeAdvice == -1) {
365            if (m_requestHeaders.general.transferEncoding.empty())
366                m_requestHeaders.general.transferEncoding.push_back("chunked");
367        } else {
368            m_requestHeaders.general.transferEncoding.clear();
369        }
370
371        m_writeException = boost::exception_ptr();
372        m_writeFuture.reset();
373        // Have to schedule this because RequestBroker::request doesn't return
374        // until the entire request is complete
375        m_writeInProgress = true;
376        MORDOR_ASSERT(Scheduler::getThis());
377        Scheduler::getThis()->schedule(
378            boost::bind(&HTTPStream::startWrite, this));
379        m_writeFuture.wait();
380        if (m_writeException)
381            Mordor::rethrow_exception(m_writeException);
382        MORDOR_ASSERT(parent());
383    }
384    size_t result = parent()->write(buffer,
385        (size_t)(std::min)((unsigned long long)length, m_writeRequested));
386    m_pos += result;
387    m_writeRequested -= result;
388    if (m_writeRequested == 0)
389        close(WRITE);
390    return result;
391}
392
393long long
394HTTPStream::seek(long long offset, Anchor anchor)
395{
396    switch (anchor) {
397        case CURRENT:
398            offset = m_pos + offset;
399            break;
400        case END:
401            stat();
402            MORDOR_ASSERT(m_size >= 0);
403            offset = m_size + offset;
404            break;
405        case BEGIN:
406            break;
407        default:
408            MORDOR_NOTREACHED();
409    }
410    if (offset < 0)
411        MORDOR_THROW_EXCEPTION(std::invalid_argument(
412            "resulting offset is before the beginning of the file"));
413    if (offset == m_pos)
414        return m_pos;
415    // Attempt to do an optimized forward seek
416    if (offset > m_pos && m_readRequested != ~0ull && parent() &&
417        parent()->supportsRead() &&
418        m_pos + m_readRequested > (unsigned long long)offset) {
419        try {
420            transferStream(*this, NullStream::get(), offset - m_pos);
421            MORDOR_ASSERT(m_pos == offset);
422            return m_pos;
423        } catch (...) {
424            clearParent(true);
425        }
426    } else {
427        clearParent();
428    }
429    return m_pos = offset;
430}
431
432bool
433HTTPStream::supportsSize()
434{
435    stat();
436    MORDOR_ASSERT(m_size != -1);
437    MORDOR_ASSERT(m_size >= -2);
438    if (m_size == -2)
439        return false;
440    else
441        return true;
442}
443
444void
445HTTPStream::stat()
446{
447    if (m_size == -1) {
448        clearParent();
449        m_requestHeaders.requestLine.method = HEAD;
450        m_requestHeaders.request.ifMatch.clear();
451        m_requestHeaders.request.ifRange = ETag();
452        m_requestHeaders.request.range.clear();
453        m_requestHeaders.request.ifNoneMatch.clear();
454        m_requestHeaders.general.transferEncoding.clear();
455        m_requestHeaders.entity.contentLength = ~0ull;
456        m_requestHeaders.entity.contentRange = ContentRange();
457        ClientRequest::ptr request = m_requestBroker->request(m_requestHeaders);
458        const Response &response = request->response();
459        switch (response.status.status) {
460            case OK:
461                m_response = response;
462                m_hasResponse = true;
463                if (response.entity.contentLength != ~0ull)
464                    m_size = (long long)response.entity.contentLength;
465                else
466                    m_size = -2;
467                if (m_eTag.unspecified &&
468                    !response.response.eTag.unspecified) {
469                    m_eTag = response.response.eTag;
470                } else if (!m_eTag.unspecified &&
471                    response.response.eTag != m_eTag) {
472                    m_eTag = response.response.eTag;
473                    clearParent();
474                    m_pos = 0;
475                    MORDOR_THROW_EXCEPTION(EntityChangedException());
476                }
477                break;
478            default:
479                MORDOR_THROW_EXCEPTION(InvalidResponseException(request));
480        }
481    }
482}
483
484long long
485HTTPStream::size()
486{
487    stat();
488    MORDOR_ASSERT(supportsSize());
489    return m_size;
490}
491
492void
493HTTPStream::truncate(long long size)
494{
495    if (parent())
496        clearParent();
497    m_requestHeaders.requestLine.method = PUT;
498    m_requestHeaders.request.ifMatch.clear();
499    m_requestHeaders.request.ifRange = ETag();
500    m_requestHeaders.request.range.clear();
501    m_requestHeaders.request.ifNoneMatch.clear();
502    m_requestHeaders.general.transferEncoding.clear();
503    m_requestHeaders.entity.contentLength = 0;
504    m_requestHeaders.entity.contentRange = ContentRange(~0ull, ~0ull, size);
505    ClientRequest::ptr request = m_requestBroker->request(m_requestHeaders);
506    if (request->response().status.status != OK)
507        MORDOR_THROW_EXCEPTION(InvalidResponseException(request));
508    request->finish();
509}
510
511void
512HTTPStream::close(CloseType type)
513{
514    if ((type & WRITE) && m_sizeAdvice == 0ll) {
515        // If we weren't supposed to write anything, we shouldn't have written
516        // anything :)
517        MORDOR_ASSERT(!parent());
518        m_requestHeaders.requestLine.method = PUT;
519        m_requestHeaders.request.ifMatch.clear();
520        m_requestHeaders.request.ifRange = ETag();
521        m_requestHeaders.request.range.clear();
522        m_requestHeaders.request.ifNoneMatch.clear();
523        m_requestHeaders.general.transferEncoding.clear();
524        m_requestHeaders.entity.contentLength = 0;
525        m_requestHeaders.entity.contentRange = ContentRange();
526        ClientRequest::ptr request = m_requestBroker->request(m_requestHeaders);
527        switch (request->response().status.status) {
528            case OK:
529            case CREATED:
530                m_response = request->response();
531                m_hasResponse = true;
532                break;
533            default:
534                MORDOR_THROW_EXCEPTION(InvalidResponseException(m_writeRequest));
535        }
536        request->finish();
537        m_sizeAdvice = ~0ull;
538        return;
539    }
540    if (parent()) {
541        parent()->close();
542    }
543
544    if (parent() && (type & WRITE) && parent()->supportsWrite()) {
545        parent(Stream::ptr());
546        m_writeFuture.reset();
547        m_writeFuture2.signal();
548        m_writeFuture.wait();
549        if (m_writeException)
550            Mordor::rethrow_exception(m_writeException);
551        MORDOR_ASSERT(m_writeRequest);
552        switch ((int)m_writeRequest->response().status.status) {
553            case OK:
554            case CREATED:
555            case 207: // Partial Update OK, from http://www.hpl.hp.com/personal/ange/archives/archives-97/http-wg-archive/2530.html
556                m_response = m_writeRequest->response();
557                m_hasResponse = true;
558                try {
559                    m_writeRequest->finish();
560                } catch (...) {
561                    m_writeRequest.reset();
562                    throw;
563                }
564                m_writeRequest.reset();
565                break;
566            default:
567                MORDOR_THROW_EXCEPTION(InvalidResponseException(m_writeRequest));
568        }
569    }
570    parent(Stream::ptr());
571}
572
573void
574HTTPStream::flush(bool flushParent)
575{
576    if (parent() && parent()->supportsWrite())
577        parent()->flush(flushParent);
578}
579
580void
581HTTPStream::clearParent(bool error)
582{
583    if (m_writeInProgress) {
584        m_abortWrite = true;
585        m_writeFuture.reset();
586        m_writeFuture2.signal();
587        m_writeFuture.wait();
588        MORDOR_ASSERT(!m_writeInProgress);
589    }
590    if (parent()) {
591        if (parent()->supportsWrite() && m_writeRequest) {
592            if (error)
593                m_writeRequest->cancel(true);
594            else
595                m_writeRequest->finish();
596            m_writeRequest.reset();
597        }
598        if (parent()->supportsRead() && m_readRequest) {
599            if (error)
600                m_readRequest->cancel(true);
601            else
602                m_readRequest->finish();
603            m_readRequest.reset();
604        }
605        parent(Stream::ptr());
606    }
607}
608
609}