/mordor/streams/http.cpp
C++ | 609 lines | 553 code | 31 blank | 25 comment | 138 complexity | 3e4f93e3e22c22d0e11dca1d727f8b4d MD5 | raw file
1// Copyright (c) 2009 - Mozy, Inc. 2 3#include "http.h" 4 5#include "mordor/fiber.h" 6#include "mordor/http/client.h" 7#include "mordor/socket.h" 8#include "null.h" 9#include "transfer.h" 10 11using namespace Mordor::HTTP; 12 13namespace Mordor { 14 15HTTPStream::HTTPStream(const URI &uri, RequestBroker::ptr requestBroker, 16 boost::function<bool (size_t)> delayDg) 17: FilterStream(Stream::ptr(), false), 18 m_hasResponse(false), 19 m_requestBroker(requestBroker), 20 m_pos(0), 21 m_size(-1), 22 m_sizeAdvice(-1), 23 m_readAdvice(~0ull), 24 m_writeAdvice(~0ull), 25 m_readRequested(0ull), 26 m_writeRequested(0ull), 27 m_delayDg(delayDg), 28 mp_retries(NULL), 29 m_writeInProgress(false), 30 m_abortWrite(false) 31{ 32 m_requestHeaders.requestLine.uri = uri; 33} 34 35HTTPStream::HTTPStream(const Request &requestHeaders, RequestBroker::ptr requestBroker, 36 boost::function<bool (size_t)> delayDg) 37: FilterStream(Stream::ptr(), false), 38 m_requestHeaders(requestHeaders), 39 m_hasResponse(false), 40 m_requestBroker(requestBroker), 41 m_pos(0), 42 m_size(-1), 43 m_sizeAdvice(-1), 44 m_readAdvice(~0ull), 45 m_writeAdvice(~0ull), 46 m_readRequested(0ull), 47 m_writeRequested(0ull), 48 m_delayDg(delayDg), 49 mp_retries(NULL), 50 m_writeInProgress(false), 51 m_abortWrite(false) 52{} 53 54HTTPStream::~HTTPStream() 55{ 56 try { 57 clearParent(); 58 } catch (...) { 59 // ignore clean up errors 60 } 61} 62 63ETag 64HTTPStream::eTag() 65{ 66 stat(); 67 return m_eTag; 68} 69 70void 71HTTPStream::start() 72{ 73 start(m_readAdvice == 0ull ? (size_t)-1 : 0u); 74} 75 76void 77HTTPStream::start(size_t length) 78{ 79 if (parent() && !parent()->supportsRead()) 80 clearParent(); 81 if (!parent() || m_readRequested == 0) { 82 if (parent()) { 83 // We've read everything, but haven't triggered EOF yet 84 MORDOR_ASSERT(m_readRequested == 0); 85 try { 86 char byte; 87 MORDOR_VERIFY(parent()->read(&byte, 1) == 0); 88 } catch (...) { 89 // Ignore errors finishing the request 90 } 91 } 92 // Don't bother doing a request that will never do anything 93 if (m_size >= 0 && m_pos >= m_size) { 94 parent(NullStream::get_ptr()); 95 return; 96 } 97 m_requestHeaders.requestLine.method = GET; 98 m_requestHeaders.request.ifNoneMatch.clear(); 99 m_requestHeaders.general.transferEncoding.clear(); 100 m_requestHeaders.entity.contentLength = ~0ull; 101 m_requestHeaders.entity.contentRange = ContentRange(); 102 if (!m_eTag.unspecified) { 103 if (m_pos != 0) { 104 m_requestHeaders.request.ifRange = m_eTag; 105 m_requestHeaders.request.ifMatch.clear(); 106 } else { 107 m_requestHeaders.request.ifMatch.insert(m_eTag); 108 m_requestHeaders.request.ifRange = ETag(); 109 } 110 } else { 111 m_requestHeaders.request.ifRange = ETag(); 112 m_requestHeaders.request.ifMatch.clear(); 113 } 114 m_requestHeaders.request.range.clear(); 115 if (m_pos != 0 || (m_readAdvice != ~0ull && length != (size_t)-1)) { 116 if (length == (size_t)-1) 117 m_readRequested = ~0ull; 118 else 119 m_readRequested = (std::max)((unsigned long long)length, 120 m_readAdvice); 121 m_requestHeaders.request.range.push_back( 122 std::make_pair((unsigned long long)m_pos, 123 m_readRequested == ~0ull ? ~0ull : m_pos + m_readRequested - 1)); 124 } else { 125 m_readRequested = ~0ull; 126 } 127 ClientRequest::ptr request = m_requestBroker->request(m_requestHeaders); 128 const Response &response = request->response(); 129 Stream::ptr responseStream; 130 switch (response.status.status) { 131 case OK: 132 case PARTIAL_CONTENT: 133 m_response = response; 134 m_hasResponse = true; 135 if (response.entity.contentRange.instance != ~0ull) 136 m_size = (long long)response.entity.contentRange.instance; 137 else if (response.entity.contentLength != ~0ull) 138 m_size = (long long)response.entity.contentLength; 139 else 140 m_size = -2; 141 if (m_eTag.unspecified && 142 !response.response.eTag.unspecified) { 143 m_eTag = response.response.eTag; 144 } else if (!m_eTag.unspecified && 145 !response.response.eTag.unspecified && 146 response.response.eTag != m_eTag) { 147 m_eTag = response.response.eTag; 148 if (m_pos != 0 && response.status.status == 149 PARTIAL_CONTENT) { 150 // Server doesn't support If-Range 151 request->cancel(true); 152 clearParent(); 153 } else { 154 m_readRequest = request; 155 parent(request->responseStream()); 156 } 157 m_pos = 0; 158 MORDOR_THROW_EXCEPTION(EntityChangedException()); 159 } 160 responseStream = request->responseStream(); 161 // Server doesn't support Range 162 if (m_pos != 0 && response.status.status == OK) { 163 m_readRequested = ~0ull; 164 transferStream(responseStream, NullStream::get(), m_pos); 165 } 166 m_readRequest = request; 167 parent(responseStream); 168 break; 169 default: 170 // The bad ClientRequest `request' is propagated along with the 171 // InvalidResponseException, the uppper level should handle the 172 // error and finish the request, fail to do so will cause the 173 // corresponding ClientConnection abortion and all the remaining 174 // requests on the connection will be canceled. 175 MORDOR_THROW_EXCEPTION(InvalidResponseException(request)); 176 } 177 } 178} 179 180bool 181HTTPStream::checkModified() 182{ 183 MORDOR_ASSERT(!m_eTag.unspecified); 184 if (parent()) 185 clearParent(); 186 187 m_requestHeaders.requestLine.method = GET; 188 m_requestHeaders.request.ifMatch.clear(); 189 m_requestHeaders.request.ifRange = ETag(); 190 m_requestHeaders.request.range.clear(); 191 m_requestHeaders.request.ifNoneMatch.insert(m_eTag); 192 m_requestHeaders.general.transferEncoding.clear(); 193 m_requestHeaders.entity.contentLength = ~0ull; 194 m_requestHeaders.entity.contentRange = ContentRange(); 195 if (m_pos != 0 || m_readAdvice != ~0ull) { 196 m_readRequested = m_readAdvice == 0ull ? ~0ull : m_readAdvice; 197 m_requestHeaders.request.range.push_back( 198 std::make_pair((unsigned long long)m_pos, 199 m_readRequested == ~0ull ? ~0ull : m_pos + m_readRequested - 1)); 200 } else { 201 m_readRequested = ~0ull; 202 } 203 ClientRequest::ptr request = m_requestBroker->request(m_requestHeaders); 204 const Response &response = request->response(); 205 switch (response.status.status) { 206 case OK: 207 case PARTIAL_CONTENT: 208 case NOT_MODIFIED: 209 m_response = response; 210 m_hasResponse = true; 211 if (response.entity.contentRange.instance != ~0ull) 212 m_size = (long long)response.entity.contentRange.instance; 213 else if (response.entity.contentLength != ~0ull) 214 m_size = (long long)response.entity.contentLength; 215 else 216 m_size = -2; 217 break; 218 default: 219 MORDOR_THROW_EXCEPTION(InvalidResponseException(request)); 220 } 221 if (response.status.status != NOT_MODIFIED) 222 m_eTag = response.response.eTag; 223 else 224 return false; 225 Stream::ptr responseStream = request->responseStream(); 226 227 // Server doesn't support Range 228 if (m_pos != 0 && response.status.status == OK) { 229 // We don't really care about any data transfer problems here, 230 // since that's not what the caller is asking for 231 try { 232 m_readRequested = ~0ull; 233 transferStream(responseStream, NullStream::get(), m_pos); 234 } catch (...) { 235 return true; 236 } 237 } 238 m_readRequest = request; 239 parent(responseStream); 240 return true; 241} 242 243const HTTP::Response & 244HTTPStream::response() 245{ 246 if (m_hasResponse) 247 return m_response; 248 // NOTE: if m_writeInProgress == true, the write operation will be aborted 249 stat(); 250 return m_response; 251} 252 253size_t 254HTTPStream::read(Buffer &buffer, size_t length) 255{ 256 size_t localRetries = 0; 257 size_t *retries = mp_retries ? mp_retries : &localRetries; 258 while (true) { 259 // remember previous retry count 260 size_t prevRetries = *retries; 261 bool new_request = !parent(); 262 start(length); 263 264 MORDOR_ASSERT(parent()); 265 try { 266 try { 267 size_t result = parent()->read(buffer, length); 268 m_pos += result; 269 m_readRequested -= result; 270 if (result > 0) 271 *retries = 0; // only do this if we've made progress 272 return result; 273 } catch(...) { 274 // if this was a new request, RetryRequestBroker reset the retry count to 0 275 // undo this, so we don't get stuck in an infinite retry loop if the server 276 // consistently hangs up on us immediately after 200 OK 277 if (new_request) 278 *retries = prevRetries; 279 throw; 280 } 281 } catch (SocketException &) { 282 clearParent(true); 283 if (!m_delayDg || !m_delayDg(++*retries)) 284 throw; 285 continue; 286 } catch (UnexpectedEofException &) { 287 clearParent(true); 288 if (!m_delayDg || !m_delayDg(++*retries)) 289 throw; 290 continue; 291 } 292 } 293} 294 295void 296HTTPStream::doWrite(ClientRequest::ptr request) 297{ 298 parent(request->requestStream()); 299 m_writeFuture2.reset(); 300 m_writeFuture.signal(); 301 m_writeFuture2.wait(); 302 if (m_abortWrite) 303 MORDOR_THROW_EXCEPTION(OperationAbortedException()); 304 MORDOR_ASSERT(!parent()); 305} 306 307void 308HTTPStream::startWrite() 309{ 310 try { 311 m_writeRequest = m_requestBroker->request(m_requestHeaders, 312 false, boost::bind(&HTTPStream::doWrite, this, _1)); 313 m_writeInProgress = false; 314 m_writeFuture.signal(); 315 } catch (OperationAbortedException &) { 316 if (!m_abortWrite) 317 m_writeException = boost::current_exception(); 318 m_writeInProgress = false; 319 m_writeFuture.signal(); 320 } catch (...) { 321 m_writeException = boost::current_exception(); 322 m_writeInProgress = false; 323 m_writeFuture.signal(); 324 } 325} 326 327size_t 328HTTPStream::write(const Buffer &buffer, size_t length) 329{ 330 MORDOR_ASSERT(m_sizeAdvice == -1 || 331 (unsigned long long)m_pos + length <= (unsigned long long)m_sizeAdvice); 332 if (parent() && !parent()->supportsWrite()) 333 clearParent(); 334 if (!parent()) { 335 m_requestHeaders.requestLine.method = PUT; 336 m_requestHeaders.request.ifMatch.clear(); 337 m_requestHeaders.request.ifRange = ETag(); 338 m_requestHeaders.request.range.clear(); 339 m_requestHeaders.request.ifNoneMatch.clear(); 340 341 if (m_pos != 0 || m_writeAdvice != ~0ull) { 342 m_requestHeaders.entity.contentRange.first = m_pos; 343 if (m_sizeAdvice != -1) { 344 m_writeRequested = (std::max)((unsigned long long)length, 345 m_writeAdvice); 346 m_writeRequested = (std::min)(m_pos + m_writeRequested, 347 (unsigned long long)m_sizeAdvice) - m_pos; 348 m_requestHeaders.entity.contentLength = m_writeRequested; 349 m_requestHeaders.entity.contentRange.last = m_pos + 350 m_writeRequested - 1; 351 } else { 352 m_requestHeaders.entity.contentLength = ~0ull; 353 m_requestHeaders.entity.contentRange.last = ~0ull; 354 m_writeRequested = m_writeAdvice; 355 } 356 m_requestHeaders.entity.contentRange.instance = 357 (unsigned long long)m_sizeAdvice; 358 } else { 359 m_requestHeaders.entity.contentLength = 360 (unsigned long long)m_sizeAdvice; 361 m_requestHeaders.entity.contentRange = ContentRange(); 362 m_writeRequested = m_sizeAdvice; 363 } 364 if (m_sizeAdvice == -1) { 365 if (m_requestHeaders.general.transferEncoding.empty()) 366 m_requestHeaders.general.transferEncoding.push_back("chunked"); 367 } else { 368 m_requestHeaders.general.transferEncoding.clear(); 369 } 370 371 m_writeException = boost::exception_ptr(); 372 m_writeFuture.reset(); 373 // Have to schedule this because RequestBroker::request doesn't return 374 // until the entire request is complete 375 m_writeInProgress = true; 376 MORDOR_ASSERT(Scheduler::getThis()); 377 Scheduler::getThis()->schedule( 378 boost::bind(&HTTPStream::startWrite, this)); 379 m_writeFuture.wait(); 380 if (m_writeException) 381 Mordor::rethrow_exception(m_writeException); 382 MORDOR_ASSERT(parent()); 383 } 384 size_t result = parent()->write(buffer, 385 (size_t)(std::min)((unsigned long long)length, m_writeRequested)); 386 m_pos += result; 387 m_writeRequested -= result; 388 if (m_writeRequested == 0) 389 close(WRITE); 390 return result; 391} 392 393long long 394HTTPStream::seek(long long offset, Anchor anchor) 395{ 396 switch (anchor) { 397 case CURRENT: 398 offset = m_pos + offset; 399 break; 400 case END: 401 stat(); 402 MORDOR_ASSERT(m_size >= 0); 403 offset = m_size + offset; 404 break; 405 case BEGIN: 406 break; 407 default: 408 MORDOR_NOTREACHED(); 409 } 410 if (offset < 0) 411 MORDOR_THROW_EXCEPTION(std::invalid_argument( 412 "resulting offset is before the beginning of the file")); 413 if (offset == m_pos) 414 return m_pos; 415 // Attempt to do an optimized forward seek 416 if (offset > m_pos && m_readRequested != ~0ull && parent() && 417 parent()->supportsRead() && 418 m_pos + m_readRequested > (unsigned long long)offset) { 419 try { 420 transferStream(*this, NullStream::get(), offset - m_pos); 421 MORDOR_ASSERT(m_pos == offset); 422 return m_pos; 423 } catch (...) { 424 clearParent(true); 425 } 426 } else { 427 clearParent(); 428 } 429 return m_pos = offset; 430} 431 432bool 433HTTPStream::supportsSize() 434{ 435 stat(); 436 MORDOR_ASSERT(m_size != -1); 437 MORDOR_ASSERT(m_size >= -2); 438 if (m_size == -2) 439 return false; 440 else 441 return true; 442} 443 444void 445HTTPStream::stat() 446{ 447 if (m_size == -1) { 448 clearParent(); 449 m_requestHeaders.requestLine.method = HEAD; 450 m_requestHeaders.request.ifMatch.clear(); 451 m_requestHeaders.request.ifRange = ETag(); 452 m_requestHeaders.request.range.clear(); 453 m_requestHeaders.request.ifNoneMatch.clear(); 454 m_requestHeaders.general.transferEncoding.clear(); 455 m_requestHeaders.entity.contentLength = ~0ull; 456 m_requestHeaders.entity.contentRange = ContentRange(); 457 ClientRequest::ptr request = m_requestBroker->request(m_requestHeaders); 458 const Response &response = request->response(); 459 switch (response.status.status) { 460 case OK: 461 m_response = response; 462 m_hasResponse = true; 463 if (response.entity.contentLength != ~0ull) 464 m_size = (long long)response.entity.contentLength; 465 else 466 m_size = -2; 467 if (m_eTag.unspecified && 468 !response.response.eTag.unspecified) { 469 m_eTag = response.response.eTag; 470 } else if (!m_eTag.unspecified && 471 response.response.eTag != m_eTag) { 472 m_eTag = response.response.eTag; 473 clearParent(); 474 m_pos = 0; 475 MORDOR_THROW_EXCEPTION(EntityChangedException()); 476 } 477 break; 478 default: 479 MORDOR_THROW_EXCEPTION(InvalidResponseException(request)); 480 } 481 } 482} 483 484long long 485HTTPStream::size() 486{ 487 stat(); 488 MORDOR_ASSERT(supportsSize()); 489 return m_size; 490} 491 492void 493HTTPStream::truncate(long long size) 494{ 495 if (parent()) 496 clearParent(); 497 m_requestHeaders.requestLine.method = PUT; 498 m_requestHeaders.request.ifMatch.clear(); 499 m_requestHeaders.request.ifRange = ETag(); 500 m_requestHeaders.request.range.clear(); 501 m_requestHeaders.request.ifNoneMatch.clear(); 502 m_requestHeaders.general.transferEncoding.clear(); 503 m_requestHeaders.entity.contentLength = 0; 504 m_requestHeaders.entity.contentRange = ContentRange(~0ull, ~0ull, size); 505 ClientRequest::ptr request = m_requestBroker->request(m_requestHeaders); 506 if (request->response().status.status != OK) 507 MORDOR_THROW_EXCEPTION(InvalidResponseException(request)); 508 request->finish(); 509} 510 511void 512HTTPStream::close(CloseType type) 513{ 514 if ((type & WRITE) && m_sizeAdvice == 0ll) { 515 // If we weren't supposed to write anything, we shouldn't have written 516 // anything :) 517 MORDOR_ASSERT(!parent()); 518 m_requestHeaders.requestLine.method = PUT; 519 m_requestHeaders.request.ifMatch.clear(); 520 m_requestHeaders.request.ifRange = ETag(); 521 m_requestHeaders.request.range.clear(); 522 m_requestHeaders.request.ifNoneMatch.clear(); 523 m_requestHeaders.general.transferEncoding.clear(); 524 m_requestHeaders.entity.contentLength = 0; 525 m_requestHeaders.entity.contentRange = ContentRange(); 526 ClientRequest::ptr request = m_requestBroker->request(m_requestHeaders); 527 switch (request->response().status.status) { 528 case OK: 529 case CREATED: 530 m_response = request->response(); 531 m_hasResponse = true; 532 break; 533 default: 534 MORDOR_THROW_EXCEPTION(InvalidResponseException(m_writeRequest)); 535 } 536 request->finish(); 537 m_sizeAdvice = ~0ull; 538 return; 539 } 540 if (parent()) { 541 parent()->close(); 542 } 543 544 if (parent() && (type & WRITE) && parent()->supportsWrite()) { 545 parent(Stream::ptr()); 546 m_writeFuture.reset(); 547 m_writeFuture2.signal(); 548 m_writeFuture.wait(); 549 if (m_writeException) 550 Mordor::rethrow_exception(m_writeException); 551 MORDOR_ASSERT(m_writeRequest); 552 switch ((int)m_writeRequest->response().status.status) { 553 case OK: 554 case CREATED: 555 case 207: // Partial Update OK, from http://www.hpl.hp.com/personal/ange/archives/archives-97/http-wg-archive/2530.html 556 m_response = m_writeRequest->response(); 557 m_hasResponse = true; 558 try { 559 m_writeRequest->finish(); 560 } catch (...) { 561 m_writeRequest.reset(); 562 throw; 563 } 564 m_writeRequest.reset(); 565 break; 566 default: 567 MORDOR_THROW_EXCEPTION(InvalidResponseException(m_writeRequest)); 568 } 569 } 570 parent(Stream::ptr()); 571} 572 573void 574HTTPStream::flush(bool flushParent) 575{ 576 if (parent() && parent()->supportsWrite()) 577 parent()->flush(flushParent); 578} 579 580void 581HTTPStream::clearParent(bool error) 582{ 583 if (m_writeInProgress) { 584 m_abortWrite = true; 585 m_writeFuture.reset(); 586 m_writeFuture2.signal(); 587 m_writeFuture.wait(); 588 MORDOR_ASSERT(!m_writeInProgress); 589 } 590 if (parent()) { 591 if (parent()->supportsWrite() && m_writeRequest) { 592 if (error) 593 m_writeRequest->cancel(true); 594 else 595 m_writeRequest->finish(); 596 m_writeRequest.reset(); 597 } 598 if (parent()->supportsRead() && m_readRequest) { 599 if (error) 600 m_readRequest->cancel(true); 601 else 602 m_readRequest->finish(); 603 m_readRequest.reset(); 604 } 605 parent(Stream::ptr()); 606 } 607} 608 609}