PageRenderTime 86ms CodeModel.GetById 17ms RepoModel.GetById 1ms app.codeStats 0ms

/mordor/streams/http.cpp

http://github.com/mozy/mordor
C++ | 609 lines | 553 code | 31 blank | 25 comment | 138 complexity | 3e4f93e3e22c22d0e11dca1d727f8b4d MD5 | raw file
Possible License(s): BSD-3-Clause
  1. // Copyright (c) 2009 - Mozy, Inc.
  2. #include "http.h"
  3. #include "mordor/fiber.h"
  4. #include "mordor/http/client.h"
  5. #include "mordor/socket.h"
  6. #include "null.h"
  7. #include "transfer.h"
  8. using namespace Mordor::HTTP;
  9. namespace Mordor {
  10. HTTPStream::HTTPStream(const URI &uri, RequestBroker::ptr requestBroker,
  11. boost::function<bool (size_t)> delayDg)
  12. : FilterStream(Stream::ptr(), false),
  13. m_hasResponse(false),
  14. m_requestBroker(requestBroker),
  15. m_pos(0),
  16. m_size(-1),
  17. m_sizeAdvice(-1),
  18. m_readAdvice(~0ull),
  19. m_writeAdvice(~0ull),
  20. m_readRequested(0ull),
  21. m_writeRequested(0ull),
  22. m_delayDg(delayDg),
  23. mp_retries(NULL),
  24. m_writeInProgress(false),
  25. m_abortWrite(false)
  26. {
  27. m_requestHeaders.requestLine.uri = uri;
  28. }
  29. HTTPStream::HTTPStream(const Request &requestHeaders, RequestBroker::ptr requestBroker,
  30. boost::function<bool (size_t)> delayDg)
  31. : FilterStream(Stream::ptr(), false),
  32. m_requestHeaders(requestHeaders),
  33. m_hasResponse(false),
  34. m_requestBroker(requestBroker),
  35. m_pos(0),
  36. m_size(-1),
  37. m_sizeAdvice(-1),
  38. m_readAdvice(~0ull),
  39. m_writeAdvice(~0ull),
  40. m_readRequested(0ull),
  41. m_writeRequested(0ull),
  42. m_delayDg(delayDg),
  43. mp_retries(NULL),
  44. m_writeInProgress(false),
  45. m_abortWrite(false)
  46. {}
  47. HTTPStream::~HTTPStream()
  48. {
  49. try {
  50. clearParent();
  51. } catch (...) {
  52. // ignore clean up errors
  53. }
  54. }
  55. ETag
  56. HTTPStream::eTag()
  57. {
  58. stat();
  59. return m_eTag;
  60. }
  61. void
  62. HTTPStream::start()
  63. {
  64. start(m_readAdvice == 0ull ? (size_t)-1 : 0u);
  65. }
  66. void
  67. HTTPStream::start(size_t length)
  68. {
  69. if (parent() && !parent()->supportsRead())
  70. clearParent();
  71. if (!parent() || m_readRequested == 0) {
  72. if (parent()) {
  73. // We've read everything, but haven't triggered EOF yet
  74. MORDOR_ASSERT(m_readRequested == 0);
  75. try {
  76. char byte;
  77. MORDOR_VERIFY(parent()->read(&byte, 1) == 0);
  78. } catch (...) {
  79. // Ignore errors finishing the request
  80. }
  81. }
  82. // Don't bother doing a request that will never do anything
  83. if (m_size >= 0 && m_pos >= m_size) {
  84. parent(NullStream::get_ptr());
  85. return;
  86. }
  87. m_requestHeaders.requestLine.method = GET;
  88. m_requestHeaders.request.ifNoneMatch.clear();
  89. m_requestHeaders.general.transferEncoding.clear();
  90. m_requestHeaders.entity.contentLength = ~0ull;
  91. m_requestHeaders.entity.contentRange = ContentRange();
  92. if (!m_eTag.unspecified) {
  93. if (m_pos != 0) {
  94. m_requestHeaders.request.ifRange = m_eTag;
  95. m_requestHeaders.request.ifMatch.clear();
  96. } else {
  97. m_requestHeaders.request.ifMatch.insert(m_eTag);
  98. m_requestHeaders.request.ifRange = ETag();
  99. }
  100. } else {
  101. m_requestHeaders.request.ifRange = ETag();
  102. m_requestHeaders.request.ifMatch.clear();
  103. }
  104. m_requestHeaders.request.range.clear();
  105. if (m_pos != 0 || (m_readAdvice != ~0ull && length != (size_t)-1)) {
  106. if (length == (size_t)-1)
  107. m_readRequested = ~0ull;
  108. else
  109. m_readRequested = (std::max)((unsigned long long)length,
  110. m_readAdvice);
  111. m_requestHeaders.request.range.push_back(
  112. std::make_pair((unsigned long long)m_pos,
  113. m_readRequested == ~0ull ? ~0ull : m_pos + m_readRequested - 1));
  114. } else {
  115. m_readRequested = ~0ull;
  116. }
  117. ClientRequest::ptr request = m_requestBroker->request(m_requestHeaders);
  118. const Response &response = request->response();
  119. Stream::ptr responseStream;
  120. switch (response.status.status) {
  121. case OK:
  122. case PARTIAL_CONTENT:
  123. m_response = response;
  124. m_hasResponse = true;
  125. if (response.entity.contentRange.instance != ~0ull)
  126. m_size = (long long)response.entity.contentRange.instance;
  127. else if (response.entity.contentLength != ~0ull)
  128. m_size = (long long)response.entity.contentLength;
  129. else
  130. m_size = -2;
  131. if (m_eTag.unspecified &&
  132. !response.response.eTag.unspecified) {
  133. m_eTag = response.response.eTag;
  134. } else if (!m_eTag.unspecified &&
  135. !response.response.eTag.unspecified &&
  136. response.response.eTag != m_eTag) {
  137. m_eTag = response.response.eTag;
  138. if (m_pos != 0 && response.status.status ==
  139. PARTIAL_CONTENT) {
  140. // Server doesn't support If-Range
  141. request->cancel(true);
  142. clearParent();
  143. } else {
  144. m_readRequest = request;
  145. parent(request->responseStream());
  146. }
  147. m_pos = 0;
  148. MORDOR_THROW_EXCEPTION(EntityChangedException());
  149. }
  150. responseStream = request->responseStream();
  151. // Server doesn't support Range
  152. if (m_pos != 0 && response.status.status == OK) {
  153. m_readRequested = ~0ull;
  154. transferStream(responseStream, NullStream::get(), m_pos);
  155. }
  156. m_readRequest = request;
  157. parent(responseStream);
  158. break;
  159. default:
  160. // The bad ClientRequest `request' is propagated along with the
  161. // InvalidResponseException, the uppper level should handle the
  162. // error and finish the request, fail to do so will cause the
  163. // corresponding ClientConnection abortion and all the remaining
  164. // requests on the connection will be canceled.
  165. MORDOR_THROW_EXCEPTION(InvalidResponseException(request));
  166. }
  167. }
  168. }
  169. bool
  170. HTTPStream::checkModified()
  171. {
  172. MORDOR_ASSERT(!m_eTag.unspecified);
  173. if (parent())
  174. clearParent();
  175. m_requestHeaders.requestLine.method = GET;
  176. m_requestHeaders.request.ifMatch.clear();
  177. m_requestHeaders.request.ifRange = ETag();
  178. m_requestHeaders.request.range.clear();
  179. m_requestHeaders.request.ifNoneMatch.insert(m_eTag);
  180. m_requestHeaders.general.transferEncoding.clear();
  181. m_requestHeaders.entity.contentLength = ~0ull;
  182. m_requestHeaders.entity.contentRange = ContentRange();
  183. if (m_pos != 0 || m_readAdvice != ~0ull) {
  184. m_readRequested = m_readAdvice == 0ull ? ~0ull : m_readAdvice;
  185. m_requestHeaders.request.range.push_back(
  186. std::make_pair((unsigned long long)m_pos,
  187. m_readRequested == ~0ull ? ~0ull : m_pos + m_readRequested - 1));
  188. } else {
  189. m_readRequested = ~0ull;
  190. }
  191. ClientRequest::ptr request = m_requestBroker->request(m_requestHeaders);
  192. const Response &response = request->response();
  193. switch (response.status.status) {
  194. case OK:
  195. case PARTIAL_CONTENT:
  196. case NOT_MODIFIED:
  197. m_response = response;
  198. m_hasResponse = true;
  199. if (response.entity.contentRange.instance != ~0ull)
  200. m_size = (long long)response.entity.contentRange.instance;
  201. else if (response.entity.contentLength != ~0ull)
  202. m_size = (long long)response.entity.contentLength;
  203. else
  204. m_size = -2;
  205. break;
  206. default:
  207. MORDOR_THROW_EXCEPTION(InvalidResponseException(request));
  208. }
  209. if (response.status.status != NOT_MODIFIED)
  210. m_eTag = response.response.eTag;
  211. else
  212. return false;
  213. Stream::ptr responseStream = request->responseStream();
  214. // Server doesn't support Range
  215. if (m_pos != 0 && response.status.status == OK) {
  216. // We don't really care about any data transfer problems here,
  217. // since that's not what the caller is asking for
  218. try {
  219. m_readRequested = ~0ull;
  220. transferStream(responseStream, NullStream::get(), m_pos);
  221. } catch (...) {
  222. return true;
  223. }
  224. }
  225. m_readRequest = request;
  226. parent(responseStream);
  227. return true;
  228. }
  229. const HTTP::Response &
  230. HTTPStream::response()
  231. {
  232. if (m_hasResponse)
  233. return m_response;
  234. // NOTE: if m_writeInProgress == true, the write operation will be aborted
  235. stat();
  236. return m_response;
  237. }
  238. size_t
  239. HTTPStream::read(Buffer &buffer, size_t length)
  240. {
  241. size_t localRetries = 0;
  242. size_t *retries = mp_retries ? mp_retries : &localRetries;
  243. while (true) {
  244. // remember previous retry count
  245. size_t prevRetries = *retries;
  246. bool new_request = !parent();
  247. start(length);
  248. MORDOR_ASSERT(parent());
  249. try {
  250. try {
  251. size_t result = parent()->read(buffer, length);
  252. m_pos += result;
  253. m_readRequested -= result;
  254. if (result > 0)
  255. *retries = 0; // only do this if we've made progress
  256. return result;
  257. } catch(...) {
  258. // if this was a new request, RetryRequestBroker reset the retry count to 0
  259. // undo this, so we don't get stuck in an infinite retry loop if the server
  260. // consistently hangs up on us immediately after 200 OK
  261. if (new_request)
  262. *retries = prevRetries;
  263. throw;
  264. }
  265. } catch (SocketException &) {
  266. clearParent(true);
  267. if (!m_delayDg || !m_delayDg(++*retries))
  268. throw;
  269. continue;
  270. } catch (UnexpectedEofException &) {
  271. clearParent(true);
  272. if (!m_delayDg || !m_delayDg(++*retries))
  273. throw;
  274. continue;
  275. }
  276. }
  277. }
  278. void
  279. HTTPStream::doWrite(ClientRequest::ptr request)
  280. {
  281. parent(request->requestStream());
  282. m_writeFuture2.reset();
  283. m_writeFuture.signal();
  284. m_writeFuture2.wait();
  285. if (m_abortWrite)
  286. MORDOR_THROW_EXCEPTION(OperationAbortedException());
  287. MORDOR_ASSERT(!parent());
  288. }
  289. void
  290. HTTPStream::startWrite()
  291. {
  292. try {
  293. m_writeRequest = m_requestBroker->request(m_requestHeaders,
  294. false, boost::bind(&HTTPStream::doWrite, this, _1));
  295. m_writeInProgress = false;
  296. m_writeFuture.signal();
  297. } catch (OperationAbortedException &) {
  298. if (!m_abortWrite)
  299. m_writeException = boost::current_exception();
  300. m_writeInProgress = false;
  301. m_writeFuture.signal();
  302. } catch (...) {
  303. m_writeException = boost::current_exception();
  304. m_writeInProgress = false;
  305. m_writeFuture.signal();
  306. }
  307. }
  308. size_t
  309. HTTPStream::write(const Buffer &buffer, size_t length)
  310. {
  311. MORDOR_ASSERT(m_sizeAdvice == -1 ||
  312. (unsigned long long)m_pos + length <= (unsigned long long)m_sizeAdvice);
  313. if (parent() && !parent()->supportsWrite())
  314. clearParent();
  315. if (!parent()) {
  316. m_requestHeaders.requestLine.method = PUT;
  317. m_requestHeaders.request.ifMatch.clear();
  318. m_requestHeaders.request.ifRange = ETag();
  319. m_requestHeaders.request.range.clear();
  320. m_requestHeaders.request.ifNoneMatch.clear();
  321. if (m_pos != 0 || m_writeAdvice != ~0ull) {
  322. m_requestHeaders.entity.contentRange.first = m_pos;
  323. if (m_sizeAdvice != -1) {
  324. m_writeRequested = (std::max)((unsigned long long)length,
  325. m_writeAdvice);
  326. m_writeRequested = (std::min)(m_pos + m_writeRequested,
  327. (unsigned long long)m_sizeAdvice) - m_pos;
  328. m_requestHeaders.entity.contentLength = m_writeRequested;
  329. m_requestHeaders.entity.contentRange.last = m_pos +
  330. m_writeRequested - 1;
  331. } else {
  332. m_requestHeaders.entity.contentLength = ~0ull;
  333. m_requestHeaders.entity.contentRange.last = ~0ull;
  334. m_writeRequested = m_writeAdvice;
  335. }
  336. m_requestHeaders.entity.contentRange.instance =
  337. (unsigned long long)m_sizeAdvice;
  338. } else {
  339. m_requestHeaders.entity.contentLength =
  340. (unsigned long long)m_sizeAdvice;
  341. m_requestHeaders.entity.contentRange = ContentRange();
  342. m_writeRequested = m_sizeAdvice;
  343. }
  344. if (m_sizeAdvice == -1) {
  345. if (m_requestHeaders.general.transferEncoding.empty())
  346. m_requestHeaders.general.transferEncoding.push_back("chunked");
  347. } else {
  348. m_requestHeaders.general.transferEncoding.clear();
  349. }
  350. m_writeException = boost::exception_ptr();
  351. m_writeFuture.reset();
  352. // Have to schedule this because RequestBroker::request doesn't return
  353. // until the entire request is complete
  354. m_writeInProgress = true;
  355. MORDOR_ASSERT(Scheduler::getThis());
  356. Scheduler::getThis()->schedule(
  357. boost::bind(&HTTPStream::startWrite, this));
  358. m_writeFuture.wait();
  359. if (m_writeException)
  360. Mordor::rethrow_exception(m_writeException);
  361. MORDOR_ASSERT(parent());
  362. }
  363. size_t result = parent()->write(buffer,
  364. (size_t)(std::min)((unsigned long long)length, m_writeRequested));
  365. m_pos += result;
  366. m_writeRequested -= result;
  367. if (m_writeRequested == 0)
  368. close(WRITE);
  369. return result;
  370. }
  371. long long
  372. HTTPStream::seek(long long offset, Anchor anchor)
  373. {
  374. switch (anchor) {
  375. case CURRENT:
  376. offset = m_pos + offset;
  377. break;
  378. case END:
  379. stat();
  380. MORDOR_ASSERT(m_size >= 0);
  381. offset = m_size + offset;
  382. break;
  383. case BEGIN:
  384. break;
  385. default:
  386. MORDOR_NOTREACHED();
  387. }
  388. if (offset < 0)
  389. MORDOR_THROW_EXCEPTION(std::invalid_argument(
  390. "resulting offset is before the beginning of the file"));
  391. if (offset == m_pos)
  392. return m_pos;
  393. // Attempt to do an optimized forward seek
  394. if (offset > m_pos && m_readRequested != ~0ull && parent() &&
  395. parent()->supportsRead() &&
  396. m_pos + m_readRequested > (unsigned long long)offset) {
  397. try {
  398. transferStream(*this, NullStream::get(), offset - m_pos);
  399. MORDOR_ASSERT(m_pos == offset);
  400. return m_pos;
  401. } catch (...) {
  402. clearParent(true);
  403. }
  404. } else {
  405. clearParent();
  406. }
  407. return m_pos = offset;
  408. }
  409. bool
  410. HTTPStream::supportsSize()
  411. {
  412. stat();
  413. MORDOR_ASSERT(m_size != -1);
  414. MORDOR_ASSERT(m_size >= -2);
  415. if (m_size == -2)
  416. return false;
  417. else
  418. return true;
  419. }
  420. void
  421. HTTPStream::stat()
  422. {
  423. if (m_size == -1) {
  424. clearParent();
  425. m_requestHeaders.requestLine.method = HEAD;
  426. m_requestHeaders.request.ifMatch.clear();
  427. m_requestHeaders.request.ifRange = ETag();
  428. m_requestHeaders.request.range.clear();
  429. m_requestHeaders.request.ifNoneMatch.clear();
  430. m_requestHeaders.general.transferEncoding.clear();
  431. m_requestHeaders.entity.contentLength = ~0ull;
  432. m_requestHeaders.entity.contentRange = ContentRange();
  433. ClientRequest::ptr request = m_requestBroker->request(m_requestHeaders);
  434. const Response &response = request->response();
  435. switch (response.status.status) {
  436. case OK:
  437. m_response = response;
  438. m_hasResponse = true;
  439. if (response.entity.contentLength != ~0ull)
  440. m_size = (long long)response.entity.contentLength;
  441. else
  442. m_size = -2;
  443. if (m_eTag.unspecified &&
  444. !response.response.eTag.unspecified) {
  445. m_eTag = response.response.eTag;
  446. } else if (!m_eTag.unspecified &&
  447. response.response.eTag != m_eTag) {
  448. m_eTag = response.response.eTag;
  449. clearParent();
  450. m_pos = 0;
  451. MORDOR_THROW_EXCEPTION(EntityChangedException());
  452. }
  453. break;
  454. default:
  455. MORDOR_THROW_EXCEPTION(InvalidResponseException(request));
  456. }
  457. }
  458. }
  459. long long
  460. HTTPStream::size()
  461. {
  462. stat();
  463. MORDOR_ASSERT(supportsSize());
  464. return m_size;
  465. }
  466. void
  467. HTTPStream::truncate(long long size)
  468. {
  469. if (parent())
  470. clearParent();
  471. m_requestHeaders.requestLine.method = PUT;
  472. m_requestHeaders.request.ifMatch.clear();
  473. m_requestHeaders.request.ifRange = ETag();
  474. m_requestHeaders.request.range.clear();
  475. m_requestHeaders.request.ifNoneMatch.clear();
  476. m_requestHeaders.general.transferEncoding.clear();
  477. m_requestHeaders.entity.contentLength = 0;
  478. m_requestHeaders.entity.contentRange = ContentRange(~0ull, ~0ull, size);
  479. ClientRequest::ptr request = m_requestBroker->request(m_requestHeaders);
  480. if (request->response().status.status != OK)
  481. MORDOR_THROW_EXCEPTION(InvalidResponseException(request));
  482. request->finish();
  483. }
  484. void
  485. HTTPStream::close(CloseType type)
  486. {
  487. if ((type & WRITE) && m_sizeAdvice == 0ll) {
  488. // If we weren't supposed to write anything, we shouldn't have written
  489. // anything :)
  490. MORDOR_ASSERT(!parent());
  491. m_requestHeaders.requestLine.method = PUT;
  492. m_requestHeaders.request.ifMatch.clear();
  493. m_requestHeaders.request.ifRange = ETag();
  494. m_requestHeaders.request.range.clear();
  495. m_requestHeaders.request.ifNoneMatch.clear();
  496. m_requestHeaders.general.transferEncoding.clear();
  497. m_requestHeaders.entity.contentLength = 0;
  498. m_requestHeaders.entity.contentRange = ContentRange();
  499. ClientRequest::ptr request = m_requestBroker->request(m_requestHeaders);
  500. switch (request->response().status.status) {
  501. case OK:
  502. case CREATED:
  503. m_response = request->response();
  504. m_hasResponse = true;
  505. break;
  506. default:
  507. MORDOR_THROW_EXCEPTION(InvalidResponseException(m_writeRequest));
  508. }
  509. request->finish();
  510. m_sizeAdvice = ~0ull;
  511. return;
  512. }
  513. if (parent()) {
  514. parent()->close();
  515. }
  516. if (parent() && (type & WRITE) && parent()->supportsWrite()) {
  517. parent(Stream::ptr());
  518. m_writeFuture.reset();
  519. m_writeFuture2.signal();
  520. m_writeFuture.wait();
  521. if (m_writeException)
  522. Mordor::rethrow_exception(m_writeException);
  523. MORDOR_ASSERT(m_writeRequest);
  524. switch ((int)m_writeRequest->response().status.status) {
  525. case OK:
  526. case CREATED:
  527. case 207: // Partial Update OK, from http://www.hpl.hp.com/personal/ange/archives/archives-97/http-wg-archive/2530.html
  528. m_response = m_writeRequest->response();
  529. m_hasResponse = true;
  530. try {
  531. m_writeRequest->finish();
  532. } catch (...) {
  533. m_writeRequest.reset();
  534. throw;
  535. }
  536. m_writeRequest.reset();
  537. break;
  538. default:
  539. MORDOR_THROW_EXCEPTION(InvalidResponseException(m_writeRequest));
  540. }
  541. }
  542. parent(Stream::ptr());
  543. }
  544. void
  545. HTTPStream::flush(bool flushParent)
  546. {
  547. if (parent() && parent()->supportsWrite())
  548. parent()->flush(flushParent);
  549. }
  550. void
  551. HTTPStream::clearParent(bool error)
  552. {
  553. if (m_writeInProgress) {
  554. m_abortWrite = true;
  555. m_writeFuture.reset();
  556. m_writeFuture2.signal();
  557. m_writeFuture.wait();
  558. MORDOR_ASSERT(!m_writeInProgress);
  559. }
  560. if (parent()) {
  561. if (parent()->supportsWrite() && m_writeRequest) {
  562. if (error)
  563. m_writeRequest->cancel(true);
  564. else
  565. m_writeRequest->finish();
  566. m_writeRequest.reset();
  567. }
  568. if (parent()->supportsRead() && m_readRequest) {
  569. if (error)
  570. m_readRequest->cancel(true);
  571. else
  572. m_readRequest->finish();
  573. m_readRequest.reset();
  574. }
  575. parent(Stream::ptr());
  576. }
  577. }
  578. }