PageRenderTime 41ms CodeModel.GetById 15ms RepoModel.GetById 1ms app.codeStats 0ms

/mordor/http/broker.cpp

http://github.com/mozy/mordor
C++ | 931 lines | 814 code | 52 blank | 65 comment | 224 complexity | 46fb4558fff9fedc8f1246afbaaba82d MD5 | raw file
Possible License(s): BSD-3-Clause
  1. // Copyright (c) 2009 - Mozy, Inc.
  2. #include "broker.h"
  3. #include "auth.h"
  4. #include "client.h"
  5. #include "mordor/atomic.h"
  6. #include "mordor/fiber.h"
  7. #include "mordor/future.h"
  8. #include "mordor/iomanager.h"
  9. #include "mordor/log.h"
  10. #include "mordor/socks.h"
  11. #include "mordor/streams/buffered.h"
  12. #include "mordor/streams/pipe.h"
  13. #include "mordor/streams/socket.h"
  14. #include "mordor/streams/ssl.h"
  15. #include "mordor/streams/timeout.h"
  16. #include "proxy.h"
  17. #include "server.h"
  18. namespace Mordor {
  19. namespace HTTP {
  20. std::pair<RequestBroker::ptr, ConnectionCache::ptr>
  21. createRequestBroker(const RequestBrokerOptions &options)
  22. {
  23. TimerManager *timerManager = const_cast<TimerManager*>(options.timerManager);
  24. if (options.ioManager && !timerManager)
  25. timerManager = const_cast<IOManager*>(options.ioManager);
  26. SocketStreamBroker::ptr socketBroker(new SocketStreamBroker(options.ioManager,
  27. options.scheduler));
  28. socketBroker->connectTimeout(options.connectTimeout);
  29. socketBroker->networkFilterCallback(options.filterNetworksCB);
  30. StreamBroker::ptr streamBroker = socketBroker;
  31. if (options.customStreamBrokerFilter) {
  32. options.customStreamBrokerFilter->parent(streamBroker);
  33. streamBroker = options.customStreamBrokerFilter;
  34. }
  35. ConnectionBroker::ptr connectionBroker;
  36. ConnectionCache::ptr connectionCache;
  37. if (!options.enableConnectionCache) {
  38. ConnectionNoCache::ptr connectionNoCache(new ConnectionNoCache(streamBroker,
  39. timerManager));
  40. connectionBroker = boost::static_pointer_cast<ConnectionBroker>(connectionNoCache);
  41. } else {
  42. connectionCache = ConnectionCache::create(streamBroker, timerManager);
  43. connectionCache->idleTimeout(options.idleTimeout);
  44. connectionCache->proxyForURI(options.proxyForURIDg);
  45. connectionCache->proxyRequestBroker(options.proxyRequestBroker);
  46. connectionCache->connectionsPerHost(options.connectionsPerHost);
  47. connectionCache->requestsPerConnection(options.requestsPerConnection);
  48. connectionBroker = boost::static_pointer_cast<ConnectionBroker>(connectionCache);
  49. }
  50. connectionBroker->httpReadTimeout(options.httpReadTimeout);
  51. connectionBroker->httpWriteTimeout(options.httpWriteTimeout);
  52. connectionBroker->sslReadTimeout(options.sslConnectReadTimeout);
  53. connectionBroker->sslWriteTimeout(options.sslConnectWriteTimeout);
  54. connectionBroker->sslCtx(options.sslCtx);
  55. connectionBroker->verifySslCertificate(options.verifySslCertificate);
  56. connectionBroker->verifySslCertificateHost(options.verifySslCertificateHost);
  57. RequestBroker::ptr requestBroker(new BaseRequestBroker(connectionBroker));
  58. if (options.getCredentialsDg || options.getProxyCredentialsDg)
  59. requestBroker.reset(new AuthRequestBroker(requestBroker,
  60. options.getCredentialsDg, options.getProxyCredentialsDg));
  61. if (options.handleRedirects)
  62. requestBroker.reset(new RedirectRequestBroker(requestBroker));
  63. if (options.delayDg)
  64. requestBroker.reset(new RetryRequestBroker(requestBroker,
  65. options.delayDg));
  66. if (!options.userAgent.empty())
  67. requestBroker.reset(new UserAgentRequestBroker(requestBroker,
  68. options.userAgent));
  69. return std::make_pair(requestBroker, connectionCache);
  70. }
  71. RequestBroker::ptr defaultRequestBroker(IOManager *ioManager,
  72. Scheduler *scheduler,
  73. ConnectionBroker::ptr *connBroker,
  74. boost::function<bool (size_t)> delayDg)
  75. {
  76. RequestBrokerOptions options;
  77. options.ioManager = ioManager;
  78. options.scheduler = scheduler;
  79. options.delayDg = delayDg;
  80. std::pair<RequestBroker::ptr, ConnectionCache::ptr> result = createRequestBroker(options);
  81. if (connBroker)
  82. *connBroker = result.second;
  83. return result.first;
  84. }
  85. StreamBroker::ptr
  86. StreamBrokerFilter::parent()
  87. {
  88. if (m_parent)
  89. return m_parent;
  90. return StreamBroker::ptr(m_weakParent);
  91. }
  92. Stream::ptr
  93. SocketStreamBroker::getStream(const URI &uri)
  94. {
  95. if (m_cancelled)
  96. MORDOR_THROW_EXCEPTION(OperationAbortedException());
  97. MORDOR_ASSERT(uri.authority.hostDefined());
  98. MORDOR_ASSERT(uri.authority.portDefined() || uri.schemeDefined());
  99. std::ostringstream os;
  100. os << uri.authority.host();
  101. if (uri.authority.portDefined())
  102. os << ":" << uri.authority.port();
  103. else if (uri.schemeDefined())
  104. os << ":" << uri.scheme();
  105. std::vector<Address::ptr> addresses;
  106. {
  107. SchedulerSwitcher switcher(m_scheduler);
  108. addresses = Address::lookup(os.str());
  109. }
  110. Socket::ptr socket;
  111. for (std::vector<Address::ptr>::const_iterator it(addresses.begin());
  112. it != addresses.end();
  113. )
  114. {
  115. if (m_ioManager)
  116. socket = (*it)->createSocket(*m_ioManager, SOCK_STREAM);
  117. else
  118. socket = (*it)->createSocket(SOCK_STREAM);
  119. std::list<Socket::ptr>::iterator it2;
  120. {
  121. boost::mutex::scoped_lock lock(m_mutex);
  122. if (m_cancelled)
  123. MORDOR_THROW_EXCEPTION(OperationAbortedException());
  124. m_pending.push_back(socket);
  125. it2 = m_pending.end();
  126. --it2;
  127. }
  128. socket->sendTimeout(m_connectTimeout);
  129. try {
  130. // if we are filtering network connections, the callback will bind
  131. // the socket to an approved network address
  132. if (m_filterNetworkCallback != NULL)
  133. {
  134. // the callback function is responsible for exiting out by throwing
  135. // an exception.
  136. m_filterNetworkCallback(socket);
  137. }
  138. socket->connect(*it);
  139. boost::mutex::scoped_lock lock(m_mutex);
  140. m_pending.erase(it2);
  141. break;
  142. } catch (...) {
  143. boost::mutex::scoped_lock lock(m_mutex);
  144. m_pending.erase(it2);
  145. if (++it == addresses.end())
  146. throw;
  147. }
  148. socket->sendTimeout(~0ull);
  149. }
  150. Stream::ptr stream(new SocketStream(socket));
  151. return stream;
  152. }
  153. void
  154. SocketStreamBroker::cancelPending()
  155. {
  156. boost::mutex::scoped_lock lock(m_mutex);
  157. m_cancelled = true;
  158. for (std::list<Socket::ptr>::iterator it(m_pending.begin());
  159. it != m_pending.end();
  160. ++it) {
  161. (*it)->cancelConnect();
  162. (*it)->cancelSend();
  163. (*it)->cancelReceive();
  164. }
  165. }
  166. static bool least(const ClientConnection::ptr &lhs,
  167. const ClientConnection::ptr &rhs)
  168. {
  169. if (lhs && rhs)
  170. return lhs->outstandingRequests() <
  171. rhs->outstandingRequests();
  172. if (!lhs)
  173. return false;
  174. if (!rhs)
  175. return true;
  176. MORDOR_NOTREACHED();
  177. }
  178. static Logger::ptr g_cacheLog = Log::lookup("mordor:http:connectioncache");
  179. std::pair<ClientConnection::ptr, bool>
  180. ConnectionCache::getConnection(const URI &uri, bool forceNewConnection)
  181. {
  182. std::vector<URI> proxies;
  183. if (m_proxyForURIDg)
  184. proxies = m_proxyForURIDg(uri);
  185. // Remove proxy types that aren't supported
  186. for (std::vector<URI>::iterator it(proxies.begin());
  187. it != proxies.end();
  188. ++it) {
  189. MORDOR_ASSERT(it->schemeDefined() || !it->isDefined());
  190. if (!it->schemeDefined())
  191. continue;
  192. std::string scheme = it->scheme();
  193. if (scheme != "http" && (scheme != "https" || !m_proxyBroker) &&
  194. scheme != "socks")
  195. it = proxies.erase(it, it);
  196. }
  197. URI schemeAndAuthority;
  198. schemeAndAuthority = uri;
  199. schemeAndAuthority.path = URI::Path();
  200. schemeAndAuthority.queryDefined(false);
  201. schemeAndAuthority.fragmentDefined(false);
  202. std::pair<ClientConnection::ptr, bool> result;
  203. FiberMutex::ScopedLock lock(m_mutex);
  204. if (g_cacheLog->enabled(Log::DEBUG)) {
  205. std::ostringstream os;
  206. os << this << " getting connection for " << schemeAndAuthority
  207. << ", proxies: {";
  208. for (std::vector<URI>::iterator it(proxies.begin());
  209. it != proxies.end();
  210. ++it) {
  211. if (it != proxies.begin())
  212. os << ", ";
  213. os << *it;
  214. }
  215. os << "}";
  216. MORDOR_LOG_DEBUG(g_cacheLog) << os.str();
  217. }
  218. if (m_closed)
  219. MORDOR_THROW_EXCEPTION(OperationAbortedException());
  220. // Clean out any dead conns
  221. cleanOutDeadConns(m_conns);
  222. if (!forceNewConnection) {
  223. if (proxies.empty()) {
  224. result = getConnectionViaProxyFromCache(schemeAndAuthority, URI());
  225. if (result.first)
  226. return result;
  227. }
  228. for (std::vector<URI>::const_iterator it(proxies.begin());
  229. it != proxies.end();
  230. ++it) {
  231. result = getConnectionViaProxyFromCache(schemeAndAuthority, *it);
  232. if (result.first)
  233. return result;
  234. }
  235. }
  236. // Create a new connection
  237. if (proxies.empty())
  238. return getConnectionViaProxy(schemeAndAuthority, URI(), lock);
  239. std::vector<URI>::const_iterator it = proxies.begin();
  240. while(true) {
  241. try {
  242. return getConnectionViaProxy(schemeAndAuthority, *it, lock);
  243. } catch (SocketException &) {
  244. if (++it == proxies.end())
  245. throw;
  246. } catch (HTTP::Exception &) {
  247. if (++it == proxies.end())
  248. throw;
  249. } catch (UnexpectedEofException &) {
  250. if (++it == proxies.end())
  251. throw;
  252. }
  253. }
  254. }
  255. std::pair<ClientConnection::ptr, bool>
  256. ConnectionNoCache::getConnection(const URI &uri, bool forceNewConnection)
  257. {
  258. URI endPoint = uri;
  259. endPoint.path = URI::Path();
  260. endPoint.queryDefined(false);
  261. endPoint.fragmentDefined(false);
  262. // Establish a new connection
  263. Stream::ptr stream = m_streamBroker->getStream(endPoint);
  264. addSSL(endPoint, stream);
  265. std::pair<ClientConnection::ptr, bool> result = std::make_pair(ClientConnection::ptr(
  266. new ClientConnection(stream, m_timerManager)), false);
  267. if (m_httpReadTimeout != ~0ull)
  268. result.first->readTimeout(m_httpReadTimeout);
  269. if (m_httpWriteTimeout != ~0ull)
  270. result.first->writeTimeout(m_httpWriteTimeout);
  271. return result;
  272. }
  273. std::pair<ClientConnection::ptr, bool>
  274. ConnectionCache::getConnectionViaProxyFromCache(const URI &uri, const URI &proxy)
  275. {
  276. // Check if an existing connection exists to the requested URI
  277. // that should be reused.
  278. // When proxy is specified this looks for a connection
  279. // to the proxy uri instead
  280. bool proxied = proxy.schemeDefined() && proxy.scheme() == "http";
  281. const URI &endpoint = proxied ? proxy : uri;
  282. CachedConnectionMap::iterator it = m_conns.find(endpoint);
  283. ConnectionList::iterator it2;
  284. while (true) {
  285. if (it != m_conns.end() &&
  286. !it->second->connections.empty() &&
  287. it->second->connections.size() >= m_connectionsPerHost) {
  288. boost::shared_ptr<ConnectionInfo> info = it->second;
  289. ConnectionList &connsForThisUri = info->connections;
  290. // Assign it2 to point to the connection with the
  291. // least number of outstanding requests
  292. it2 = std::min_element(connsForThisUri.begin(),
  293. connsForThisUri.end(), &least);
  294. // No connection has completed yet (but it's in progress)
  295. if (!*it2) {
  296. MORDOR_LOG_TRACE(g_cacheLog) << this << " waiting for connection to "
  297. << endpoint;
  298. // Wait for somebody to let us try again
  299. unsigned long long start = TimerManager::now();
  300. info->condition.wait();
  301. if (info->lastFailedConnectionTimestamp <= start)
  302. MORDOR_THROW_EXCEPTION(PriorConnectionFailedException());
  303. if (m_closed)
  304. MORDOR_THROW_EXCEPTION(OperationAbortedException());
  305. // We let go of the mutex, and the last connection may have
  306. // disappeared
  307. it = m_conns.find(endpoint);
  308. } else {
  309. MORDOR_LOG_TRACE(g_cacheLog) << this << " returning cached connection "
  310. << *it2 << " to " << endpoint;
  311. // Return the existing, completed connection
  312. return std::make_pair(*it2, proxied);
  313. }
  314. } else {
  315. // No existing connections
  316. return std::make_pair(ClientConnection::ptr(), false);
  317. }
  318. }
  319. }
  320. std::pair<ClientConnection::ptr, bool>
  321. ConnectionCache::getConnectionViaProxy(const URI &uri, const URI &proxy,
  322. FiberMutex::ScopedLock &lock)
  323. {
  324. // Create a new Connection to the requested URI, using
  325. // the proxy if specified
  326. std::string proxyScheme;
  327. if (proxy.schemeDefined())
  328. proxyScheme = proxy.scheme();
  329. bool proxied = proxyScheme == "http";
  330. const URI &endpoint = proxied ? proxy : uri;
  331. // Make sure we have a ConnectionList and mutex for this endpoint
  332. CachedConnectionMap::iterator it = m_conns.find(endpoint);
  333. boost::shared_ptr<ConnectionInfo> info;
  334. if (it == m_conns.end()) {
  335. info.reset(new ConnectionInfo(m_mutex));
  336. it = m_conns.insert(std::make_pair(endpoint, info)).first;
  337. } else {
  338. info = it->second;
  339. }
  340. // Add a placeholder for the new connection
  341. info->connections.push_back(ClientConnection::ptr());
  342. MORDOR_LOG_TRACE(g_cacheLog) << this << " establishing connection to "
  343. << endpoint;
  344. unsigned long long start = TimerManager::now();
  345. lock.unlock();
  346. ConnectionList::iterator it2;
  347. std::pair<ClientConnection::ptr, bool> result;
  348. // Establish a new connection
  349. try {
  350. Stream::ptr stream;
  351. if (proxyScheme == "https") {
  352. stream = tunnel(m_proxyBroker, proxy, uri);
  353. } else if (proxyScheme == "socks") {
  354. unsigned short port;
  355. if (uri.authority.portDefined())
  356. port = uri.authority.port();
  357. else if (uri.scheme() == "http")
  358. port = 80;
  359. else if (uri.scheme() == "https")
  360. port = 443;
  361. else
  362. // TODO: can this be looked up using the system? (getaddrinfo)
  363. MORDOR_THROW_EXCEPTION(std::invalid_argument("Unknown protocol for proxying connection"));
  364. stream = SOCKS::tunnel(m_streamBroker, proxy, IPAddress::ptr(),
  365. uri.authority.host(), port);
  366. } else {
  367. stream = m_streamBroker->getStream(endpoint);
  368. }
  369. addSSL(endpoint, stream);
  370. lock.lock();
  371. // Somebody called abortConnections while we were unlocked; just throw
  372. // this connection away
  373. if (m_closed)
  374. MORDOR_THROW_EXCEPTION(OperationAbortedException());
  375. result = std::make_pair(ClientConnection::ptr(
  376. new ClientConnection(stream, m_timerManager)), proxied);
  377. MORDOR_LOG_TRACE(g_cacheLog) << this << " connection " << result.first
  378. << " to " << endpoint << " established";
  379. result.first->maxRequestCount(m_requestsPerConnection);
  380. stream->onRemoteClose(boost::bind(&ConnectionCache::dropConnection,
  381. this, weak_ptr(shared_from_this()), endpoint, result.first.get()));
  382. if (m_httpReadTimeout != ~0ull)
  383. result.first->readTimeout(m_httpReadTimeout);
  384. if (m_httpWriteTimeout != ~0ull)
  385. result.first->writeTimeout(m_httpWriteTimeout);
  386. if (m_idleTimeout != ~0ull)
  387. result.first->idleTimeout(m_idleTimeout,
  388. boost::bind(&ConnectionCache::dropConnection,
  389. this, weak_ptr(shared_from_this()), endpoint, result.first.get()));
  390. // Assign this connection to the first blank connection for this
  391. // schemeAndAuthority
  392. for (it2 = info->connections.begin();
  393. it2 != info->connections.end();
  394. ++it2) {
  395. if (!*it2) {
  396. *it2 = result.first;
  397. break;
  398. }
  399. }
  400. // We should have assigned this connection *somewhere*
  401. MORDOR_ASSERT(it2 != info->connections.end());
  402. // Unblock all waiters for them to choose an existing connection
  403. info->condition.broadcast();
  404. } catch (...) {
  405. lock.lock();
  406. MORDOR_LOG_TRACE(g_cacheLog) << this << " connection to " << endpoint
  407. << " failed: " << boost::current_exception_diagnostic_information();
  408. // Somebody called abortConnections while we were unlocked; no need to
  409. // clean up the temporary spot for this connection, since it's gone;
  410. // pass the original exception on, though
  411. if (m_closed)
  412. throw;
  413. // This connection attempt failed; remove the first blank connection
  414. // for this schemeAndAuthority to let someone else try to establish a
  415. // connection
  416. // it should still be valid, even if the map changed
  417. for (it2 = info->connections.begin();
  418. it2 != info->connections.end();
  419. ++it2) {
  420. if (!*it2) {
  421. info->connections.erase(it2);
  422. break;
  423. }
  424. }
  425. info->lastFailedConnectionTimestamp = start;
  426. info->condition.broadcast();
  427. if (info->connections.empty())
  428. m_conns.erase(it);
  429. throw;
  430. }
  431. return result;
  432. }
  433. void
  434. ConnectionCache::closeIdleConnections()
  435. {
  436. FiberMutex::ScopedLock lock(m_mutex);
  437. MORDOR_LOG_DEBUG(g_cacheLog) << " dropping idle connections";
  438. // We don't just clear the list, because there may be a connection in
  439. // progress that has an iterator into it
  440. CachedConnectionMap::iterator it, extraIt;
  441. for (it = m_conns.begin(); it != m_conns.end();) {
  442. it->second->condition.broadcast();
  443. for (ConnectionList::iterator it2 = it->second->connections.begin();
  444. it2 != it->second->connections.end();) {
  445. if (*it2) {
  446. Stream::ptr connStream = (*it2)->stream();
  447. connStream->cancelRead();
  448. connStream->cancelWrite();
  449. if (m_idleTimeout != ~0ull)
  450. (*it2)->idleTimeout(~0ull, NULL);
  451. it2 = it->second->connections.erase(it2);
  452. } else {
  453. ++it2;
  454. }
  455. }
  456. if (it->second->connections.empty()) {
  457. extraIt = it;
  458. ++it;
  459. m_conns.erase(extraIt);
  460. } else {
  461. ++it;
  462. }
  463. }
  464. }
  465. void
  466. ConnectionCache::abortConnections()
  467. {
  468. FiberMutex::ScopedLock lock(m_mutex);
  469. MORDOR_LOG_DEBUG(g_cacheLog) << " aborting all connections";
  470. m_closed = true;
  471. CachedConnectionMap::iterator it;
  472. for (it = m_conns.begin(); it != m_conns.end(); ++it) {
  473. it->second->condition.broadcast();
  474. for (ConnectionList::iterator it2 = it->second->connections.begin();
  475. it2 != it->second->connections.end();
  476. ++it2) {
  477. if (*it2) {
  478. Stream::ptr connStream = (*it2)->stream();
  479. connStream->cancelRead();
  480. connStream->cancelWrite();
  481. if (m_idleTimeout != ~0ull)
  482. (*it2)->idleTimeout(~0ull, NULL);
  483. }
  484. }
  485. }
  486. m_conns.clear();
  487. lock.unlock();
  488. m_streamBroker->cancelPending();
  489. }
  490. void
  491. ConnectionCache::cleanOutDeadConns(CachedConnectionMap &conns)
  492. {
  493. CachedConnectionMap::iterator it, it3;
  494. ConnectionList::iterator it2;
  495. for (it = conns.begin(); it != conns.end();) {
  496. for (it2 = it->second->connections.begin();
  497. it2 != it->second->connections.end();) {
  498. if (*it2 && !(*it2)->newRequestsAllowed()) {
  499. if (m_idleTimeout != ~0ull)
  500. (*it2)->idleTimeout(~0ull, NULL);
  501. it2 = it->second->connections.erase(it2);
  502. } else {
  503. ++it2;
  504. }
  505. }
  506. if (it->second->connections.empty()) {
  507. it3 = it;
  508. ++it3;
  509. conns.erase(it);
  510. it = it3;
  511. } else {
  512. ++it;
  513. }
  514. }
  515. }
  516. void
  517. ConnectionBroker::addSSL(const URI &uri, Stream::ptr &stream)
  518. {
  519. if (uri.schemeDefined() && uri.scheme() == "https") {
  520. TimeoutStream::ptr timeoutStream;
  521. if (m_timerManager) {
  522. timeoutStream.reset(new TimeoutStream(stream, *m_timerManager));
  523. timeoutStream->readTimeout(m_sslReadTimeout);
  524. timeoutStream->writeTimeout(m_sslWriteTimeout);
  525. stream = timeoutStream;
  526. }
  527. BufferedStream::ptr bufferedStream(new BufferedStream(stream));
  528. bufferedStream->allowPartialReads(true);
  529. SSLStream::ptr sslStream(new SSLStream(bufferedStream, true, true, m_sslCtx));
  530. // Only do SNI when required to verify host name
  531. if (m_verifySslCertificateHost)
  532. sslStream->serverNameIndication(uri.authority.host());
  533. sslStream->connect();
  534. if (m_verifySslCertificate)
  535. sslStream->verifyPeerCertificate();
  536. if (m_verifySslCertificateHost)
  537. sslStream->verifyPeerCertificate(uri.authority.host());
  538. if (timeoutStream) {
  539. bufferedStream->parent(timeoutStream->parent());
  540. timeoutStream.reset();
  541. }
  542. bufferedStream.reset(new BufferedStream(sslStream));
  543. // Max data in each SSL record
  544. bufferedStream->bufferSize(16384);
  545. bufferedStream->flushMultiplesOfBuffer(true);
  546. bufferedStream->allowPartialReads(true);
  547. stream = bufferedStream;
  548. }
  549. }
  550. namespace {
  551. struct CompareConn
  552. {
  553. CompareConn(const ClientConnection *conn)
  554. : m_conn(conn)
  555. {}
  556. bool operator()(const ClientConnection::ptr &lhs) const
  557. {
  558. return lhs.get() == m_conn;
  559. }
  560. const ClientConnection *m_conn;
  561. };
  562. }
  563. void
  564. ConnectionCache::dropConnection(weak_ptr self,
  565. const URI &uri,
  566. const ClientConnection *connection)
  567. {
  568. ptr strongSelf = self.lock();
  569. if (!strongSelf) {
  570. return;
  571. }
  572. FiberMutex::ScopedLock lock(m_mutex);
  573. CachedConnectionMap::iterator it = m_conns.find(uri);
  574. if (it == m_conns.end())
  575. {
  576. MORDOR_LOG_TRACE(g_cacheLog) << this << " Failed to drop connection to " << uri << "not found in our cache ";
  577. return;
  578. }
  579. ConnectionList::iterator it2 = std::find_if(it->second->connections.begin(),
  580. it->second->connections.end(), CompareConn(connection));
  581. if (it2 != it->second->connections.end()) {
  582. MORDOR_LOG_TRACE(g_cacheLog) << this << " dropping connection "
  583. << connection << " to " << uri;
  584. if (m_idleTimeout != ~0ull)
  585. (*it2)->idleTimeout(~0ull, NULL);
  586. it->second->connections.erase(it2);
  587. if (it->second->connections.empty())
  588. m_conns.erase(it);
  589. }
  590. }
  591. // Get the number of active connections
  592. // Can be used to determine throttling with multiple connections.
  593. size_t
  594. ConnectionCache::getActiveConnections()
  595. {
  596. size_t result = 0;
  597. size_t tmpCount;
  598. FiberMutex::ScopedLock lock(m_mutex);
  599. CachedConnectionMap::iterator it;
  600. ConnectionList::iterator it2;
  601. for (it = m_conns.begin(); it != m_conns.end(); it++) {
  602. boost::shared_ptr<ConnectionInfo> info = it->second;
  603. tmpCount = 0;
  604. for (it2 = info->connections.begin();
  605. it2 != info->connections.end();
  606. ++it2) {
  607. if (*it2 != NULL && (*it2)->outstandingRequests() > 0){
  608. tmpCount++;
  609. }
  610. }
  611. MORDOR_LOG_TRACE(g_cacheLog) << this << " Active connections to " << it->first.toString() << " - " << tmpCount;
  612. result += tmpCount;
  613. }
  614. MORDOR_LOG_TRACE(g_cacheLog) << this << " Total connections " << result;
  615. return result;
  616. }
  617. std::pair<ClientConnection::ptr, bool>
  618. MockConnectionBroker::getConnection(const URI &uri, bool forceNewConnection)
  619. {
  620. URI schemeAndAuthority = uri;
  621. schemeAndAuthority.path = URI::Path();
  622. schemeAndAuthority.queryDefined(false);
  623. schemeAndAuthority.fragmentDefined(false);
  624. ConnectionCache::iterator it = m_conns.find(schemeAndAuthority);
  625. if (it != m_conns.end() && !it->second->newRequestsAllowed()) {
  626. m_conns.erase(it);
  627. it = m_conns.end();
  628. }
  629. if (it == m_conns.end()) {
  630. std::pair<Stream::ptr, Stream::ptr> pipes = pipeStream();
  631. ClientConnection::ptr client(
  632. new ClientConnection(pipes.first, m_timerManager));
  633. if (m_timerManager) {
  634. client->readTimeout(m_httpReadTimeout);
  635. client->writeTimeout(m_httpWriteTimeout);
  636. }
  637. ServerConnection::ptr server(
  638. new ServerConnection(pipes.second, boost::bind(m_dg,
  639. schemeAndAuthority, _1)));
  640. Scheduler::getThis()->schedule(Fiber::ptr(new Fiber(boost::bind(
  641. &ServerConnection::processRequests, server))));
  642. m_conns[schemeAndAuthority] = client;
  643. return std::make_pair(client, false);
  644. }
  645. return std::make_pair(it->second, false);
  646. }
  647. RequestBroker::ptr
  648. RequestBrokerFilter::parent()
  649. {
  650. if (m_parent)
  651. return m_parent;
  652. return RequestBroker::ptr(m_weakParent);
  653. }
  654. static void doBody(ClientRequest::ptr request,
  655. boost::function<void (ClientRequest::ptr)> bodyDg,
  656. Future<> &future,
  657. boost::exception_ptr &exception, bool &exceptionWasHttp)
  658. {
  659. exceptionWasHttp = false;
  660. try {
  661. bodyDg(request);
  662. } catch (boost::exception &ex) {
  663. exceptionWasHttp = request->requestState() == ClientRequest::ERROR;
  664. if (exceptionWasHttp)
  665. ex << errinfo_source(HTTP);
  666. // Make sure the request is fully aborted so we don't hang waiting for
  667. // a response (since the request object is still in scope by the
  668. // caller, it won't do this automatically)
  669. request->cancel();
  670. exception = boost::current_exception();
  671. } catch (...) {
  672. exceptionWasHttp = request->requestState() == ClientRequest::ERROR;
  673. // Make sure the request is fully aborted so we don't hang waiting for
  674. // a response (since the request object is still in scope by the
  675. // caller, it won't do this automatically)
  676. request->cancel();
  677. exception = boost::current_exception();
  678. }
  679. future.signal();
  680. }
  681. ClientRequest::ptr
  682. BaseRequestBroker::request(Request &requestHeaders, bool forceNewConnection,
  683. boost::function<void (ClientRequest::ptr)> bodyDg)
  684. {
  685. URI &currentUri = requestHeaders.requestLine.uri;
  686. URI originalUri = currentUri;
  687. bool connect = requestHeaders.requestLine.method == CONNECT;
  688. MORDOR_ASSERT(originalUri.authority.hostDefined());
  689. if (!connect) {
  690. requestHeaders.request.host = originalUri.authority.host();
  691. } else {
  692. MORDOR_ASSERT(originalUri.scheme() == "http");
  693. MORDOR_ASSERT(originalUri.path.segments.size() == 2);
  694. currentUri = URI();
  695. currentUri.authority = originalUri.path.segments[1];
  696. }
  697. ConnectionBroker::ptr connectionBroker = m_connectionBroker;
  698. if (!connectionBroker)
  699. connectionBroker = m_weakConnectionBroker.lock();
  700. std::pair<ClientConnection::ptr, bool> conn;
  701. try {
  702. conn = connectionBroker->getConnection(
  703. connect ? originalUri : currentUri, forceNewConnection);
  704. } catch (boost::exception &ex) {
  705. currentUri = originalUri;
  706. ex << errinfo_source(CONNECTION);
  707. throw;
  708. } catch (...) {
  709. currentUri = originalUri;
  710. throw;
  711. }
  712. // We use an absolute URI if we're talking to a proxy, or if the path
  713. // starts with "//" (path_absolute does not allow an empty first segment;
  714. // path_abempty as part of absolute_URI does); otherwise use only the path
  715. // (and query)
  716. if (!connect && !conn.second && !(originalUri.path.isAbsolute() &&
  717. originalUri.path.segments.size() > 2 &&
  718. originalUri.path.segments[1].empty())) {
  719. currentUri.schemeDefined(false);
  720. currentUri.authority.hostDefined(false);
  721. }
  722. try {
  723. ClientRequest::ptr request;
  724. try {
  725. request = conn.first->request(requestHeaders);
  726. if (!bodyDg)
  727. request->doRequest();
  728. } catch (boost::exception &ex) {
  729. ex << errinfo_source(HTTP);
  730. throw;
  731. }
  732. Future<> future;
  733. boost::exception_ptr exception;
  734. bool exceptionWasHttp = false;
  735. if (bodyDg)
  736. Scheduler::getThis()->schedule(boost::bind(&doBody,
  737. request, bodyDg, boost::ref(future), boost::ref(exception),
  738. boost::ref(exceptionWasHttp)));
  739. currentUri = originalUri;
  740. try {
  741. // Force reading the response here to check for connectivity problems
  742. request->response();
  743. } catch (boost::exception &ex) {
  744. ex << errinfo_source(HTTP);
  745. if (bodyDg)
  746. future.wait();
  747. // Prefer to throw an exception from send, if there was one
  748. if (exception)
  749. Mordor::rethrow_exception(exception);
  750. throw;
  751. } catch (...) {
  752. if (bodyDg)
  753. future.wait();
  754. // Prefer to throw an exception from send, if there was one
  755. if (exception)
  756. Mordor::rethrow_exception(exception);
  757. throw;
  758. }
  759. if (bodyDg)
  760. future.wait();
  761. // Rethrow any exception from bodyDg *if* it didn't come from the
  762. // HTTP framework, or directly from a stream within the framework
  763. // (i.e. an exception from the user's code)
  764. // otherwise, ignore it - we have a response, and we don't really
  765. // care that the request didn't complete
  766. if (exception && !exceptionWasHttp)
  767. Mordor::rethrow_exception(exception);
  768. return request;
  769. } catch (...) {
  770. currentUri = originalUri;
  771. throw;
  772. }
  773. }
  774. ClientRequest::ptr
  775. RetryRequestBroker::request(Request &requestHeaders, bool forceNewConnection,
  776. boost::function<void (ClientRequest::ptr)> bodyDg)
  777. {
  778. size_t localRetries = 0;
  779. size_t *retries = mp_retries ? mp_retries : &localRetries;
  780. while (true) {
  781. try {
  782. ClientRequest::ptr request =
  783. parent()->request(requestHeaders, forceNewConnection, bodyDg);
  784. // Successful request resets shared retry counter
  785. *retries = 0;
  786. return request;
  787. } catch (SocketException &ex) {
  788. const ExceptionSource *source = boost::get_error_info<errinfo_source>(ex);
  789. if (!source || (*source != HTTP && *source != CONNECTION))
  790. throw;
  791. if (m_delayDg && !m_delayDg(atomicIncrement(*retries)))
  792. throw;
  793. continue;
  794. } catch (PriorRequestFailedException &ex) {
  795. const ExceptionSource *source = boost::get_error_info<errinfo_source>(ex);
  796. if (!source || *source != HTTP)
  797. throw;
  798. if (m_delayDg && !m_delayDg(*retries + 1))
  799. throw;
  800. continue;
  801. } catch (PriorConnectionFailedException &ex) {
  802. const ExceptionSource *source = boost::get_error_info<errinfo_source>(ex);
  803. if (!source || (*source != HTTP && *source != CONNECTION))
  804. throw;
  805. if (m_delayDg && !m_delayDg(*retries + 1))
  806. throw;
  807. continue;
  808. } catch (UnexpectedEofException &ex) {
  809. const ExceptionSource *source = boost::get_error_info<errinfo_source>(ex);
  810. if (!source || *source != HTTP)
  811. throw;
  812. if (m_delayDg && !m_delayDg(atomicIncrement(*retries)))
  813. throw;
  814. continue;
  815. }
  816. MORDOR_NOTREACHED();
  817. }
  818. }
  819. ClientRequest::ptr
  820. RedirectRequestBroker::request(Request &requestHeaders, bool forceNewConnection,
  821. boost::function<void (ClientRequest::ptr)> bodyDg)
  822. {
  823. URI &currentUri = requestHeaders.requestLine.uri;
  824. URI originalUri = currentUri;
  825. std::list<URI> uris;
  826. uris.push_back(originalUri);
  827. size_t redirects = 0;
  828. while (true) {
  829. try {
  830. ClientRequest::ptr request = parent()->request(requestHeaders,
  831. forceNewConnection, bodyDg);
  832. bool handleRedirect = false;
  833. switch (request->response().status.status)
  834. {
  835. case FOUND:
  836. handleRedirect = m_handle302;
  837. break;
  838. case TEMPORARY_REDIRECT:
  839. handleRedirect = m_handle307;
  840. break;
  841. case MOVED_PERMANENTLY:
  842. handleRedirect = m_handle301;
  843. break;
  844. default:
  845. currentUri = originalUri;
  846. return request;
  847. }
  848. if (handleRedirect) {
  849. if (++redirects == m_maxRedirects)
  850. MORDOR_THROW_EXCEPTION(CircularRedirectException(originalUri));
  851. currentUri = URI::transform(currentUri,
  852. request->response().response.location);
  853. if (std::find(uris.begin(), uris.end(), currentUri)
  854. != uris.end())
  855. MORDOR_THROW_EXCEPTION(CircularRedirectException(originalUri));
  856. uris.push_back(currentUri);
  857. if (request->response().status.status == MOVED_PERMANENTLY)
  858. originalUri = currentUri;
  859. request->finish();
  860. continue;
  861. } else {
  862. currentUri = originalUri;
  863. return request;
  864. }
  865. } catch (...) {
  866. currentUri = originalUri;
  867. throw;
  868. }
  869. MORDOR_NOTREACHED();
  870. }
  871. }
  872. ClientRequest::ptr
  873. UserAgentRequestBroker::request(Request &requestHeaders, bool forceNewConnection,
  874. boost::function<void (ClientRequest::ptr)> bodyDg)
  875. {
  876. requestHeaders.request.userAgent = m_userAgent;
  877. return parent()->request(requestHeaders, forceNewConnection, bodyDg);
  878. }
  879. }}