/mordor/http/broker.cpp
C++ | 931 lines | 814 code | 52 blank | 65 comment | 224 complexity | 46fb4558fff9fedc8f1246afbaaba82d MD5 | raw file
1// Copyright (c) 2009 - Mozy, Inc. 2 3#include "broker.h" 4 5#include "auth.h" 6#include "client.h" 7#include "mordor/atomic.h" 8#include "mordor/fiber.h" 9#include "mordor/future.h" 10#include "mordor/iomanager.h" 11#include "mordor/log.h" 12#include "mordor/socks.h" 13#include "mordor/streams/buffered.h" 14#include "mordor/streams/pipe.h" 15#include "mordor/streams/socket.h" 16#include "mordor/streams/ssl.h" 17#include "mordor/streams/timeout.h" 18#include "proxy.h" 19#include "server.h" 20 21namespace Mordor { 22namespace HTTP { 23 24std::pair<RequestBroker::ptr, ConnectionCache::ptr> 25createRequestBroker(const RequestBrokerOptions &options) 26{ 27 TimerManager *timerManager = const_cast<TimerManager*>(options.timerManager); 28 if (options.ioManager && !timerManager) 29 timerManager = const_cast<IOManager*>(options.ioManager); 30 31 SocketStreamBroker::ptr socketBroker(new SocketStreamBroker(options.ioManager, 32 options.scheduler)); 33 socketBroker->connectTimeout(options.connectTimeout); 34 socketBroker->networkFilterCallback(options.filterNetworksCB); 35 36 StreamBroker::ptr streamBroker = socketBroker; 37 if (options.customStreamBrokerFilter) { 38 options.customStreamBrokerFilter->parent(streamBroker); 39 streamBroker = options.customStreamBrokerFilter; 40 } 41 42 ConnectionBroker::ptr connectionBroker; 43 ConnectionCache::ptr connectionCache; 44 if (!options.enableConnectionCache) { 45 ConnectionNoCache::ptr connectionNoCache(new ConnectionNoCache(streamBroker, 46 timerManager)); 47 connectionBroker = boost::static_pointer_cast<ConnectionBroker>(connectionNoCache); 48 } else { 49 connectionCache = ConnectionCache::create(streamBroker, timerManager); 50 connectionCache->idleTimeout(options.idleTimeout); 51 connectionCache->proxyForURI(options.proxyForURIDg); 52 connectionCache->proxyRequestBroker(options.proxyRequestBroker); 53 connectionCache->connectionsPerHost(options.connectionsPerHost); 54 connectionCache->requestsPerConnection(options.requestsPerConnection); 55 connectionBroker = boost::static_pointer_cast<ConnectionBroker>(connectionCache); 56 } 57 connectionBroker->httpReadTimeout(options.httpReadTimeout); 58 connectionBroker->httpWriteTimeout(options.httpWriteTimeout); 59 connectionBroker->sslReadTimeout(options.sslConnectReadTimeout); 60 connectionBroker->sslWriteTimeout(options.sslConnectWriteTimeout); 61 connectionBroker->sslCtx(options.sslCtx); 62 connectionBroker->verifySslCertificate(options.verifySslCertificate); 63 connectionBroker->verifySslCertificateHost(options.verifySslCertificateHost); 64 65 RequestBroker::ptr requestBroker(new BaseRequestBroker(connectionBroker)); 66 67 if (options.getCredentialsDg || options.getProxyCredentialsDg) 68 requestBroker.reset(new AuthRequestBroker(requestBroker, 69 options.getCredentialsDg, options.getProxyCredentialsDg)); 70 if (options.handleRedirects) 71 requestBroker.reset(new RedirectRequestBroker(requestBroker)); 72 if (options.delayDg) 73 requestBroker.reset(new RetryRequestBroker(requestBroker, 74 options.delayDg)); 75 if (!options.userAgent.empty()) 76 requestBroker.reset(new UserAgentRequestBroker(requestBroker, 77 options.userAgent)); 78 return std::make_pair(requestBroker, connectionCache); 79} 80 81RequestBroker::ptr defaultRequestBroker(IOManager *ioManager, 82 Scheduler *scheduler, 83 ConnectionBroker::ptr *connBroker, 84 boost::function<bool (size_t)> delayDg) 85{ 86 RequestBrokerOptions options; 87 options.ioManager = ioManager; 88 options.scheduler = scheduler; 89 options.delayDg = delayDg; 90 std::pair<RequestBroker::ptr, ConnectionCache::ptr> result = createRequestBroker(options); 91 if (connBroker) 92 *connBroker = result.second; 93 return result.first; 94} 95 96 97StreamBroker::ptr 98StreamBrokerFilter::parent() 99{ 100 if (m_parent) 101 return m_parent; 102 return StreamBroker::ptr(m_weakParent); 103} 104 105Stream::ptr 106SocketStreamBroker::getStream(const URI &uri) 107{ 108 if (m_cancelled) 109 MORDOR_THROW_EXCEPTION(OperationAbortedException()); 110 111 MORDOR_ASSERT(uri.authority.hostDefined()); 112 MORDOR_ASSERT(uri.authority.portDefined() || uri.schemeDefined()); 113 std::ostringstream os; 114 os << uri.authority.host(); 115 if (uri.authority.portDefined()) 116 os << ":" << uri.authority.port(); 117 else if (uri.schemeDefined()) 118 os << ":" << uri.scheme(); 119 std::vector<Address::ptr> addresses; 120 { 121 SchedulerSwitcher switcher(m_scheduler); 122 addresses = Address::lookup(os.str()); 123 } 124 Socket::ptr socket; 125 for (std::vector<Address::ptr>::const_iterator it(addresses.begin()); 126 it != addresses.end(); 127 ) 128 { 129 if (m_ioManager) 130 socket = (*it)->createSocket(*m_ioManager, SOCK_STREAM); 131 else 132 socket = (*it)->createSocket(SOCK_STREAM); 133 std::list<Socket::ptr>::iterator it2; 134 { 135 boost::mutex::scoped_lock lock(m_mutex); 136 if (m_cancelled) 137 MORDOR_THROW_EXCEPTION(OperationAbortedException()); 138 m_pending.push_back(socket); 139 it2 = m_pending.end(); 140 --it2; 141 } 142 socket->sendTimeout(m_connectTimeout); 143 try { 144 // if we are filtering network connections, the callback will bind 145 // the socket to an approved network address 146 if (m_filterNetworkCallback != NULL) 147 { 148 // the callback function is responsible for exiting out by throwing 149 // an exception. 150 m_filterNetworkCallback(socket); 151 } 152 socket->connect(*it); 153 boost::mutex::scoped_lock lock(m_mutex); 154 m_pending.erase(it2); 155 break; 156 } catch (...) { 157 boost::mutex::scoped_lock lock(m_mutex); 158 m_pending.erase(it2); 159 if (++it == addresses.end()) 160 throw; 161 } 162 socket->sendTimeout(~0ull); 163 } 164 Stream::ptr stream(new SocketStream(socket)); 165 return stream; 166} 167 168void 169SocketStreamBroker::cancelPending() 170{ 171 boost::mutex::scoped_lock lock(m_mutex); 172 m_cancelled = true; 173 for (std::list<Socket::ptr>::iterator it(m_pending.begin()); 174 it != m_pending.end(); 175 ++it) { 176 (*it)->cancelConnect(); 177 (*it)->cancelSend(); 178 (*it)->cancelReceive(); 179 } 180} 181 182static bool least(const ClientConnection::ptr &lhs, 183 const ClientConnection::ptr &rhs) 184{ 185 if (lhs && rhs) 186 return lhs->outstandingRequests() < 187 rhs->outstandingRequests(); 188 if (!lhs) 189 return false; 190 if (!rhs) 191 return true; 192 MORDOR_NOTREACHED(); 193} 194 195static Logger::ptr g_cacheLog = Log::lookup("mordor:http:connectioncache"); 196 197std::pair<ClientConnection::ptr, bool> 198ConnectionCache::getConnection(const URI &uri, bool forceNewConnection) 199{ 200 std::vector<URI> proxies; 201 if (m_proxyForURIDg) 202 proxies = m_proxyForURIDg(uri); 203 // Remove proxy types that aren't supported 204 for (std::vector<URI>::iterator it(proxies.begin()); 205 it != proxies.end(); 206 ++it) { 207 MORDOR_ASSERT(it->schemeDefined() || !it->isDefined()); 208 if (!it->schemeDefined()) 209 continue; 210 std::string scheme = it->scheme(); 211 if (scheme != "http" && (scheme != "https" || !m_proxyBroker) && 212 scheme != "socks") 213 it = proxies.erase(it, it); 214 } 215 URI schemeAndAuthority; 216 schemeAndAuthority = uri; 217 schemeAndAuthority.path = URI::Path(); 218 schemeAndAuthority.queryDefined(false); 219 schemeAndAuthority.fragmentDefined(false); 220 std::pair<ClientConnection::ptr, bool> result; 221 222 FiberMutex::ScopedLock lock(m_mutex); 223 224 if (g_cacheLog->enabled(Log::DEBUG)) { 225 std::ostringstream os; 226 os << this << " getting connection for " << schemeAndAuthority 227 << ", proxies: {"; 228 for (std::vector<URI>::iterator it(proxies.begin()); 229 it != proxies.end(); 230 ++it) { 231 if (it != proxies.begin()) 232 os << ", "; 233 os << *it; 234 } 235 os << "}"; 236 MORDOR_LOG_DEBUG(g_cacheLog) << os.str(); 237 } 238 239 if (m_closed) 240 MORDOR_THROW_EXCEPTION(OperationAbortedException()); 241 // Clean out any dead conns 242 cleanOutDeadConns(m_conns); 243 244 if (!forceNewConnection) { 245 if (proxies.empty()) { 246 result = getConnectionViaProxyFromCache(schemeAndAuthority, URI()); 247 if (result.first) 248 return result; 249 } 250 for (std::vector<URI>::const_iterator it(proxies.begin()); 251 it != proxies.end(); 252 ++it) { 253 result = getConnectionViaProxyFromCache(schemeAndAuthority, *it); 254 if (result.first) 255 return result; 256 } 257 } 258 259 // Create a new connection 260 if (proxies.empty()) 261 return getConnectionViaProxy(schemeAndAuthority, URI(), lock); 262 std::vector<URI>::const_iterator it = proxies.begin(); 263 while(true) { 264 try { 265 return getConnectionViaProxy(schemeAndAuthority, *it, lock); 266 } catch (SocketException &) { 267 if (++it == proxies.end()) 268 throw; 269 } catch (HTTP::Exception &) { 270 if (++it == proxies.end()) 271 throw; 272 } catch (UnexpectedEofException &) { 273 if (++it == proxies.end()) 274 throw; 275 } 276 } 277} 278 279std::pair<ClientConnection::ptr, bool> 280ConnectionNoCache::getConnection(const URI &uri, bool forceNewConnection) 281{ 282 URI endPoint = uri; 283 endPoint.path = URI::Path(); 284 endPoint.queryDefined(false); 285 endPoint.fragmentDefined(false); 286 287 // Establish a new connection 288 Stream::ptr stream = m_streamBroker->getStream(endPoint); 289 addSSL(endPoint, stream); 290 291 std::pair<ClientConnection::ptr, bool> result = std::make_pair(ClientConnection::ptr( 292 new ClientConnection(stream, m_timerManager)), false); 293 if (m_httpReadTimeout != ~0ull) 294 result.first->readTimeout(m_httpReadTimeout); 295 if (m_httpWriteTimeout != ~0ull) 296 result.first->writeTimeout(m_httpWriteTimeout); 297 298 return result; 299} 300 301std::pair<ClientConnection::ptr, bool> 302ConnectionCache::getConnectionViaProxyFromCache(const URI &uri, const URI &proxy) 303{ 304 // Check if an existing connection exists to the requested URI 305 // that should be reused. 306 // When proxy is specified this looks for a connection 307 // to the proxy uri instead 308 309 bool proxied = proxy.schemeDefined() && proxy.scheme() == "http"; 310 const URI &endpoint = proxied ? proxy : uri; 311 CachedConnectionMap::iterator it = m_conns.find(endpoint); 312 ConnectionList::iterator it2; 313 while (true) { 314 if (it != m_conns.end() && 315 !it->second->connections.empty() && 316 it->second->connections.size() >= m_connectionsPerHost) { 317 boost::shared_ptr<ConnectionInfo> info = it->second; 318 ConnectionList &connsForThisUri = info->connections; 319 // Assign it2 to point to the connection with the 320 // least number of outstanding requests 321 it2 = std::min_element(connsForThisUri.begin(), 322 connsForThisUri.end(), &least); 323 // No connection has completed yet (but it's in progress) 324 if (!*it2) { 325 MORDOR_LOG_TRACE(g_cacheLog) << this << " waiting for connection to " 326 << endpoint; 327 // Wait for somebody to let us try again 328 unsigned long long start = TimerManager::now(); 329 info->condition.wait(); 330 if (info->lastFailedConnectionTimestamp <= start) 331 MORDOR_THROW_EXCEPTION(PriorConnectionFailedException()); 332 if (m_closed) 333 MORDOR_THROW_EXCEPTION(OperationAbortedException()); 334 // We let go of the mutex, and the last connection may have 335 // disappeared 336 it = m_conns.find(endpoint); 337 } else { 338 MORDOR_LOG_TRACE(g_cacheLog) << this << " returning cached connection " 339 << *it2 << " to " << endpoint; 340 // Return the existing, completed connection 341 return std::make_pair(*it2, proxied); 342 } 343 } else { 344 // No existing connections 345 return std::make_pair(ClientConnection::ptr(), false); 346 } 347 } 348} 349 350std::pair<ClientConnection::ptr, bool> 351ConnectionCache::getConnectionViaProxy(const URI &uri, const URI &proxy, 352 FiberMutex::ScopedLock &lock) 353{ 354 // Create a new Connection to the requested URI, using 355 // the proxy if specified 356 357 std::string proxyScheme; 358 if (proxy.schemeDefined()) 359 proxyScheme = proxy.scheme(); 360 bool proxied = proxyScheme == "http"; 361 const URI &endpoint = proxied ? proxy : uri; 362 363 // Make sure we have a ConnectionList and mutex for this endpoint 364 CachedConnectionMap::iterator it = m_conns.find(endpoint); 365 boost::shared_ptr<ConnectionInfo> info; 366 if (it == m_conns.end()) { 367 info.reset(new ConnectionInfo(m_mutex)); 368 it = m_conns.insert(std::make_pair(endpoint, info)).first; 369 } else { 370 info = it->second; 371 } 372 // Add a placeholder for the new connection 373 info->connections.push_back(ClientConnection::ptr()); 374 375 MORDOR_LOG_TRACE(g_cacheLog) << this << " establishing connection to " 376 << endpoint; 377 unsigned long long start = TimerManager::now(); 378 lock.unlock(); 379 380 ConnectionList::iterator it2; 381 std::pair<ClientConnection::ptr, bool> result; 382 // Establish a new connection 383 try { 384 Stream::ptr stream; 385 if (proxyScheme == "https") { 386 stream = tunnel(m_proxyBroker, proxy, uri); 387 } else if (proxyScheme == "socks") { 388 unsigned short port; 389 if (uri.authority.portDefined()) 390 port = uri.authority.port(); 391 else if (uri.scheme() == "http") 392 port = 80; 393 else if (uri.scheme() == "https") 394 port = 443; 395 else 396 // TODO: can this be looked up using the system? (getaddrinfo) 397 MORDOR_THROW_EXCEPTION(std::invalid_argument("Unknown protocol for proxying connection")); 398 stream = SOCKS::tunnel(m_streamBroker, proxy, IPAddress::ptr(), 399 uri.authority.host(), port); 400 } else { 401 stream = m_streamBroker->getStream(endpoint); 402 } 403 addSSL(endpoint, stream); 404 lock.lock(); 405 // Somebody called abortConnections while we were unlocked; just throw 406 // this connection away 407 if (m_closed) 408 MORDOR_THROW_EXCEPTION(OperationAbortedException()); 409 result = std::make_pair(ClientConnection::ptr( 410 new ClientConnection(stream, m_timerManager)), proxied); 411 MORDOR_LOG_TRACE(g_cacheLog) << this << " connection " << result.first 412 << " to " << endpoint << " established"; 413 result.first->maxRequestCount(m_requestsPerConnection); 414 stream->onRemoteClose(boost::bind(&ConnectionCache::dropConnection, 415 this, weak_ptr(shared_from_this()), endpoint, result.first.get())); 416 if (m_httpReadTimeout != ~0ull) 417 result.first->readTimeout(m_httpReadTimeout); 418 if (m_httpWriteTimeout != ~0ull) 419 result.first->writeTimeout(m_httpWriteTimeout); 420 if (m_idleTimeout != ~0ull) 421 result.first->idleTimeout(m_idleTimeout, 422 boost::bind(&ConnectionCache::dropConnection, 423 this, weak_ptr(shared_from_this()), endpoint, result.first.get())); 424 // Assign this connection to the first blank connection for this 425 // schemeAndAuthority 426 for (it2 = info->connections.begin(); 427 it2 != info->connections.end(); 428 ++it2) { 429 if (!*it2) { 430 *it2 = result.first; 431 break; 432 } 433 } 434 // We should have assigned this connection *somewhere* 435 MORDOR_ASSERT(it2 != info->connections.end()); 436 // Unblock all waiters for them to choose an existing connection 437 info->condition.broadcast(); 438 } catch (...) { 439 lock.lock(); 440 MORDOR_LOG_TRACE(g_cacheLog) << this << " connection to " << endpoint 441 << " failed: " << boost::current_exception_diagnostic_information(); 442 // Somebody called abortConnections while we were unlocked; no need to 443 // clean up the temporary spot for this connection, since it's gone; 444 // pass the original exception on, though 445 if (m_closed) 446 throw; 447 // This connection attempt failed; remove the first blank connection 448 // for this schemeAndAuthority to let someone else try to establish a 449 // connection 450 // it should still be valid, even if the map changed 451 for (it2 = info->connections.begin(); 452 it2 != info->connections.end(); 453 ++it2) { 454 if (!*it2) { 455 info->connections.erase(it2); 456 break; 457 } 458 } 459 info->lastFailedConnectionTimestamp = start; 460 info->condition.broadcast(); 461 if (info->connections.empty()) 462 m_conns.erase(it); 463 throw; 464 } 465 return result; 466} 467 468void 469ConnectionCache::closeIdleConnections() 470{ 471 FiberMutex::ScopedLock lock(m_mutex); 472 MORDOR_LOG_DEBUG(g_cacheLog) << " dropping idle connections"; 473 // We don't just clear the list, because there may be a connection in 474 // progress that has an iterator into it 475 CachedConnectionMap::iterator it, extraIt; 476 for (it = m_conns.begin(); it != m_conns.end();) { 477 it->second->condition.broadcast(); 478 for (ConnectionList::iterator it2 = it->second->connections.begin(); 479 it2 != it->second->connections.end();) { 480 if (*it2) { 481 Stream::ptr connStream = (*it2)->stream(); 482 connStream->cancelRead(); 483 connStream->cancelWrite(); 484 if (m_idleTimeout != ~0ull) 485 (*it2)->idleTimeout(~0ull, NULL); 486 it2 = it->second->connections.erase(it2); 487 } else { 488 ++it2; 489 } 490 } 491 if (it->second->connections.empty()) { 492 extraIt = it; 493 ++it; 494 m_conns.erase(extraIt); 495 } else { 496 ++it; 497 } 498 } 499} 500 501void 502ConnectionCache::abortConnections() 503{ 504 FiberMutex::ScopedLock lock(m_mutex); 505 MORDOR_LOG_DEBUG(g_cacheLog) << " aborting all connections"; 506 m_closed = true; 507 CachedConnectionMap::iterator it; 508 for (it = m_conns.begin(); it != m_conns.end(); ++it) { 509 it->second->condition.broadcast(); 510 for (ConnectionList::iterator it2 = it->second->connections.begin(); 511 it2 != it->second->connections.end(); 512 ++it2) { 513 if (*it2) { 514 Stream::ptr connStream = (*it2)->stream(); 515 connStream->cancelRead(); 516 connStream->cancelWrite(); 517 if (m_idleTimeout != ~0ull) 518 (*it2)->idleTimeout(~0ull, NULL); 519 } 520 } 521 } 522 m_conns.clear(); 523 lock.unlock(); 524 m_streamBroker->cancelPending(); 525} 526 527void 528ConnectionCache::cleanOutDeadConns(CachedConnectionMap &conns) 529{ 530 CachedConnectionMap::iterator it, it3; 531 ConnectionList::iterator it2; 532 for (it = conns.begin(); it != conns.end();) { 533 for (it2 = it->second->connections.begin(); 534 it2 != it->second->connections.end();) { 535 if (*it2 && !(*it2)->newRequestsAllowed()) { 536 if (m_idleTimeout != ~0ull) 537 (*it2)->idleTimeout(~0ull, NULL); 538 it2 = it->second->connections.erase(it2); 539 } else { 540 ++it2; 541 } 542 } 543 if (it->second->connections.empty()) { 544 it3 = it; 545 ++it3; 546 conns.erase(it); 547 it = it3; 548 } else { 549 ++it; 550 } 551 } 552} 553 554void 555ConnectionBroker::addSSL(const URI &uri, Stream::ptr &stream) 556{ 557 if (uri.schemeDefined() && uri.scheme() == "https") { 558 TimeoutStream::ptr timeoutStream; 559 if (m_timerManager) { 560 timeoutStream.reset(new TimeoutStream(stream, *m_timerManager)); 561 timeoutStream->readTimeout(m_sslReadTimeout); 562 timeoutStream->writeTimeout(m_sslWriteTimeout); 563 stream = timeoutStream; 564 } 565 BufferedStream::ptr bufferedStream(new BufferedStream(stream)); 566 bufferedStream->allowPartialReads(true); 567 SSLStream::ptr sslStream(new SSLStream(bufferedStream, true, true, m_sslCtx)); 568 // Only do SNI when required to verify host name 569 if (m_verifySslCertificateHost) 570 sslStream->serverNameIndication(uri.authority.host()); 571 sslStream->connect(); 572 if (m_verifySslCertificate) 573 sslStream->verifyPeerCertificate(); 574 if (m_verifySslCertificateHost) 575 sslStream->verifyPeerCertificate(uri.authority.host()); 576 if (timeoutStream) { 577 bufferedStream->parent(timeoutStream->parent()); 578 timeoutStream.reset(); 579 } 580 bufferedStream.reset(new BufferedStream(sslStream)); 581 // Max data in each SSL record 582 bufferedStream->bufferSize(16384); 583 bufferedStream->flushMultiplesOfBuffer(true); 584 bufferedStream->allowPartialReads(true); 585 stream = bufferedStream; 586 } 587} 588 589namespace { 590struct CompareConn 591{ 592 CompareConn(const ClientConnection *conn) 593 : m_conn(conn) 594 {} 595 596 bool operator()(const ClientConnection::ptr &lhs) const 597 { 598 return lhs.get() == m_conn; 599 } 600 601 const ClientConnection *m_conn; 602}; 603} 604 605void 606ConnectionCache::dropConnection(weak_ptr self, 607 const URI &uri, 608 const ClientConnection *connection) 609{ 610 ptr strongSelf = self.lock(); 611 if (!strongSelf) { 612 return; 613 } 614 615 FiberMutex::ScopedLock lock(m_mutex); 616 CachedConnectionMap::iterator it = m_conns.find(uri); 617 if (it == m_conns.end()) 618 { 619 MORDOR_LOG_TRACE(g_cacheLog) << this << " Failed to drop connection to " << uri << "not found in our cache "; 620 return; 621 } 622 ConnectionList::iterator it2 = std::find_if(it->second->connections.begin(), 623 it->second->connections.end(), CompareConn(connection)); 624 if (it2 != it->second->connections.end()) { 625 MORDOR_LOG_TRACE(g_cacheLog) << this << " dropping connection " 626 << connection << " to " << uri; 627 if (m_idleTimeout != ~0ull) 628 (*it2)->idleTimeout(~0ull, NULL); 629 it->second->connections.erase(it2); 630 if (it->second->connections.empty()) 631 m_conns.erase(it); 632 } 633} 634 635// Get the number of active connections 636// Can be used to determine throttling with multiple connections. 637size_t 638ConnectionCache::getActiveConnections() 639{ 640 size_t result = 0; 641 size_t tmpCount; 642 FiberMutex::ScopedLock lock(m_mutex); 643 CachedConnectionMap::iterator it; 644 ConnectionList::iterator it2; 645 for (it = m_conns.begin(); it != m_conns.end(); it++) { 646 boost::shared_ptr<ConnectionInfo> info = it->second; 647 tmpCount = 0; 648 for (it2 = info->connections.begin(); 649 it2 != info->connections.end(); 650 ++it2) { 651 if (*it2 != NULL && (*it2)->outstandingRequests() > 0){ 652 tmpCount++; 653 } 654 } 655 MORDOR_LOG_TRACE(g_cacheLog) << this << " Active connections to " << it->first.toString() << " - " << tmpCount; 656 result += tmpCount; 657 } 658 MORDOR_LOG_TRACE(g_cacheLog) << this << " Total connections " << result; 659 return result; 660} 661 662std::pair<ClientConnection::ptr, bool> 663MockConnectionBroker::getConnection(const URI &uri, bool forceNewConnection) 664{ 665 URI schemeAndAuthority = uri; 666 schemeAndAuthority.path = URI::Path(); 667 schemeAndAuthority.queryDefined(false); 668 schemeAndAuthority.fragmentDefined(false); 669 ConnectionCache::iterator it = m_conns.find(schemeAndAuthority); 670 if (it != m_conns.end() && !it->second->newRequestsAllowed()) { 671 m_conns.erase(it); 672 it = m_conns.end(); 673 } 674 if (it == m_conns.end()) { 675 std::pair<Stream::ptr, Stream::ptr> pipes = pipeStream(); 676 ClientConnection::ptr client( 677 new ClientConnection(pipes.first, m_timerManager)); 678 if (m_timerManager) { 679 client->readTimeout(m_httpReadTimeout); 680 client->writeTimeout(m_httpWriteTimeout); 681 } 682 ServerConnection::ptr server( 683 new ServerConnection(pipes.second, boost::bind(m_dg, 684 schemeAndAuthority, _1))); 685 Scheduler::getThis()->schedule(Fiber::ptr(new Fiber(boost::bind( 686 &ServerConnection::processRequests, server)))); 687 m_conns[schemeAndAuthority] = client; 688 return std::make_pair(client, false); 689 } 690 return std::make_pair(it->second, false); 691} 692 693RequestBroker::ptr 694RequestBrokerFilter::parent() 695{ 696 if (m_parent) 697 return m_parent; 698 return RequestBroker::ptr(m_weakParent); 699} 700 701static void doBody(ClientRequest::ptr request, 702 boost::function<void (ClientRequest::ptr)> bodyDg, 703 Future<> &future, 704 boost::exception_ptr &exception, bool &exceptionWasHttp) 705{ 706 exceptionWasHttp = false; 707 try { 708 bodyDg(request); 709 } catch (boost::exception &ex) { 710 exceptionWasHttp = request->requestState() == ClientRequest::ERROR; 711 if (exceptionWasHttp) 712 ex << errinfo_source(HTTP); 713 // Make sure the request is fully aborted so we don't hang waiting for 714 // a response (since the request object is still in scope by the 715 // caller, it won't do this automatically) 716 request->cancel(); 717 exception = boost::current_exception(); 718 } catch (...) { 719 exceptionWasHttp = request->requestState() == ClientRequest::ERROR; 720 // Make sure the request is fully aborted so we don't hang waiting for 721 // a response (since the request object is still in scope by the 722 // caller, it won't do this automatically) 723 request->cancel(); 724 exception = boost::current_exception(); 725 } 726 future.signal(); 727} 728 729ClientRequest::ptr 730BaseRequestBroker::request(Request &requestHeaders, bool forceNewConnection, 731 boost::function<void (ClientRequest::ptr)> bodyDg) 732{ 733 URI ¤tUri = requestHeaders.requestLine.uri; 734 URI originalUri = currentUri; 735 bool connect = requestHeaders.requestLine.method == CONNECT; 736 MORDOR_ASSERT(originalUri.authority.hostDefined()); 737 if (!connect) { 738 requestHeaders.request.host = originalUri.authority.host(); 739 } else { 740 MORDOR_ASSERT(originalUri.scheme() == "http"); 741 MORDOR_ASSERT(originalUri.path.segments.size() == 2); 742 currentUri = URI(); 743 currentUri.authority = originalUri.path.segments[1]; 744 } 745 ConnectionBroker::ptr connectionBroker = m_connectionBroker; 746 if (!connectionBroker) 747 connectionBroker = m_weakConnectionBroker.lock(); 748 std::pair<ClientConnection::ptr, bool> conn; 749 try { 750 conn = connectionBroker->getConnection( 751 connect ? originalUri : currentUri, forceNewConnection); 752 } catch (boost::exception &ex) { 753 currentUri = originalUri; 754 ex << errinfo_source(CONNECTION); 755 throw; 756 } catch (...) { 757 currentUri = originalUri; 758 throw; 759 } 760 // We use an absolute URI if we're talking to a proxy, or if the path 761 // starts with "//" (path_absolute does not allow an empty first segment; 762 // path_abempty as part of absolute_URI does); otherwise use only the path 763 // (and query) 764 if (!connect && !conn.second && !(originalUri.path.isAbsolute() && 765 originalUri.path.segments.size() > 2 && 766 originalUri.path.segments[1].empty())) { 767 currentUri.schemeDefined(false); 768 currentUri.authority.hostDefined(false); 769 } 770 try { 771 ClientRequest::ptr request; 772 try { 773 request = conn.first->request(requestHeaders); 774 if (!bodyDg) 775 request->doRequest(); 776 } catch (boost::exception &ex) { 777 ex << errinfo_source(HTTP); 778 throw; 779 } 780 Future<> future; 781 boost::exception_ptr exception; 782 bool exceptionWasHttp = false; 783 if (bodyDg) 784 Scheduler::getThis()->schedule(boost::bind(&doBody, 785 request, bodyDg, boost::ref(future), boost::ref(exception), 786 boost::ref(exceptionWasHttp))); 787 currentUri = originalUri; 788 try { 789 // Force reading the response here to check for connectivity problems 790 request->response(); 791 } catch (boost::exception &ex) { 792 ex << errinfo_source(HTTP); 793 if (bodyDg) 794 future.wait(); 795 // Prefer to throw an exception from send, if there was one 796 if (exception) 797 Mordor::rethrow_exception(exception); 798 throw; 799 } catch (...) { 800 if (bodyDg) 801 future.wait(); 802 // Prefer to throw an exception from send, if there was one 803 if (exception) 804 Mordor::rethrow_exception(exception); 805 throw; 806 } 807 if (bodyDg) 808 future.wait(); 809 // Rethrow any exception from bodyDg *if* it didn't come from the 810 // HTTP framework, or directly from a stream within the framework 811 // (i.e. an exception from the user's code) 812 // otherwise, ignore it - we have a response, and we don't really 813 // care that the request didn't complete 814 if (exception && !exceptionWasHttp) 815 Mordor::rethrow_exception(exception); 816 return request; 817 } catch (...) { 818 currentUri = originalUri; 819 throw; 820 } 821} 822 823ClientRequest::ptr 824RetryRequestBroker::request(Request &requestHeaders, bool forceNewConnection, 825 boost::function<void (ClientRequest::ptr)> bodyDg) 826{ 827 size_t localRetries = 0; 828 size_t *retries = mp_retries ? mp_retries : &localRetries; 829 while (true) { 830 try { 831 ClientRequest::ptr request = 832 parent()->request(requestHeaders, forceNewConnection, bodyDg); 833 // Successful request resets shared retry counter 834 *retries = 0; 835 return request; 836 } catch (SocketException &ex) { 837 const ExceptionSource *source = boost::get_error_info<errinfo_source>(ex); 838 if (!source || (*source != HTTP && *source != CONNECTION)) 839 throw; 840 if (m_delayDg && !m_delayDg(atomicIncrement(*retries))) 841 throw; 842 continue; 843 } catch (PriorRequestFailedException &ex) { 844 const ExceptionSource *source = boost::get_error_info<errinfo_source>(ex); 845 if (!source || *source != HTTP) 846 throw; 847 if (m_delayDg && !m_delayDg(*retries + 1)) 848 throw; 849 continue; 850 } catch (PriorConnectionFailedException &ex) { 851 const ExceptionSource *source = boost::get_error_info<errinfo_source>(ex); 852 if (!source || (*source != HTTP && *source != CONNECTION)) 853 throw; 854 if (m_delayDg && !m_delayDg(*retries + 1)) 855 throw; 856 continue; 857 } catch (UnexpectedEofException &ex) { 858 const ExceptionSource *source = boost::get_error_info<errinfo_source>(ex); 859 if (!source || *source != HTTP) 860 throw; 861 if (m_delayDg && !m_delayDg(atomicIncrement(*retries))) 862 throw; 863 continue; 864 } 865 MORDOR_NOTREACHED(); 866 } 867} 868 869ClientRequest::ptr 870RedirectRequestBroker::request(Request &requestHeaders, bool forceNewConnection, 871 boost::function<void (ClientRequest::ptr)> bodyDg) 872{ 873 URI ¤tUri = requestHeaders.requestLine.uri; 874 URI originalUri = currentUri; 875 std::list<URI> uris; 876 uris.push_back(originalUri); 877 size_t redirects = 0; 878 while (true) { 879 try { 880 ClientRequest::ptr request = parent()->request(requestHeaders, 881 forceNewConnection, bodyDg); 882 bool handleRedirect = false; 883 switch (request->response().status.status) 884 { 885 case FOUND: 886 handleRedirect = m_handle302; 887 break; 888 case TEMPORARY_REDIRECT: 889 handleRedirect = m_handle307; 890 break; 891 case MOVED_PERMANENTLY: 892 handleRedirect = m_handle301; 893 break; 894 default: 895 currentUri = originalUri; 896 return request; 897 } 898 if (handleRedirect) { 899 if (++redirects == m_maxRedirects) 900 MORDOR_THROW_EXCEPTION(CircularRedirectException(originalUri)); 901 currentUri = URI::transform(currentUri, 902 request->response().response.location); 903 if (std::find(uris.begin(), uris.end(), currentUri) 904 != uris.end()) 905 MORDOR_THROW_EXCEPTION(CircularRedirectException(originalUri)); 906 uris.push_back(currentUri); 907 if (request->response().status.status == MOVED_PERMANENTLY) 908 originalUri = currentUri; 909 request->finish(); 910 continue; 911 } else { 912 currentUri = originalUri; 913 return request; 914 } 915 } catch (...) { 916 currentUri = originalUri; 917 throw; 918 } 919 MORDOR_NOTREACHED(); 920 } 921} 922 923ClientRequest::ptr 924UserAgentRequestBroker::request(Request &requestHeaders, bool forceNewConnection, 925 boost::function<void (ClientRequest::ptr)> bodyDg) 926{ 927 requestHeaders.request.userAgent = m_userAgent; 928 return parent()->request(requestHeaders, forceNewConnection, bodyDg); 929} 930 931}}