/mordor/http/broker.h
C Header | 565 lines | 378 code | 94 blank | 93 comment | 0 complexity | 1eab169475b8564553bc1f66599e5de7 MD5 | raw file
1#ifndef __HTTP_BROKER_H__ 2#define __HTTP_BROKER_H__ 3// Copyright (c) 2009 - Mozy, Inc. 4 5#include <openssl/ssl.h> 6 7#include "http.h" 8#include "mordor/fibersynchronization.h" 9 10namespace Mordor { 11 12class IOManager; 13class Scheduler; 14class Socket; 15class Stream; 16class TimerManager; 17 18namespace HTTP { 19 20class ClientConnection; 21class ClientRequest; 22class RequestBroker; 23class ServerConnection; 24class ServerRequest; 25 26class StreamBroker 27{ 28public: 29 typedef boost::shared_ptr<StreamBroker> ptr; 30 typedef boost::weak_ptr<StreamBroker> weak_ptr; 31 32public: 33 virtual ~StreamBroker() {} 34 35 virtual boost::shared_ptr<Stream> getStream(const URI &uri) = 0; 36 virtual void cancelPending() {} 37}; 38 39class StreamBrokerFilter : public StreamBroker 40{ 41public: 42 typedef boost::shared_ptr<StreamBrokerFilter> ptr; 43 44public: 45 StreamBrokerFilter(StreamBroker::ptr parent, 46 StreamBroker::weak_ptr weakParent = StreamBroker::weak_ptr()) 47 : m_parent(parent), 48 m_weakParent(weakParent) 49 {} 50 51 StreamBroker::ptr parent(); 52 void parent(StreamBroker::ptr parent) 53 { m_parent = parent; m_weakParent.reset(); } 54 void parent(StreamBroker::weak_ptr parent) 55 { m_weakParent = parent; m_parent.reset(); } 56 57 void cancelPending() { parent()->cancelPending(); } 58 59private: 60 StreamBroker::ptr m_parent; 61 StreamBroker::weak_ptr m_weakParent; 62}; 63 64class SocketStreamBroker : public StreamBroker 65{ 66public: 67 typedef boost::shared_ptr<SocketStreamBroker> ptr; 68 69public: 70 SocketStreamBroker(IOManager *ioManager = NULL, Scheduler *scheduler = NULL) 71 : m_cancelled(false), 72 m_ioManager(ioManager), 73 m_scheduler(scheduler), 74 m_connectTimeout(~0ull), 75 m_filterNetworkCallback(NULL) 76 {} 77 78 void connectTimeout(unsigned long long timeout) { m_connectTimeout = timeout; } 79 80 // Resolve the uri to its IP address, create a socket, then connect 81 boost::shared_ptr<Stream> getStream(const URI &uri); 82 void cancelPending(); 83 84 void networkFilterCallback(boost::function<void (boost::shared_ptr<Socket>)> fnCallback) 85 { m_filterNetworkCallback = fnCallback; } 86 87private: 88 boost::mutex m_mutex; 89 bool m_cancelled; 90 std::list<boost::shared_ptr<Socket> > m_pending; // Multiple connections may be attempted when getaddrinfo returns multiple addresses 91 IOManager *m_ioManager; 92 Scheduler *m_scheduler; 93 unsigned long long m_connectTimeout; 94 boost::function<void (boost::shared_ptr<Socket>)> m_filterNetworkCallback; 95}; 96 97class ConnectionBroker 98{ 99public: 100 typedef boost::shared_ptr<ConnectionBroker> ptr; 101 typedef boost::weak_ptr<ConnectionBroker> weak_ptr; 102 103public: 104 ConnectionBroker(): 105 m_verifySslCertificate(false), 106 m_verifySslCertificateHost(true), 107 m_httpReadTimeout(~0ull), 108 m_httpWriteTimeout(~0ull), 109 m_idleTimeout(~0ull), 110 m_sslReadTimeout(~0ull), 111 m_sslWriteTimeout(~0ull), 112 m_sslCtx(NULL), 113 m_timerManager(NULL) 114 {} 115 virtual ~ConnectionBroker() {} 116 117 virtual std::pair<boost::shared_ptr<ClientConnection>, bool> 118 getConnection(const URI &uri, bool forceNewConnection = false) = 0; 119 120 void httpReadTimeout(unsigned long long timeout) { m_httpReadTimeout = timeout; } 121 void httpWriteTimeout(unsigned long long timeout) { m_httpWriteTimeout = timeout; } 122 void idleTimeout(unsigned long long timeout) { m_idleTimeout = timeout; } 123 void sslReadTimeout(unsigned long long timeout) { m_sslReadTimeout = timeout; } 124 void sslWriteTimeout(unsigned long long timeout) { m_sslWriteTimeout = timeout; } 125 void sslCtx(SSL_CTX *ctx) { m_sslCtx = ctx; } 126 void verifySslCertificate(bool verify) { m_verifySslCertificate = verify; } 127 void verifySslCertificateHost(bool verify) { m_verifySslCertificateHost = verify; } 128 129protected: 130 void addSSL(const URI &uri, boost::shared_ptr<Stream> &stream); 131 132protected: 133 bool m_verifySslCertificate, m_verifySslCertificateHost; 134 unsigned long long m_httpReadTimeout, m_httpWriteTimeout, m_idleTimeout, 135 m_sslReadTimeout, m_sslWriteTimeout; 136 SSL_CTX * m_sslCtx; 137 TimerManager *m_timerManager; 138}; 139 140struct PriorConnectionFailedException : virtual Exception {}; 141 142// The ConnectionCache holds all connections associated with a RequestBroker. 143// This is not a global cache of all connections - each RequestBroker instance 144// will have its own. 145// 146// It permits a single RequestBroker to maintain multiple connections to 147// multiple servers at the same time. Connections are held active in this cache 148// so that multiple requests can be performed over a single connection and 149// the cache will take care of automatically reopening connections after they 150// close when a new request arrives. 151// It understands how to use proxies, but relies on the caller provided callback 152// to determine the proxy rules for each request. 153// 154// Although exposed by createRequestBroker(), normal clients will not manipulate 155// the ConnectionCache directly, apart from calling abortConnections or closeIdleConnections 156class ConnectionCache : public boost::enable_shared_from_this<ConnectionCache>, public ConnectionBroker 157{ 158public: 159 typedef boost::shared_ptr<ConnectionCache> ptr; 160 typedef boost::weak_ptr<ConnectionCache> weak_ptr; 161 162protected: 163 ConnectionCache(StreamBroker::ptr streamBroker, TimerManager *timerManager = NULL) 164 : m_streamBroker(streamBroker), 165 m_connectionsPerHost(1u), 166 m_requestsPerConnection((size_t)~0), 167 m_closed(false) 168 { 169 m_timerManager = timerManager; 170 } 171 172public: 173 static ConnectionCache::ptr create(StreamBroker::ptr streamBroker, TimerManager *timerManager = NULL) 174 { return ConnectionCache::ptr(new ConnectionCache(streamBroker, timerManager)); } 175 176 // Specify the maximum number of seperate connections to allow to a specific host (or proxy) 177 // at a time 178 void connectionsPerHost(size_t connections) { m_connectionsPerHost = connections; } 179 void requestsPerConnection(size_t requests) { m_requestsPerConnection = requests; } 180 181 // Get number of active connections 182 size_t getActiveConnections(); 183 184 // Proxy support requires this callback. It is expected to return an 185 // array of candidate Proxy servers to handle the requested URI. 186 // If none are returned the request will be performed directly 187 void proxyForURI(boost::function<std::vector<URI> (const URI &)> proxyForURIDg) 188 { m_proxyForURIDg = proxyForURIDg; } 189 190 // Required to support HTTPS proxies 191 void proxyRequestBroker(boost::shared_ptr<RequestBroker> broker) 192 { m_proxyBroker = broker; } 193 194 // Get the connection associated with a URI. An existing one may be reused, 195 // or a new one established. 196 std::pair<boost::shared_ptr<ClientConnection>, bool /*is proxy connection*/> 197 getConnection(const URI &uri, bool forceNewConnection = false); 198 199 void closeIdleConnections(); 200 201 // Cancel all connections, even the active ones. 202 // Clients should expect OperationAbortedException, and PriorRequestFailedException 203 // to be thrown if requests are active 204 void abortConnections(); 205 206private: 207 typedef std::list<boost::shared_ptr<ClientConnection> > ConnectionList; 208 209 // Tracks active connections to a particular host 210 // e.g. there might be 5 active connections to http://example.com 211 struct ConnectionInfo 212 { 213 ConnectionInfo(FiberMutex &mutex) 214 : condition(mutex), 215 lastFailedConnectionTimestamp(~0ull) 216 {} 217 218 ConnectionList connections; 219 FiberCondition condition; 220 unsigned long long lastFailedConnectionTimestamp; 221 }; 222 223 // Table of active connections for each scheme+host 224 // e.g. if a single RequestBroker is connected to two servers at the same time 225 // or doing both http and https requests then this will contain multiple entries 226 typedef std::map<URI, boost::shared_ptr<ConnectionInfo> > CachedConnectionMap; 227 228private: 229 std::pair<boost::shared_ptr<ClientConnection>, bool> 230 getConnectionViaProxyFromCache(const URI &uri, const URI &proxy); 231 std::pair<boost::shared_ptr<ClientConnection>, bool> 232 getConnectionViaProxy(const URI &uri, const URI &proxy, 233 FiberMutex::ScopedLock &lock); 234 void cleanOutDeadConns(CachedConnectionMap &conns); 235 void dropConnection(weak_ptr self, const URI &uri, const ClientConnection *connection); 236 237private: 238 FiberMutex m_mutex; 239 StreamBroker::ptr m_streamBroker; 240 size_t m_connectionsPerHost, m_requestsPerConnection; 241 242 CachedConnectionMap m_conns; 243 bool m_closed; 244 boost::function<std::vector<URI> (const URI &)> m_proxyForURIDg; 245 boost::shared_ptr<RequestBroker> m_proxyBroker; 246}; 247 248// The ConnectionNoCache has no support for proxies. 249class ConnectionNoCache : public ConnectionBroker 250{ 251public: 252 typedef boost::shared_ptr<ConnectionNoCache> ptr; 253 254 ConnectionNoCache(StreamBroker::ptr streamBroker, TimerManager *timerManager = NULL) 255 : m_streamBroker(streamBroker) 256 { 257 m_timerManager = timerManager; 258 } 259 260 // Get a new connection associated with a URI. Do not cache it. 261 std::pair<boost::shared_ptr<ClientConnection>, bool /*is proxy connection*/> 262 getConnection(const URI &uri, bool forceNewConnection = false); 263 264private: 265 StreamBroker::ptr m_streamBroker; 266}; 267 268// Mock object useful for unit tests. Rather than 269// making a real network call, each request will be 270// processed directly by the provided callback 271class MockConnectionBroker : public ConnectionBroker 272{ 273private: 274 typedef std::map<URI, boost::shared_ptr<ClientConnection> > 275 ConnectionCache; // warning - not the same as class ConnectionCache 276public: 277 MockConnectionBroker(boost::function<void (const URI &uri, 278 boost::shared_ptr<ServerRequest>)> dg, 279 TimerManager *timerManager = NULL, unsigned long long readTimeout = ~0ull, 280 unsigned long long writeTimeout = ~0ull) 281 : m_dg(dg) 282 { 283 m_timerManager = timerManager; 284 } 285 286 std::pair<boost::shared_ptr<ClientConnection>, bool /*is proxy connection*/> 287 getConnection(const URI &uri, bool forceNewConnection = false); 288 289private: 290 boost::function<void (const URI &uri, boost::shared_ptr<ServerRequest>)> m_dg; 291 ConnectionCache m_conns; 292}; 293 294// Abstract base-class for all the RequestBroker objects. 295// RequestBrokers are typically instantiated by calling createRequestBroker(). 296// RequestBrokers abstract the actual network connection. Reusing a connection 297// for multiple requests is achieved by reusing a RequestBroker. 298class RequestBroker 299{ 300public: 301 typedef boost::shared_ptr<RequestBroker> ptr; 302 typedef boost::weak_ptr<RequestBroker> weak_ptr; 303 304public: 305 virtual ~RequestBroker() {} 306 307 // Perform a request. The caller should prepare the requestHeaders 308 // as much as they like, and the chain of RequestBrokerFilters will 309 // also potentially make further adjustments 310 // 311 // Tip: Typically the full URI for the destination should be set in 312 // requestHeaders.requestLine.uri, even though the scheme and authority are 313 // not sent in the first line of the HTTP request. Also fill in 314 // the requestHeaders.request.host. 315 virtual boost::shared_ptr<ClientRequest> request(Request &requestHeaders, 316 bool forceNewConnection = false, 317 boost::function<void (boost::shared_ptr<ClientRequest>)> bodyDg = NULL) 318 = 0; 319}; 320 321// RequestBrokerFilter is the base class for filter request brokers, 322// which exist in a chain leading to the BaseRequestBroker. 323// Each derived class can implement special logic in the request() 324// method and then call request on the next element in the chain (its parent). 325// A filter can stop a request by throwing an exception. 326class RequestBrokerFilter : public RequestBroker 327{ 328public: 329 // When created a RequestBrokerFilter is inserted at the 330 // beginning of a chain of RequestBrokers, with the parent being the 331 // next broker in the chain 332 RequestBrokerFilter(RequestBroker::ptr parent, 333 RequestBroker::weak_ptr weakParent = RequestBroker::weak_ptr()) 334 : m_parent(parent), 335 m_weakParent(weakParent) 336 {} 337 338 RequestBroker::ptr parent(); 339 340 boost::shared_ptr<ClientRequest> request(Request &requestHeaders, 341 bool forceNewConnection = false, 342 boost::function<void (boost::shared_ptr<ClientRequest>)> bodyDg = NULL) 343 = 0; 344 345private: 346 RequestBroker::ptr m_parent; 347 RequestBroker::weak_ptr m_weakParent; 348}; 349 350/// An exception coming from BaseRequestBroker will be tagged with CONNECTION 351/// if the exception came while trying to establish the connection, HTTP 352/// if the exception came specifically from HTTP communications, or no 353/// ExceptionSource at all if it came from other code 354enum ExceptionSource 355{ 356 CONNECTION, 357 HTTP 358}; 359typedef boost::error_info<struct tag_source, ExceptionSource > errinfo_source; 360 361// The BaseRequestBroker is the final broker in a chain of requestbrokers 362// and sends the fully prepared request to its ConnectionBroker to be performed 363class BaseRequestBroker : public RequestBroker 364{ 365public: 366 typedef boost::shared_ptr<BaseRequestBroker> ptr; 367 368public: 369 BaseRequestBroker(ConnectionBroker::ptr connectionBroker) 370 : m_connectionBroker(connectionBroker) 371 {} 372 BaseRequestBroker(ConnectionBroker::weak_ptr connectionBroker) 373 : m_weakConnectionBroker(connectionBroker) 374 {} 375 376 boost::shared_ptr<ClientRequest> request(Request &requestHeaders, 377 bool forceNewConnection = false, 378 boost::function<void (boost::shared_ptr<ClientRequest>)> bodyDg = NULL); 379 380private: 381 ConnectionBroker::ptr m_connectionBroker; 382 ConnectionBroker::weak_ptr m_weakConnectionBroker; 383}; 384 385/// Retries connection error and PriorRequestFailed errors 386class RetryRequestBroker : public RequestBrokerFilter 387{ 388public: 389 typedef boost::shared_ptr<RetryRequestBroker> ptr; 390 391public: 392 RetryRequestBroker(RequestBroker::ptr parent, 393 boost::function<bool (size_t)> delayDg = NULL) 394 : RequestBrokerFilter(parent), 395 m_delayDg(delayDg), 396 mp_retries(NULL) 397 {} 398 399 void sharedRetryCounter(size_t *retries) { mp_retries = retries; } 400 401 boost::shared_ptr<ClientRequest> request(Request &requestHeaders, 402 bool forceNewConnection = false, 403 boost::function<void (boost::shared_ptr<ClientRequest>)> bodyDg = NULL); 404 405private: 406 boost::function<bool (size_t)> m_delayDg; 407 size_t *mp_retries; 408}; 409 410struct CircularRedirectException : Exception 411{ 412 CircularRedirectException(const URI &uri) 413 : m_uri(uri) 414 {} 415 ~CircularRedirectException() throw() {} 416 417 URI uri() { return m_uri; } 418 419private: 420 URI m_uri; 421}; 422 423class RedirectRequestBroker : public RequestBrokerFilter 424{ 425public: 426 typedef boost::shared_ptr<RedirectRequestBroker> ptr; 427 428public: 429 RedirectRequestBroker(RequestBroker::ptr parent, size_t maxRedirects = 70) 430 : RequestBrokerFilter(parent), 431 m_maxRedirects(maxRedirects), 432 m_handle301(true), 433 m_handle302(true), 434 m_handle307(true) 435 {} 436 437 void handlePermanentRedirect(bool handle) { m_handle301 = handle; } 438 void handleFound(bool handle) { m_handle302 = handle; } 439 void handleTemporaryRedirect(bool handle) { m_handle307 = handle; } 440 441 boost::shared_ptr<ClientRequest> request(Request &requestHeaders, 442 bool forceNewConnection = false, 443 boost::function<void (boost::shared_ptr<ClientRequest>)> bodyDg = NULL); 444 445private: 446 size_t m_maxRedirects; 447 bool m_handle301, m_handle302, m_handle307; 448}; 449 450// Takes care of filling in the User-Agent header 451class UserAgentRequestBroker : public RequestBrokerFilter 452{ 453public: 454 UserAgentRequestBroker(RequestBroker::ptr parent, 455 const ProductAndCommentList &userAgent) 456 : RequestBrokerFilter(parent), 457 m_userAgent(userAgent) 458 {} 459 460 boost::shared_ptr<ClientRequest> request(Request &requestHeaders, 461 bool forceNewConnection = false, 462 boost::function<void (boost::shared_ptr<ClientRequest>)> bodyDg = NULL); 463 464private: 465 ProductAndCommentList m_userAgent; 466}; 467 468// When creating a new RequestBroker this structure is used to 469// defines the configuration of a chain of requestBrokers 470// and associated services. The most common configuration options 471// are exposed here, but further custom RequestBroker behavior 472// can be achieved by directly creating RequestBrokerFilter objects 473// and adding them to the chain 474struct RequestBrokerOptions 475{ 476 RequestBrokerOptions() : 477 ioManager(NULL), 478 scheduler(NULL), 479 filterNetworksCB(NULL), 480 handleRedirects(true), 481 timerManager(NULL), 482 connectTimeout(~0ull), 483 sslConnectReadTimeout(~0ull), 484 sslConnectWriteTimeout(~0ull), 485 httpReadTimeout(~0ull), 486 httpWriteTimeout(~0ull), 487 idleTimeout(~0ull), 488 connectionsPerHost(1u), 489 requestsPerConnection((size_t)~0), 490 sslCtx(NULL), 491 verifySslCertificate(false), 492 verifySslCertificateHost(true), 493 enableConnectionCache(true) 494 {} 495 496 IOManager *ioManager; 497 Scheduler *scheduler; 498 499 // When specified a RetryRequestBroker will be installed. If a request fails 500 // the callback will be called with the current retry count. If the callback 501 // returns false then no further retries are attempted. 502 boost::function<bool (size_t /*retry count*/)> delayDg; 503 504 // Callback to call directly before a socket connection happens. 505 // Implementation should throw an exception if it wants to prevent the connection 506 boost::function<void (boost::shared_ptr<Socket>)> filterNetworksCB; 507 508 bool handleRedirects; // Whether to add a RedirectRequestBroker to the chain of RequestBrokers 509 TimerManager *timerManager; // When not specified the iomanager will be used 510 511 // Optional timeout values (us) 512 unsigned long long connectTimeout; 513 unsigned long long sslConnectReadTimeout; 514 unsigned long long sslConnectWriteTimeout; 515 unsigned long long httpReadTimeout; 516 unsigned long long httpWriteTimeout; 517 unsigned long long idleTimeout; 518 size_t connectionsPerHost, requestsPerConnection; 519 520 // Callback to find proxy for an URI, see ConnectionCache::proxyForURI 521 boost::function<std::vector<URI> (const URI &)> proxyForURIDg; 522 523 /// Required to enable https proxy support 524 RequestBroker::ptr proxyRequestBroker; 525 526 // When specified these callbacks will be invoked to add authorization to the 527 // request. An alternative is to add the BasicAuth header before 528 // calling RequestBroker::request (see HTTP::BasicAuth::authorize()) 529 boost::function<bool (const URI &, 530 boost::shared_ptr<ClientRequest> /* priorRequest = ClientRequest::ptr() */, 531 std::string & /* scheme */, std::string & /* realm */, 532 std::string & /* username */, std::string & /* password */, 533 size_t /* attempts */)> 534 getCredentialsDg, getProxyCredentialsDg; 535 536 // When specified the provided object will be installed as a filter 537 // in front of the SocketStreamBroker 538 StreamBrokerFilter::ptr customStreamBrokerFilter; 539 540 SSL_CTX *sslCtx; 541 bool verifySslCertificate; 542 bool verifySslCertificateHost; 543 bool enableConnectionCache; 544 545 // When specified a UserAgentRequestBroker will take care of adding 546 // the User-Agent header to each request 547 ProductAndCommentList userAgent; 548}; 549 550// Factory method to create a chain of request broker objects based on the 551// the specified configuration options. It also creates and returns a 552// shared pointer to the request's new ConnectionCache. 553// clients do not typically work directly with the ConnectionCache, except to 554// shut down connections with ConnectionCache::abortConnections 555std::pair<RequestBroker::ptr, ConnectionCache::ptr> 556 createRequestBroker(const RequestBrokerOptions &options = RequestBrokerOptions()); 557 558/// @deprecated Use createRequestBroker instead 559RequestBroker::ptr defaultRequestBroker(IOManager *ioManager = NULL, 560 Scheduler *scheduler = NULL, 561 ConnectionBroker::ptr *connBroker = NULL, 562 boost::function<bool (size_t)> delayDg = NULL); 563}} 564 565#endif