PageRenderTime 31ms CodeModel.GetById 14ms RepoModel.GetById 0ms app.codeStats 0ms

/mordor/http/broker.h

http://github.com/mozy/mordor
C Header | 565 lines | 378 code | 94 blank | 93 comment | 0 complexity | 1eab169475b8564553bc1f66599e5de7 MD5 | raw file
Possible License(s): BSD-3-Clause
  1. #ifndef __HTTP_BROKER_H__
  2. #define __HTTP_BROKER_H__
  3. // Copyright (c) 2009 - Mozy, Inc.
  4. #include <openssl/ssl.h>
  5. #include "http.h"
  6. #include "mordor/fibersynchronization.h"
  7. namespace Mordor {
  8. class IOManager;
  9. class Scheduler;
  10. class Socket;
  11. class Stream;
  12. class TimerManager;
  13. namespace HTTP {
  14. class ClientConnection;
  15. class ClientRequest;
  16. class RequestBroker;
  17. class ServerConnection;
  18. class ServerRequest;
  19. class StreamBroker
  20. {
  21. public:
  22. typedef boost::shared_ptr<StreamBroker> ptr;
  23. typedef boost::weak_ptr<StreamBroker> weak_ptr;
  24. public:
  25. virtual ~StreamBroker() {}
  26. virtual boost::shared_ptr<Stream> getStream(const URI &uri) = 0;
  27. virtual void cancelPending() {}
  28. };
  29. class StreamBrokerFilter : public StreamBroker
  30. {
  31. public:
  32. typedef boost::shared_ptr<StreamBrokerFilter> ptr;
  33. public:
  34. StreamBrokerFilter(StreamBroker::ptr parent,
  35. StreamBroker::weak_ptr weakParent = StreamBroker::weak_ptr())
  36. : m_parent(parent),
  37. m_weakParent(weakParent)
  38. {}
  39. StreamBroker::ptr parent();
  40. void parent(StreamBroker::ptr parent)
  41. { m_parent = parent; m_weakParent.reset(); }
  42. void parent(StreamBroker::weak_ptr parent)
  43. { m_weakParent = parent; m_parent.reset(); }
  44. void cancelPending() { parent()->cancelPending(); }
  45. private:
  46. StreamBroker::ptr m_parent;
  47. StreamBroker::weak_ptr m_weakParent;
  48. };
  49. class SocketStreamBroker : public StreamBroker
  50. {
  51. public:
  52. typedef boost::shared_ptr<SocketStreamBroker> ptr;
  53. public:
  54. SocketStreamBroker(IOManager *ioManager = NULL, Scheduler *scheduler = NULL)
  55. : m_cancelled(false),
  56. m_ioManager(ioManager),
  57. m_scheduler(scheduler),
  58. m_connectTimeout(~0ull),
  59. m_filterNetworkCallback(NULL)
  60. {}
  61. void connectTimeout(unsigned long long timeout) { m_connectTimeout = timeout; }
  62. // Resolve the uri to its IP address, create a socket, then connect
  63. boost::shared_ptr<Stream> getStream(const URI &uri);
  64. void cancelPending();
  65. void networkFilterCallback(boost::function<void (boost::shared_ptr<Socket>)> fnCallback)
  66. { m_filterNetworkCallback = fnCallback; }
  67. private:
  68. boost::mutex m_mutex;
  69. bool m_cancelled;
  70. std::list<boost::shared_ptr<Socket> > m_pending; // Multiple connections may be attempted when getaddrinfo returns multiple addresses
  71. IOManager *m_ioManager;
  72. Scheduler *m_scheduler;
  73. unsigned long long m_connectTimeout;
  74. boost::function<void (boost::shared_ptr<Socket>)> m_filterNetworkCallback;
  75. };
  76. class ConnectionBroker
  77. {
  78. public:
  79. typedef boost::shared_ptr<ConnectionBroker> ptr;
  80. typedef boost::weak_ptr<ConnectionBroker> weak_ptr;
  81. public:
  82. ConnectionBroker():
  83. m_verifySslCertificate(false),
  84. m_verifySslCertificateHost(true),
  85. m_httpReadTimeout(~0ull),
  86. m_httpWriteTimeout(~0ull),
  87. m_idleTimeout(~0ull),
  88. m_sslReadTimeout(~0ull),
  89. m_sslWriteTimeout(~0ull),
  90. m_sslCtx(NULL),
  91. m_timerManager(NULL)
  92. {}
  93. virtual ~ConnectionBroker() {}
  94. virtual std::pair<boost::shared_ptr<ClientConnection>, bool>
  95. getConnection(const URI &uri, bool forceNewConnection = false) = 0;
  96. void httpReadTimeout(unsigned long long timeout) { m_httpReadTimeout = timeout; }
  97. void httpWriteTimeout(unsigned long long timeout) { m_httpWriteTimeout = timeout; }
  98. void idleTimeout(unsigned long long timeout) { m_idleTimeout = timeout; }
  99. void sslReadTimeout(unsigned long long timeout) { m_sslReadTimeout = timeout; }
  100. void sslWriteTimeout(unsigned long long timeout) { m_sslWriteTimeout = timeout; }
  101. void sslCtx(SSL_CTX *ctx) { m_sslCtx = ctx; }
  102. void verifySslCertificate(bool verify) { m_verifySslCertificate = verify; }
  103. void verifySslCertificateHost(bool verify) { m_verifySslCertificateHost = verify; }
  104. protected:
  105. void addSSL(const URI &uri, boost::shared_ptr<Stream> &stream);
  106. protected:
  107. bool m_verifySslCertificate, m_verifySslCertificateHost;
  108. unsigned long long m_httpReadTimeout, m_httpWriteTimeout, m_idleTimeout,
  109. m_sslReadTimeout, m_sslWriteTimeout;
  110. SSL_CTX * m_sslCtx;
  111. TimerManager *m_timerManager;
  112. };
  113. struct PriorConnectionFailedException : virtual Exception {};
  114. // The ConnectionCache holds all connections associated with a RequestBroker.
  115. // This is not a global cache of all connections - each RequestBroker instance
  116. // will have its own.
  117. //
  118. // It permits a single RequestBroker to maintain multiple connections to
  119. // multiple servers at the same time. Connections are held active in this cache
  120. // so that multiple requests can be performed over a single connection and
  121. // the cache will take care of automatically reopening connections after they
  122. // close when a new request arrives.
  123. // It understands how to use proxies, but relies on the caller provided callback
  124. // to determine the proxy rules for each request.
  125. //
  126. // Although exposed by createRequestBroker(), normal clients will not manipulate
  127. // the ConnectionCache directly, apart from calling abortConnections or closeIdleConnections
  128. class ConnectionCache : public boost::enable_shared_from_this<ConnectionCache>, public ConnectionBroker
  129. {
  130. public:
  131. typedef boost::shared_ptr<ConnectionCache> ptr;
  132. typedef boost::weak_ptr<ConnectionCache> weak_ptr;
  133. protected:
  134. ConnectionCache(StreamBroker::ptr streamBroker, TimerManager *timerManager = NULL)
  135. : m_streamBroker(streamBroker),
  136. m_connectionsPerHost(1u),
  137. m_requestsPerConnection((size_t)~0),
  138. m_closed(false)
  139. {
  140. m_timerManager = timerManager;
  141. }
  142. public:
  143. static ConnectionCache::ptr create(StreamBroker::ptr streamBroker, TimerManager *timerManager = NULL)
  144. { return ConnectionCache::ptr(new ConnectionCache(streamBroker, timerManager)); }
  145. // Specify the maximum number of seperate connections to allow to a specific host (or proxy)
  146. // at a time
  147. void connectionsPerHost(size_t connections) { m_connectionsPerHost = connections; }
  148. void requestsPerConnection(size_t requests) { m_requestsPerConnection = requests; }
  149. // Get number of active connections
  150. size_t getActiveConnections();
  151. // Proxy support requires this callback. It is expected to return an
  152. // array of candidate Proxy servers to handle the requested URI.
  153. // If none are returned the request will be performed directly
  154. void proxyForURI(boost::function<std::vector<URI> (const URI &)> proxyForURIDg)
  155. { m_proxyForURIDg = proxyForURIDg; }
  156. // Required to support HTTPS proxies
  157. void proxyRequestBroker(boost::shared_ptr<RequestBroker> broker)
  158. { m_proxyBroker = broker; }
  159. // Get the connection associated with a URI. An existing one may be reused,
  160. // or a new one established.
  161. std::pair<boost::shared_ptr<ClientConnection>, bool /*is proxy connection*/>
  162. getConnection(const URI &uri, bool forceNewConnection = false);
  163. void closeIdleConnections();
  164. // Cancel all connections, even the active ones.
  165. // Clients should expect OperationAbortedException, and PriorRequestFailedException
  166. // to be thrown if requests are active
  167. void abortConnections();
  168. private:
  169. typedef std::list<boost::shared_ptr<ClientConnection> > ConnectionList;
  170. // Tracks active connections to a particular host
  171. // e.g. there might be 5 active connections to http://example.com
  172. struct ConnectionInfo
  173. {
  174. ConnectionInfo(FiberMutex &mutex)
  175. : condition(mutex),
  176. lastFailedConnectionTimestamp(~0ull)
  177. {}
  178. ConnectionList connections;
  179. FiberCondition condition;
  180. unsigned long long lastFailedConnectionTimestamp;
  181. };
  182. // Table of active connections for each scheme+host
  183. // e.g. if a single RequestBroker is connected to two servers at the same time
  184. // or doing both http and https requests then this will contain multiple entries
  185. typedef std::map<URI, boost::shared_ptr<ConnectionInfo> > CachedConnectionMap;
  186. private:
  187. std::pair<boost::shared_ptr<ClientConnection>, bool>
  188. getConnectionViaProxyFromCache(const URI &uri, const URI &proxy);
  189. std::pair<boost::shared_ptr<ClientConnection>, bool>
  190. getConnectionViaProxy(const URI &uri, const URI &proxy,
  191. FiberMutex::ScopedLock &lock);
  192. void cleanOutDeadConns(CachedConnectionMap &conns);
  193. void dropConnection(weak_ptr self, const URI &uri, const ClientConnection *connection);
  194. private:
  195. FiberMutex m_mutex;
  196. StreamBroker::ptr m_streamBroker;
  197. size_t m_connectionsPerHost, m_requestsPerConnection;
  198. CachedConnectionMap m_conns;
  199. bool m_closed;
  200. boost::function<std::vector<URI> (const URI &)> m_proxyForURIDg;
  201. boost::shared_ptr<RequestBroker> m_proxyBroker;
  202. };
  203. // The ConnectionNoCache has no support for proxies.
  204. class ConnectionNoCache : public ConnectionBroker
  205. {
  206. public:
  207. typedef boost::shared_ptr<ConnectionNoCache> ptr;
  208. ConnectionNoCache(StreamBroker::ptr streamBroker, TimerManager *timerManager = NULL)
  209. : m_streamBroker(streamBroker)
  210. {
  211. m_timerManager = timerManager;
  212. }
  213. // Get a new connection associated with a URI. Do not cache it.
  214. std::pair<boost::shared_ptr<ClientConnection>, bool /*is proxy connection*/>
  215. getConnection(const URI &uri, bool forceNewConnection = false);
  216. private:
  217. StreamBroker::ptr m_streamBroker;
  218. };
  219. // Mock object useful for unit tests. Rather than
  220. // making a real network call, each request will be
  221. // processed directly by the provided callback
  222. class MockConnectionBroker : public ConnectionBroker
  223. {
  224. private:
  225. typedef std::map<URI, boost::shared_ptr<ClientConnection> >
  226. ConnectionCache; // warning - not the same as class ConnectionCache
  227. public:
  228. MockConnectionBroker(boost::function<void (const URI &uri,
  229. boost::shared_ptr<ServerRequest>)> dg,
  230. TimerManager *timerManager = NULL, unsigned long long readTimeout = ~0ull,
  231. unsigned long long writeTimeout = ~0ull)
  232. : m_dg(dg)
  233. {
  234. m_timerManager = timerManager;
  235. }
  236. std::pair<boost::shared_ptr<ClientConnection>, bool /*is proxy connection*/>
  237. getConnection(const URI &uri, bool forceNewConnection = false);
  238. private:
  239. boost::function<void (const URI &uri, boost::shared_ptr<ServerRequest>)> m_dg;
  240. ConnectionCache m_conns;
  241. };
  242. // Abstract base-class for all the RequestBroker objects.
  243. // RequestBrokers are typically instantiated by calling createRequestBroker().
  244. // RequestBrokers abstract the actual network connection. Reusing a connection
  245. // for multiple requests is achieved by reusing a RequestBroker.
  246. class RequestBroker
  247. {
  248. public:
  249. typedef boost::shared_ptr<RequestBroker> ptr;
  250. typedef boost::weak_ptr<RequestBroker> weak_ptr;
  251. public:
  252. virtual ~RequestBroker() {}
  253. // Perform a request. The caller should prepare the requestHeaders
  254. // as much as they like, and the chain of RequestBrokerFilters will
  255. // also potentially make further adjustments
  256. //
  257. // Tip: Typically the full URI for the destination should be set in
  258. // requestHeaders.requestLine.uri, even though the scheme and authority are
  259. // not sent in the first line of the HTTP request. Also fill in
  260. // the requestHeaders.request.host.
  261. virtual boost::shared_ptr<ClientRequest> request(Request &requestHeaders,
  262. bool forceNewConnection = false,
  263. boost::function<void (boost::shared_ptr<ClientRequest>)> bodyDg = NULL)
  264. = 0;
  265. };
  266. // RequestBrokerFilter is the base class for filter request brokers,
  267. // which exist in a chain leading to the BaseRequestBroker.
  268. // Each derived class can implement special logic in the request()
  269. // method and then call request on the next element in the chain (its parent).
  270. // A filter can stop a request by throwing an exception.
  271. class RequestBrokerFilter : public RequestBroker
  272. {
  273. public:
  274. // When created a RequestBrokerFilter is inserted at the
  275. // beginning of a chain of RequestBrokers, with the parent being the
  276. // next broker in the chain
  277. RequestBrokerFilter(RequestBroker::ptr parent,
  278. RequestBroker::weak_ptr weakParent = RequestBroker::weak_ptr())
  279. : m_parent(parent),
  280. m_weakParent(weakParent)
  281. {}
  282. RequestBroker::ptr parent();
  283. boost::shared_ptr<ClientRequest> request(Request &requestHeaders,
  284. bool forceNewConnection = false,
  285. boost::function<void (boost::shared_ptr<ClientRequest>)> bodyDg = NULL)
  286. = 0;
  287. private:
  288. RequestBroker::ptr m_parent;
  289. RequestBroker::weak_ptr m_weakParent;
  290. };
  291. /// An exception coming from BaseRequestBroker will be tagged with CONNECTION
  292. /// if the exception came while trying to establish the connection, HTTP
  293. /// if the exception came specifically from HTTP communications, or no
  294. /// ExceptionSource at all if it came from other code
  295. enum ExceptionSource
  296. {
  297. CONNECTION,
  298. HTTP
  299. };
  300. typedef boost::error_info<struct tag_source, ExceptionSource > errinfo_source;
  301. // The BaseRequestBroker is the final broker in a chain of requestbrokers
  302. // and sends the fully prepared request to its ConnectionBroker to be performed
  303. class BaseRequestBroker : public RequestBroker
  304. {
  305. public:
  306. typedef boost::shared_ptr<BaseRequestBroker> ptr;
  307. public:
  308. BaseRequestBroker(ConnectionBroker::ptr connectionBroker)
  309. : m_connectionBroker(connectionBroker)
  310. {}
  311. BaseRequestBroker(ConnectionBroker::weak_ptr connectionBroker)
  312. : m_weakConnectionBroker(connectionBroker)
  313. {}
  314. boost::shared_ptr<ClientRequest> request(Request &requestHeaders,
  315. bool forceNewConnection = false,
  316. boost::function<void (boost::shared_ptr<ClientRequest>)> bodyDg = NULL);
  317. private:
  318. ConnectionBroker::ptr m_connectionBroker;
  319. ConnectionBroker::weak_ptr m_weakConnectionBroker;
  320. };
  321. /// Retries connection error and PriorRequestFailed errors
  322. class RetryRequestBroker : public RequestBrokerFilter
  323. {
  324. public:
  325. typedef boost::shared_ptr<RetryRequestBroker> ptr;
  326. public:
  327. RetryRequestBroker(RequestBroker::ptr parent,
  328. boost::function<bool (size_t)> delayDg = NULL)
  329. : RequestBrokerFilter(parent),
  330. m_delayDg(delayDg),
  331. mp_retries(NULL)
  332. {}
  333. void sharedRetryCounter(size_t *retries) { mp_retries = retries; }
  334. boost::shared_ptr<ClientRequest> request(Request &requestHeaders,
  335. bool forceNewConnection = false,
  336. boost::function<void (boost::shared_ptr<ClientRequest>)> bodyDg = NULL);
  337. private:
  338. boost::function<bool (size_t)> m_delayDg;
  339. size_t *mp_retries;
  340. };
  341. struct CircularRedirectException : Exception
  342. {
  343. CircularRedirectException(const URI &uri)
  344. : m_uri(uri)
  345. {}
  346. ~CircularRedirectException() throw() {}
  347. URI uri() { return m_uri; }
  348. private:
  349. URI m_uri;
  350. };
  351. class RedirectRequestBroker : public RequestBrokerFilter
  352. {
  353. public:
  354. typedef boost::shared_ptr<RedirectRequestBroker> ptr;
  355. public:
  356. RedirectRequestBroker(RequestBroker::ptr parent, size_t maxRedirects = 70)
  357. : RequestBrokerFilter(parent),
  358. m_maxRedirects(maxRedirects),
  359. m_handle301(true),
  360. m_handle302(true),
  361. m_handle307(true)
  362. {}
  363. void handlePermanentRedirect(bool handle) { m_handle301 = handle; }
  364. void handleFound(bool handle) { m_handle302 = handle; }
  365. void handleTemporaryRedirect(bool handle) { m_handle307 = handle; }
  366. boost::shared_ptr<ClientRequest> request(Request &requestHeaders,
  367. bool forceNewConnection = false,
  368. boost::function<void (boost::shared_ptr<ClientRequest>)> bodyDg = NULL);
  369. private:
  370. size_t m_maxRedirects;
  371. bool m_handle301, m_handle302, m_handle307;
  372. };
  373. // Takes care of filling in the User-Agent header
  374. class UserAgentRequestBroker : public RequestBrokerFilter
  375. {
  376. public:
  377. UserAgentRequestBroker(RequestBroker::ptr parent,
  378. const ProductAndCommentList &userAgent)
  379. : RequestBrokerFilter(parent),
  380. m_userAgent(userAgent)
  381. {}
  382. boost::shared_ptr<ClientRequest> request(Request &requestHeaders,
  383. bool forceNewConnection = false,
  384. boost::function<void (boost::shared_ptr<ClientRequest>)> bodyDg = NULL);
  385. private:
  386. ProductAndCommentList m_userAgent;
  387. };
  388. // When creating a new RequestBroker this structure is used to
  389. // defines the configuration of a chain of requestBrokers
  390. // and associated services. The most common configuration options
  391. // are exposed here, but further custom RequestBroker behavior
  392. // can be achieved by directly creating RequestBrokerFilter objects
  393. // and adding them to the chain
  394. struct RequestBrokerOptions
  395. {
  396. RequestBrokerOptions() :
  397. ioManager(NULL),
  398. scheduler(NULL),
  399. filterNetworksCB(NULL),
  400. handleRedirects(true),
  401. timerManager(NULL),
  402. connectTimeout(~0ull),
  403. sslConnectReadTimeout(~0ull),
  404. sslConnectWriteTimeout(~0ull),
  405. httpReadTimeout(~0ull),
  406. httpWriteTimeout(~0ull),
  407. idleTimeout(~0ull),
  408. connectionsPerHost(1u),
  409. requestsPerConnection((size_t)~0),
  410. sslCtx(NULL),
  411. verifySslCertificate(false),
  412. verifySslCertificateHost(true),
  413. enableConnectionCache(true)
  414. {}
  415. IOManager *ioManager;
  416. Scheduler *scheduler;
  417. // When specified a RetryRequestBroker will be installed. If a request fails
  418. // the callback will be called with the current retry count. If the callback
  419. // returns false then no further retries are attempted.
  420. boost::function<bool (size_t /*retry count*/)> delayDg;
  421. // Callback to call directly before a socket connection happens.
  422. // Implementation should throw an exception if it wants to prevent the connection
  423. boost::function<void (boost::shared_ptr<Socket>)> filterNetworksCB;
  424. bool handleRedirects; // Whether to add a RedirectRequestBroker to the chain of RequestBrokers
  425. TimerManager *timerManager; // When not specified the iomanager will be used
  426. // Optional timeout values (us)
  427. unsigned long long connectTimeout;
  428. unsigned long long sslConnectReadTimeout;
  429. unsigned long long sslConnectWriteTimeout;
  430. unsigned long long httpReadTimeout;
  431. unsigned long long httpWriteTimeout;
  432. unsigned long long idleTimeout;
  433. size_t connectionsPerHost, requestsPerConnection;
  434. // Callback to find proxy for an URI, see ConnectionCache::proxyForURI
  435. boost::function<std::vector<URI> (const URI &)> proxyForURIDg;
  436. /// Required to enable https proxy support
  437. RequestBroker::ptr proxyRequestBroker;
  438. // When specified these callbacks will be invoked to add authorization to the
  439. // request. An alternative is to add the BasicAuth header before
  440. // calling RequestBroker::request (see HTTP::BasicAuth::authorize())
  441. boost::function<bool (const URI &,
  442. boost::shared_ptr<ClientRequest> /* priorRequest = ClientRequest::ptr() */,
  443. std::string & /* scheme */, std::string & /* realm */,
  444. std::string & /* username */, std::string & /* password */,
  445. size_t /* attempts */)>
  446. getCredentialsDg, getProxyCredentialsDg;
  447. // When specified the provided object will be installed as a filter
  448. // in front of the SocketStreamBroker
  449. StreamBrokerFilter::ptr customStreamBrokerFilter;
  450. SSL_CTX *sslCtx;
  451. bool verifySslCertificate;
  452. bool verifySslCertificateHost;
  453. bool enableConnectionCache;
  454. // When specified a UserAgentRequestBroker will take care of adding
  455. // the User-Agent header to each request
  456. ProductAndCommentList userAgent;
  457. };
  458. // Factory method to create a chain of request broker objects based on the
  459. // the specified configuration options. It also creates and returns a
  460. // shared pointer to the request's new ConnectionCache.
  461. // clients do not typically work directly with the ConnectionCache, except to
  462. // shut down connections with ConnectionCache::abortConnections
  463. std::pair<RequestBroker::ptr, ConnectionCache::ptr>
  464. createRequestBroker(const RequestBrokerOptions &options = RequestBrokerOptions());
  465. /// @deprecated Use createRequestBroker instead
  466. RequestBroker::ptr defaultRequestBroker(IOManager *ioManager = NULL,
  467. Scheduler *scheduler = NULL,
  468. ConnectionBroker::ptr *connBroker = NULL,
  469. boost::function<bool (size_t)> delayDg = NULL);
  470. }}
  471. #endif