/mordor/http/broker.h
C Header | 565 lines | 378 code | 94 blank | 93 comment | 0 complexity | 1eab169475b8564553bc1f66599e5de7 MD5 | raw file
Possible License(s): BSD-3-Clause
- #ifndef __HTTP_BROKER_H__
- #define __HTTP_BROKER_H__
- // Copyright (c) 2009 - Mozy, Inc.
- #include <openssl/ssl.h>
- #include "http.h"
- #include "mordor/fibersynchronization.h"
- namespace Mordor {
- class IOManager;
- class Scheduler;
- class Socket;
- class Stream;
- class TimerManager;
- namespace HTTP {
- class ClientConnection;
- class ClientRequest;
- class RequestBroker;
- class ServerConnection;
- class ServerRequest;
- class StreamBroker
- {
- public:
- typedef boost::shared_ptr<StreamBroker> ptr;
- typedef boost::weak_ptr<StreamBroker> weak_ptr;
- public:
- virtual ~StreamBroker() {}
- virtual boost::shared_ptr<Stream> getStream(const URI &uri) = 0;
- virtual void cancelPending() {}
- };
- class StreamBrokerFilter : public StreamBroker
- {
- public:
- typedef boost::shared_ptr<StreamBrokerFilter> ptr;
- public:
- StreamBrokerFilter(StreamBroker::ptr parent,
- StreamBroker::weak_ptr weakParent = StreamBroker::weak_ptr())
- : m_parent(parent),
- m_weakParent(weakParent)
- {}
- StreamBroker::ptr parent();
- void parent(StreamBroker::ptr parent)
- { m_parent = parent; m_weakParent.reset(); }
- void parent(StreamBroker::weak_ptr parent)
- { m_weakParent = parent; m_parent.reset(); }
- void cancelPending() { parent()->cancelPending(); }
- private:
- StreamBroker::ptr m_parent;
- StreamBroker::weak_ptr m_weakParent;
- };
- class SocketStreamBroker : public StreamBroker
- {
- public:
- typedef boost::shared_ptr<SocketStreamBroker> ptr;
- public:
- SocketStreamBroker(IOManager *ioManager = NULL, Scheduler *scheduler = NULL)
- : m_cancelled(false),
- m_ioManager(ioManager),
- m_scheduler(scheduler),
- m_connectTimeout(~0ull),
- m_filterNetworkCallback(NULL)
- {}
- void connectTimeout(unsigned long long timeout) { m_connectTimeout = timeout; }
- // Resolve the uri to its IP address, create a socket, then connect
- boost::shared_ptr<Stream> getStream(const URI &uri);
- void cancelPending();
- void networkFilterCallback(boost::function<void (boost::shared_ptr<Socket>)> fnCallback)
- { m_filterNetworkCallback = fnCallback; }
- private:
- boost::mutex m_mutex;
- bool m_cancelled;
- std::list<boost::shared_ptr<Socket> > m_pending; // Multiple connections may be attempted when getaddrinfo returns multiple addresses
- IOManager *m_ioManager;
- Scheduler *m_scheduler;
- unsigned long long m_connectTimeout;
- boost::function<void (boost::shared_ptr<Socket>)> m_filterNetworkCallback;
- };
- class ConnectionBroker
- {
- public:
- typedef boost::shared_ptr<ConnectionBroker> ptr;
- typedef boost::weak_ptr<ConnectionBroker> weak_ptr;
- public:
- ConnectionBroker():
- m_verifySslCertificate(false),
- m_verifySslCertificateHost(true),
- m_httpReadTimeout(~0ull),
- m_httpWriteTimeout(~0ull),
- m_idleTimeout(~0ull),
- m_sslReadTimeout(~0ull),
- m_sslWriteTimeout(~0ull),
- m_sslCtx(NULL),
- m_timerManager(NULL)
- {}
- virtual ~ConnectionBroker() {}
- virtual std::pair<boost::shared_ptr<ClientConnection>, bool>
- getConnection(const URI &uri, bool forceNewConnection = false) = 0;
- void httpReadTimeout(unsigned long long timeout) { m_httpReadTimeout = timeout; }
- void httpWriteTimeout(unsigned long long timeout) { m_httpWriteTimeout = timeout; }
- void idleTimeout(unsigned long long timeout) { m_idleTimeout = timeout; }
- void sslReadTimeout(unsigned long long timeout) { m_sslReadTimeout = timeout; }
- void sslWriteTimeout(unsigned long long timeout) { m_sslWriteTimeout = timeout; }
- void sslCtx(SSL_CTX *ctx) { m_sslCtx = ctx; }
- void verifySslCertificate(bool verify) { m_verifySslCertificate = verify; }
- void verifySslCertificateHost(bool verify) { m_verifySslCertificateHost = verify; }
- protected:
- void addSSL(const URI &uri, boost::shared_ptr<Stream> &stream);
- protected:
- bool m_verifySslCertificate, m_verifySslCertificateHost;
- unsigned long long m_httpReadTimeout, m_httpWriteTimeout, m_idleTimeout,
- m_sslReadTimeout, m_sslWriteTimeout;
- SSL_CTX * m_sslCtx;
- TimerManager *m_timerManager;
- };
- struct PriorConnectionFailedException : virtual Exception {};
- // The ConnectionCache holds all connections associated with a RequestBroker.
- // This is not a global cache of all connections - each RequestBroker instance
- // will have its own.
- //
- // It permits a single RequestBroker to maintain multiple connections to
- // multiple servers at the same time. Connections are held active in this cache
- // so that multiple requests can be performed over a single connection and
- // the cache will take care of automatically reopening connections after they
- // close when a new request arrives.
- // It understands how to use proxies, but relies on the caller provided callback
- // to determine the proxy rules for each request.
- //
- // Although exposed by createRequestBroker(), normal clients will not manipulate
- // the ConnectionCache directly, apart from calling abortConnections or closeIdleConnections
- class ConnectionCache : public boost::enable_shared_from_this<ConnectionCache>, public ConnectionBroker
- {
- public:
- typedef boost::shared_ptr<ConnectionCache> ptr;
- typedef boost::weak_ptr<ConnectionCache> weak_ptr;
- protected:
- ConnectionCache(StreamBroker::ptr streamBroker, TimerManager *timerManager = NULL)
- : m_streamBroker(streamBroker),
- m_connectionsPerHost(1u),
- m_requestsPerConnection((size_t)~0),
- m_closed(false)
- {
- m_timerManager = timerManager;
- }
- public:
- static ConnectionCache::ptr create(StreamBroker::ptr streamBroker, TimerManager *timerManager = NULL)
- { return ConnectionCache::ptr(new ConnectionCache(streamBroker, timerManager)); }
- // Specify the maximum number of seperate connections to allow to a specific host (or proxy)
- // at a time
- void connectionsPerHost(size_t connections) { m_connectionsPerHost = connections; }
- void requestsPerConnection(size_t requests) { m_requestsPerConnection = requests; }
- // Get number of active connections
- size_t getActiveConnections();
- // Proxy support requires this callback. It is expected to return an
- // array of candidate Proxy servers to handle the requested URI.
- // If none are returned the request will be performed directly
- void proxyForURI(boost::function<std::vector<URI> (const URI &)> proxyForURIDg)
- { m_proxyForURIDg = proxyForURIDg; }
- // Required to support HTTPS proxies
- void proxyRequestBroker(boost::shared_ptr<RequestBroker> broker)
- { m_proxyBroker = broker; }
- // Get the connection associated with a URI. An existing one may be reused,
- // or a new one established.
- std::pair<boost::shared_ptr<ClientConnection>, bool /*is proxy connection*/>
- getConnection(const URI &uri, bool forceNewConnection = false);
- void closeIdleConnections();
- // Cancel all connections, even the active ones.
- // Clients should expect OperationAbortedException, and PriorRequestFailedException
- // to be thrown if requests are active
- void abortConnections();
- private:
- typedef std::list<boost::shared_ptr<ClientConnection> > ConnectionList;
- // Tracks active connections to a particular host
- // e.g. there might be 5 active connections to http://example.com
- struct ConnectionInfo
- {
- ConnectionInfo(FiberMutex &mutex)
- : condition(mutex),
- lastFailedConnectionTimestamp(~0ull)
- {}
- ConnectionList connections;
- FiberCondition condition;
- unsigned long long lastFailedConnectionTimestamp;
- };
- // Table of active connections for each scheme+host
- // e.g. if a single RequestBroker is connected to two servers at the same time
- // or doing both http and https requests then this will contain multiple entries
- typedef std::map<URI, boost::shared_ptr<ConnectionInfo> > CachedConnectionMap;
- private:
- std::pair<boost::shared_ptr<ClientConnection>, bool>
- getConnectionViaProxyFromCache(const URI &uri, const URI &proxy);
- std::pair<boost::shared_ptr<ClientConnection>, bool>
- getConnectionViaProxy(const URI &uri, const URI &proxy,
- FiberMutex::ScopedLock &lock);
- void cleanOutDeadConns(CachedConnectionMap &conns);
- void dropConnection(weak_ptr self, const URI &uri, const ClientConnection *connection);
- private:
- FiberMutex m_mutex;
- StreamBroker::ptr m_streamBroker;
- size_t m_connectionsPerHost, m_requestsPerConnection;
- CachedConnectionMap m_conns;
- bool m_closed;
- boost::function<std::vector<URI> (const URI &)> m_proxyForURIDg;
- boost::shared_ptr<RequestBroker> m_proxyBroker;
- };
- // The ConnectionNoCache has no support for proxies.
- class ConnectionNoCache : public ConnectionBroker
- {
- public:
- typedef boost::shared_ptr<ConnectionNoCache> ptr;
- ConnectionNoCache(StreamBroker::ptr streamBroker, TimerManager *timerManager = NULL)
- : m_streamBroker(streamBroker)
- {
- m_timerManager = timerManager;
- }
- // Get a new connection associated with a URI. Do not cache it.
- std::pair<boost::shared_ptr<ClientConnection>, bool /*is proxy connection*/>
- getConnection(const URI &uri, bool forceNewConnection = false);
- private:
- StreamBroker::ptr m_streamBroker;
- };
- // Mock object useful for unit tests. Rather than
- // making a real network call, each request will be
- // processed directly by the provided callback
- class MockConnectionBroker : public ConnectionBroker
- {
- private:
- typedef std::map<URI, boost::shared_ptr<ClientConnection> >
- ConnectionCache; // warning - not the same as class ConnectionCache
- public:
- MockConnectionBroker(boost::function<void (const URI &uri,
- boost::shared_ptr<ServerRequest>)> dg,
- TimerManager *timerManager = NULL, unsigned long long readTimeout = ~0ull,
- unsigned long long writeTimeout = ~0ull)
- : m_dg(dg)
- {
- m_timerManager = timerManager;
- }
- std::pair<boost::shared_ptr<ClientConnection>, bool /*is proxy connection*/>
- getConnection(const URI &uri, bool forceNewConnection = false);
- private:
- boost::function<void (const URI &uri, boost::shared_ptr<ServerRequest>)> m_dg;
- ConnectionCache m_conns;
- };
- // Abstract base-class for all the RequestBroker objects.
- // RequestBrokers are typically instantiated by calling createRequestBroker().
- // RequestBrokers abstract the actual network connection. Reusing a connection
- // for multiple requests is achieved by reusing a RequestBroker.
- class RequestBroker
- {
- public:
- typedef boost::shared_ptr<RequestBroker> ptr;
- typedef boost::weak_ptr<RequestBroker> weak_ptr;
- public:
- virtual ~RequestBroker() {}
- // Perform a request. The caller should prepare the requestHeaders
- // as much as they like, and the chain of RequestBrokerFilters will
- // also potentially make further adjustments
- //
- // Tip: Typically the full URI for the destination should be set in
- // requestHeaders.requestLine.uri, even though the scheme and authority are
- // not sent in the first line of the HTTP request. Also fill in
- // the requestHeaders.request.host.
- virtual boost::shared_ptr<ClientRequest> request(Request &requestHeaders,
- bool forceNewConnection = false,
- boost::function<void (boost::shared_ptr<ClientRequest>)> bodyDg = NULL)
- = 0;
- };
- // RequestBrokerFilter is the base class for filter request brokers,
- // which exist in a chain leading to the BaseRequestBroker.
- // Each derived class can implement special logic in the request()
- // method and then call request on the next element in the chain (its parent).
- // A filter can stop a request by throwing an exception.
- class RequestBrokerFilter : public RequestBroker
- {
- public:
- // When created a RequestBrokerFilter is inserted at the
- // beginning of a chain of RequestBrokers, with the parent being the
- // next broker in the chain
- RequestBrokerFilter(RequestBroker::ptr parent,
- RequestBroker::weak_ptr weakParent = RequestBroker::weak_ptr())
- : m_parent(parent),
- m_weakParent(weakParent)
- {}
- RequestBroker::ptr parent();
- boost::shared_ptr<ClientRequest> request(Request &requestHeaders,
- bool forceNewConnection = false,
- boost::function<void (boost::shared_ptr<ClientRequest>)> bodyDg = NULL)
- = 0;
- private:
- RequestBroker::ptr m_parent;
- RequestBroker::weak_ptr m_weakParent;
- };
- /// An exception coming from BaseRequestBroker will be tagged with CONNECTION
- /// if the exception came while trying to establish the connection, HTTP
- /// if the exception came specifically from HTTP communications, or no
- /// ExceptionSource at all if it came from other code
- enum ExceptionSource
- {
- CONNECTION,
- HTTP
- };
- typedef boost::error_info<struct tag_source, ExceptionSource > errinfo_source;
- // The BaseRequestBroker is the final broker in a chain of requestbrokers
- // and sends the fully prepared request to its ConnectionBroker to be performed
- class BaseRequestBroker : public RequestBroker
- {
- public:
- typedef boost::shared_ptr<BaseRequestBroker> ptr;
- public:
- BaseRequestBroker(ConnectionBroker::ptr connectionBroker)
- : m_connectionBroker(connectionBroker)
- {}
- BaseRequestBroker(ConnectionBroker::weak_ptr connectionBroker)
- : m_weakConnectionBroker(connectionBroker)
- {}
- boost::shared_ptr<ClientRequest> request(Request &requestHeaders,
- bool forceNewConnection = false,
- boost::function<void (boost::shared_ptr<ClientRequest>)> bodyDg = NULL);
- private:
- ConnectionBroker::ptr m_connectionBroker;
- ConnectionBroker::weak_ptr m_weakConnectionBroker;
- };
- /// Retries connection error and PriorRequestFailed errors
- class RetryRequestBroker : public RequestBrokerFilter
- {
- public:
- typedef boost::shared_ptr<RetryRequestBroker> ptr;
- public:
- RetryRequestBroker(RequestBroker::ptr parent,
- boost::function<bool (size_t)> delayDg = NULL)
- : RequestBrokerFilter(parent),
- m_delayDg(delayDg),
- mp_retries(NULL)
- {}
- void sharedRetryCounter(size_t *retries) { mp_retries = retries; }
- boost::shared_ptr<ClientRequest> request(Request &requestHeaders,
- bool forceNewConnection = false,
- boost::function<void (boost::shared_ptr<ClientRequest>)> bodyDg = NULL);
- private:
- boost::function<bool (size_t)> m_delayDg;
- size_t *mp_retries;
- };
- struct CircularRedirectException : Exception
- {
- CircularRedirectException(const URI &uri)
- : m_uri(uri)
- {}
- ~CircularRedirectException() throw() {}
- URI uri() { return m_uri; }
- private:
- URI m_uri;
- };
- class RedirectRequestBroker : public RequestBrokerFilter
- {
- public:
- typedef boost::shared_ptr<RedirectRequestBroker> ptr;
- public:
- RedirectRequestBroker(RequestBroker::ptr parent, size_t maxRedirects = 70)
- : RequestBrokerFilter(parent),
- m_maxRedirects(maxRedirects),
- m_handle301(true),
- m_handle302(true),
- m_handle307(true)
- {}
- void handlePermanentRedirect(bool handle) { m_handle301 = handle; }
- void handleFound(bool handle) { m_handle302 = handle; }
- void handleTemporaryRedirect(bool handle) { m_handle307 = handle; }
- boost::shared_ptr<ClientRequest> request(Request &requestHeaders,
- bool forceNewConnection = false,
- boost::function<void (boost::shared_ptr<ClientRequest>)> bodyDg = NULL);
- private:
- size_t m_maxRedirects;
- bool m_handle301, m_handle302, m_handle307;
- };
- // Takes care of filling in the User-Agent header
- class UserAgentRequestBroker : public RequestBrokerFilter
- {
- public:
- UserAgentRequestBroker(RequestBroker::ptr parent,
- const ProductAndCommentList &userAgent)
- : RequestBrokerFilter(parent),
- m_userAgent(userAgent)
- {}
- boost::shared_ptr<ClientRequest> request(Request &requestHeaders,
- bool forceNewConnection = false,
- boost::function<void (boost::shared_ptr<ClientRequest>)> bodyDg = NULL);
- private:
- ProductAndCommentList m_userAgent;
- };
- // When creating a new RequestBroker this structure is used to
- // defines the configuration of a chain of requestBrokers
- // and associated services. The most common configuration options
- // are exposed here, but further custom RequestBroker behavior
- // can be achieved by directly creating RequestBrokerFilter objects
- // and adding them to the chain
- struct RequestBrokerOptions
- {
- RequestBrokerOptions() :
- ioManager(NULL),
- scheduler(NULL),
- filterNetworksCB(NULL),
- handleRedirects(true),
- timerManager(NULL),
- connectTimeout(~0ull),
- sslConnectReadTimeout(~0ull),
- sslConnectWriteTimeout(~0ull),
- httpReadTimeout(~0ull),
- httpWriteTimeout(~0ull),
- idleTimeout(~0ull),
- connectionsPerHost(1u),
- requestsPerConnection((size_t)~0),
- sslCtx(NULL),
- verifySslCertificate(false),
- verifySslCertificateHost(true),
- enableConnectionCache(true)
- {}
- IOManager *ioManager;
- Scheduler *scheduler;
- // When specified a RetryRequestBroker will be installed. If a request fails
- // the callback will be called with the current retry count. If the callback
- // returns false then no further retries are attempted.
- boost::function<bool (size_t /*retry count*/)> delayDg;
- // Callback to call directly before a socket connection happens.
- // Implementation should throw an exception if it wants to prevent the connection
- boost::function<void (boost::shared_ptr<Socket>)> filterNetworksCB;
- bool handleRedirects; // Whether to add a RedirectRequestBroker to the chain of RequestBrokers
- TimerManager *timerManager; // When not specified the iomanager will be used
- // Optional timeout values (us)
- unsigned long long connectTimeout;
- unsigned long long sslConnectReadTimeout;
- unsigned long long sslConnectWriteTimeout;
- unsigned long long httpReadTimeout;
- unsigned long long httpWriteTimeout;
- unsigned long long idleTimeout;
- size_t connectionsPerHost, requestsPerConnection;
- // Callback to find proxy for an URI, see ConnectionCache::proxyForURI
- boost::function<std::vector<URI> (const URI &)> proxyForURIDg;
- /// Required to enable https proxy support
- RequestBroker::ptr proxyRequestBroker;
- // When specified these callbacks will be invoked to add authorization to the
- // request. An alternative is to add the BasicAuth header before
- // calling RequestBroker::request (see HTTP::BasicAuth::authorize())
- boost::function<bool (const URI &,
- boost::shared_ptr<ClientRequest> /* priorRequest = ClientRequest::ptr() */,
- std::string & /* scheme */, std::string & /* realm */,
- std::string & /* username */, std::string & /* password */,
- size_t /* attempts */)>
- getCredentialsDg, getProxyCredentialsDg;
- // When specified the provided object will be installed as a filter
- // in front of the SocketStreamBroker
- StreamBrokerFilter::ptr customStreamBrokerFilter;
- SSL_CTX *sslCtx;
- bool verifySslCertificate;
- bool verifySslCertificateHost;
- bool enableConnectionCache;
- // When specified a UserAgentRequestBroker will take care of adding
- // the User-Agent header to each request
- ProductAndCommentList userAgent;
- };
- // Factory method to create a chain of request broker objects based on the
- // the specified configuration options. It also creates and returns a
- // shared pointer to the request's new ConnectionCache.
- // clients do not typically work directly with the ConnectionCache, except to
- // shut down connections with ConnectionCache::abortConnections
- std::pair<RequestBroker::ptr, ConnectionCache::ptr>
- createRequestBroker(const RequestBrokerOptions &options = RequestBrokerOptions());
- /// @deprecated Use createRequestBroker instead
- RequestBroker::ptr defaultRequestBroker(IOManager *ioManager = NULL,
- Scheduler *scheduler = NULL,
- ConnectionBroker::ptr *connBroker = NULL,
- boost::function<bool (size_t)> delayDg = NULL);
- }}
- #endif