/mordor/http/broker.cpp

  1// Copyright (c) 2009 - Mozy, Inc.
  2
  3#include "broker.h"
  4
  5#include "auth.h"
  6#include "client.h"
  7#include "mordor/atomic.h"
  8#include "mordor/fiber.h"
  9#include "mordor/future.h"
 10#include "mordor/iomanager.h"
 11#include "mordor/log.h"
 12#include "mordor/socks.h"
 13#include "mordor/streams/buffered.h"
 14#include "mordor/streams/pipe.h"
 15#include "mordor/streams/socket.h"
 16#include "mordor/streams/ssl.h"
 17#include "mordor/streams/timeout.h"
 18#include "proxy.h"
 19#include "server.h"
 20
 21namespace Mordor {
 22namespace HTTP {
 23
 24std::pair<RequestBroker::ptr, ConnectionCache::ptr>
 25createRequestBroker(const RequestBrokerOptions &options)
 26{
 27    TimerManager *timerManager = const_cast<TimerManager*>(options.timerManager);
 28    if (options.ioManager && !timerManager)
 29        timerManager = const_cast<IOManager*>(options.ioManager);
 30
 31    SocketStreamBroker::ptr socketBroker(new SocketStreamBroker(options.ioManager,
 32        options.scheduler));
 33    socketBroker->connectTimeout(options.connectTimeout);
 34    socketBroker->networkFilterCallback(options.filterNetworksCB);
 35
 36    StreamBroker::ptr streamBroker = socketBroker;
 37    if (options.customStreamBrokerFilter) {
 38        options.customStreamBrokerFilter->parent(streamBroker);
 39        streamBroker = options.customStreamBrokerFilter;
 40    }
 41
 42    ConnectionBroker::ptr connectionBroker;
 43    ConnectionCache::ptr connectionCache;
 44    if (!options.enableConnectionCache) {
 45        ConnectionNoCache::ptr connectionNoCache(new ConnectionNoCache(streamBroker,
 46            timerManager));
 47        connectionBroker = boost::static_pointer_cast<ConnectionBroker>(connectionNoCache);
 48    } else {
 49        connectionCache = ConnectionCache::create(streamBroker, timerManager);
 50        connectionCache->idleTimeout(options.idleTimeout);
 51        connectionCache->proxyForURI(options.proxyForURIDg);
 52        connectionCache->proxyRequestBroker(options.proxyRequestBroker);
 53        connectionCache->connectionsPerHost(options.connectionsPerHost);
 54        connectionCache->requestsPerConnection(options.requestsPerConnection);
 55        connectionBroker = boost::static_pointer_cast<ConnectionBroker>(connectionCache);
 56    }
 57    connectionBroker->httpReadTimeout(options.httpReadTimeout);
 58    connectionBroker->httpWriteTimeout(options.httpWriteTimeout);
 59    connectionBroker->sslReadTimeout(options.sslConnectReadTimeout);
 60    connectionBroker->sslWriteTimeout(options.sslConnectWriteTimeout);
 61    connectionBroker->sslCtx(options.sslCtx);
 62    connectionBroker->verifySslCertificate(options.verifySslCertificate);
 63    connectionBroker->verifySslCertificateHost(options.verifySslCertificateHost);
 64
 65    RequestBroker::ptr requestBroker(new BaseRequestBroker(connectionBroker));
 66
 67    if (options.getCredentialsDg || options.getProxyCredentialsDg)
 68        requestBroker.reset(new AuthRequestBroker(requestBroker,
 69            options.getCredentialsDg, options.getProxyCredentialsDg));
 70    if (options.handleRedirects)
 71        requestBroker.reset(new RedirectRequestBroker(requestBroker));
 72    if (options.delayDg)
 73        requestBroker.reset(new RetryRequestBroker(requestBroker,
 74        options.delayDg));
 75    if (!options.userAgent.empty())
 76        requestBroker.reset(new UserAgentRequestBroker(requestBroker,
 77            options.userAgent));
 78    return std::make_pair(requestBroker, connectionCache);
 79}
 80
 81RequestBroker::ptr defaultRequestBroker(IOManager *ioManager,
 82                                        Scheduler *scheduler,
 83                                        ConnectionBroker::ptr *connBroker,
 84                                        boost::function<bool (size_t)> delayDg)
 85{
 86   RequestBrokerOptions options;
 87   options.ioManager = ioManager;
 88   options.scheduler = scheduler;
 89   options.delayDg = delayDg;
 90   std::pair<RequestBroker::ptr, ConnectionCache::ptr> result = createRequestBroker(options);
 91   if (connBroker)
 92       *connBroker = result.second;
 93   return result.first;
 94}
 95
 96
 97StreamBroker::ptr
 98StreamBrokerFilter::parent()
 99{
100    if (m_parent)
101        return m_parent;
102    return StreamBroker::ptr(m_weakParent);
103}
104
105Stream::ptr
106SocketStreamBroker::getStream(const URI &uri)
107{
108    if (m_cancelled)
109        MORDOR_THROW_EXCEPTION(OperationAbortedException());
110
111    MORDOR_ASSERT(uri.authority.hostDefined());
112    MORDOR_ASSERT(uri.authority.portDefined() || uri.schemeDefined());
113    std::ostringstream os;
114    os << uri.authority.host();
115    if (uri.authority.portDefined())
116        os << ":" << uri.authority.port();
117    else if (uri.schemeDefined())
118        os << ":" << uri.scheme();
119    std::vector<Address::ptr> addresses;
120    {
121        SchedulerSwitcher switcher(m_scheduler);
122        addresses = Address::lookup(os.str());
123    }
124    Socket::ptr socket;
125    for (std::vector<Address::ptr>::const_iterator it(addresses.begin());
126        it != addresses.end();
127        )
128    {
129        if (m_ioManager)
130            socket = (*it)->createSocket(*m_ioManager, SOCK_STREAM);
131        else
132            socket = (*it)->createSocket(SOCK_STREAM);
133        std::list<Socket::ptr>::iterator it2;
134        {
135            boost::mutex::scoped_lock lock(m_mutex);
136            if (m_cancelled)
137                MORDOR_THROW_EXCEPTION(OperationAbortedException());
138            m_pending.push_back(socket);
139            it2 = m_pending.end();
140            --it2;
141        }
142        socket->sendTimeout(m_connectTimeout);
143        try {
144            // if we are filtering network connections, the callback will bind
145            // the socket to an approved network address
146            if (m_filterNetworkCallback != NULL)
147            {
148                // the callback function is responsible for exiting out by throwing
149                // an exception.
150                m_filterNetworkCallback(socket);
151            }
152            socket->connect(*it);
153            boost::mutex::scoped_lock lock(m_mutex);
154            m_pending.erase(it2);
155            break;
156        } catch (...) {
157            boost::mutex::scoped_lock lock(m_mutex);
158            m_pending.erase(it2);
159            if (++it == addresses.end())
160                throw;
161        }
162        socket->sendTimeout(~0ull);
163    }
164    Stream::ptr stream(new SocketStream(socket));
165    return stream;
166}
167
168void
169SocketStreamBroker::cancelPending()
170{
171    boost::mutex::scoped_lock lock(m_mutex);
172    m_cancelled = true;
173    for (std::list<Socket::ptr>::iterator it(m_pending.begin());
174        it != m_pending.end();
175        ++it) {
176        (*it)->cancelConnect();
177        (*it)->cancelSend();
178        (*it)->cancelReceive();
179    }
180}
181
182static bool least(const ClientConnection::ptr &lhs,
183                  const ClientConnection::ptr &rhs)
184{
185    if (lhs && rhs)
186        return lhs->outstandingRequests() <
187            rhs->outstandingRequests();
188    if (!lhs)
189        return false;
190    if (!rhs)
191        return true;
192    MORDOR_NOTREACHED();
193}
194
195static Logger::ptr g_cacheLog = Log::lookup("mordor:http:connectioncache");
196
197std::pair<ClientConnection::ptr, bool>
198ConnectionCache::getConnection(const URI &uri, bool forceNewConnection)
199{
200    std::vector<URI> proxies;
201    if (m_proxyForURIDg)
202        proxies = m_proxyForURIDg(uri);
203    // Remove proxy types that aren't supported
204    for (std::vector<URI>::iterator it(proxies.begin());
205        it != proxies.end();
206        ++it) {
207        MORDOR_ASSERT(it->schemeDefined() || !it->isDefined());
208        if (!it->schemeDefined())
209            continue;
210        std::string scheme = it->scheme();
211        if (scheme != "http" && (scheme != "https" || !m_proxyBroker) &&
212            scheme != "socks")
213            it = proxies.erase(it, it);
214    }
215    URI schemeAndAuthority;
216    schemeAndAuthority = uri;
217    schemeAndAuthority.path = URI::Path();
218    schemeAndAuthority.queryDefined(false);
219    schemeAndAuthority.fragmentDefined(false);
220    std::pair<ClientConnection::ptr, bool> result;
221
222    FiberMutex::ScopedLock lock(m_mutex);
223
224    if (g_cacheLog->enabled(Log::DEBUG)) {
225        std::ostringstream os;
226        os << this << " getting connection for " << schemeAndAuthority
227        << ", proxies: {";
228        for (std::vector<URI>::iterator it(proxies.begin());
229             it != proxies.end();
230             ++it) {
231            if (it != proxies.begin())
232                os << ", ";
233            os << *it;
234        }
235        os << "}";
236        MORDOR_LOG_DEBUG(g_cacheLog) << os.str();
237    }
238
239    if (m_closed)
240        MORDOR_THROW_EXCEPTION(OperationAbortedException());
241    // Clean out any dead conns
242    cleanOutDeadConns(m_conns);
243
244    if (!forceNewConnection) {
245        if (proxies.empty()) {
246            result = getConnectionViaProxyFromCache(schemeAndAuthority, URI());
247            if (result.first)
248                return result;
249        }
250        for (std::vector<URI>::const_iterator it(proxies.begin());
251            it != proxies.end();
252            ++it) {
253            result = getConnectionViaProxyFromCache(schemeAndAuthority, *it);
254            if (result.first)
255                return result;
256        }
257    }
258
259    // Create a new connection
260    if (proxies.empty())
261        return getConnectionViaProxy(schemeAndAuthority, URI(), lock);
262    std::vector<URI>::const_iterator it = proxies.begin();
263    while(true) {
264        try {
265            return getConnectionViaProxy(schemeAndAuthority, *it, lock);
266        } catch (SocketException &) {
267            if (++it == proxies.end())
268                throw;
269        } catch (HTTP::Exception &) {
270            if (++it == proxies.end())
271                throw;
272        } catch (UnexpectedEofException &) {
273            if (++it == proxies.end())
274                throw;
275        }
276    }
277}
278
279std::pair<ClientConnection::ptr, bool>
280ConnectionNoCache::getConnection(const URI &uri, bool forceNewConnection)
281{
282    URI endPoint = uri;
283    endPoint.path = URI::Path();
284    endPoint.queryDefined(false);
285    endPoint.fragmentDefined(false);
286
287    // Establish a new connection
288    Stream::ptr stream = m_streamBroker->getStream(endPoint);
289    addSSL(endPoint, stream);
290
291    std::pair<ClientConnection::ptr, bool> result = std::make_pair(ClientConnection::ptr(
292        new ClientConnection(stream, m_timerManager)), false);
293    if (m_httpReadTimeout != ~0ull)
294        result.first->readTimeout(m_httpReadTimeout);
295    if (m_httpWriteTimeout != ~0ull)
296        result.first->writeTimeout(m_httpWriteTimeout);
297
298    return result;
299}
300
301std::pair<ClientConnection::ptr, bool>
302ConnectionCache::getConnectionViaProxyFromCache(const URI &uri, const URI &proxy)
303{
304    // Check if an existing connection exists to the requested URI
305    // that should be reused.
306    // When proxy is specified this looks for a connection
307    // to the proxy uri instead
308
309    bool proxied = proxy.schemeDefined() && proxy.scheme() == "http";
310    const URI &endpoint = proxied ? proxy : uri;
311    CachedConnectionMap::iterator it = m_conns.find(endpoint);
312    ConnectionList::iterator it2;
313    while (true) {
314        if (it != m_conns.end() &&
315            !it->second->connections.empty() &&
316            it->second->connections.size() >= m_connectionsPerHost) {
317            boost::shared_ptr<ConnectionInfo> info = it->second;
318            ConnectionList &connsForThisUri = info->connections;
319            // Assign it2 to point to the connection with the
320            // least number of outstanding requests
321            it2 = std::min_element(connsForThisUri.begin(),
322                connsForThisUri.end(), &least);
323            // No connection has completed yet (but it's in progress)
324            if (!*it2) {
325                MORDOR_LOG_TRACE(g_cacheLog) << this << " waiting for connection to "
326                    << endpoint;
327                // Wait for somebody to let us try again
328                unsigned long long start = TimerManager::now();
329                info->condition.wait();
330                if (info->lastFailedConnectionTimestamp <= start)
331                    MORDOR_THROW_EXCEPTION(PriorConnectionFailedException());
332                if (m_closed)
333                    MORDOR_THROW_EXCEPTION(OperationAbortedException());
334                // We let go of the mutex, and the last connection may have
335                // disappeared
336                it = m_conns.find(endpoint);
337            } else {
338                MORDOR_LOG_TRACE(g_cacheLog) << this << " returning cached connection "
339                    << *it2 << " to " << endpoint;
340                // Return the existing, completed connection
341                return std::make_pair(*it2, proxied);
342            }
343        } else {
344            // No existing connections
345            return std::make_pair(ClientConnection::ptr(), false);
346        }
347    }
348}
349
350std::pair<ClientConnection::ptr, bool>
351ConnectionCache::getConnectionViaProxy(const URI &uri, const URI &proxy,
352    FiberMutex::ScopedLock &lock)
353{
354    // Create a new Connection to the requested URI, using
355    // the proxy if specified
356
357    std::string proxyScheme;
358    if (proxy.schemeDefined())
359        proxyScheme = proxy.scheme();
360    bool proxied = proxyScheme == "http";
361    const URI &endpoint = proxied ? proxy : uri;
362
363    // Make sure we have a ConnectionList and mutex for this endpoint
364    CachedConnectionMap::iterator it = m_conns.find(endpoint);
365    boost::shared_ptr<ConnectionInfo> info;
366    if (it == m_conns.end()) {
367        info.reset(new ConnectionInfo(m_mutex));
368        it = m_conns.insert(std::make_pair(endpoint, info)).first;
369    } else {
370        info = it->second;
371    }
372    // Add a placeholder for the new connection
373    info->connections.push_back(ClientConnection::ptr());
374
375    MORDOR_LOG_TRACE(g_cacheLog) << this << " establishing connection to "
376        << endpoint;
377    unsigned long long start = TimerManager::now();
378    lock.unlock();
379
380    ConnectionList::iterator it2;
381    std::pair<ClientConnection::ptr, bool> result;
382    // Establish a new connection
383    try {
384        Stream::ptr stream;
385        if (proxyScheme == "https") {
386            stream = tunnel(m_proxyBroker, proxy, uri);
387        } else if (proxyScheme == "socks") {
388            unsigned short port;
389            if (uri.authority.portDefined())
390                port = uri.authority.port();
391            else if (uri.scheme() == "http")
392                port = 80;
393            else if (uri.scheme() == "https")
394                port = 443;
395            else
396                // TODO: can this be looked up using the system? (getaddrinfo)
397                MORDOR_THROW_EXCEPTION(std::invalid_argument("Unknown protocol for proxying connection"));
398            stream = SOCKS::tunnel(m_streamBroker, proxy, IPAddress::ptr(),
399                uri.authority.host(), port);
400        } else {
401            stream = m_streamBroker->getStream(endpoint);
402        }
403        addSSL(endpoint, stream);
404        lock.lock();
405        // Somebody called abortConnections while we were unlocked; just throw
406        // this connection away
407        if (m_closed)
408            MORDOR_THROW_EXCEPTION(OperationAbortedException());
409        result = std::make_pair(ClientConnection::ptr(
410            new ClientConnection(stream, m_timerManager)), proxied);
411        MORDOR_LOG_TRACE(g_cacheLog) << this << " connection " << result.first
412            << " to " << endpoint << " established";
413        result.first->maxRequestCount(m_requestsPerConnection);
414        stream->onRemoteClose(boost::bind(&ConnectionCache::dropConnection,
415            this, weak_ptr(shared_from_this()), endpoint, result.first.get()));
416        if (m_httpReadTimeout != ~0ull)
417            result.first->readTimeout(m_httpReadTimeout);
418        if (m_httpWriteTimeout != ~0ull)
419            result.first->writeTimeout(m_httpWriteTimeout);
420        if (m_idleTimeout != ~0ull)
421            result.first->idleTimeout(m_idleTimeout,
422            boost::bind(&ConnectionCache::dropConnection,
423                this, weak_ptr(shared_from_this()), endpoint, result.first.get()));
424        // Assign this connection to the first blank connection for this
425        // schemeAndAuthority
426        for (it2 = info->connections.begin();
427            it2 != info->connections.end();
428            ++it2) {
429            if (!*it2) {
430                *it2 = result.first;
431                break;
432            }
433        }
434        // We should have assigned this connection *somewhere*
435        MORDOR_ASSERT(it2 != info->connections.end());
436        // Unblock all waiters for them to choose an existing connection
437        info->condition.broadcast();
438    } catch (...) {
439        lock.lock();
440        MORDOR_LOG_TRACE(g_cacheLog) << this << " connection to " << endpoint
441            << " failed: " << boost::current_exception_diagnostic_information();
442        // Somebody called abortConnections while we were unlocked; no need to
443        // clean up the temporary spot for this connection, since it's gone;
444        // pass the original exception on, though
445        if (m_closed)
446            throw;
447        // This connection attempt failed; remove the first blank connection
448        // for this schemeAndAuthority to let someone else try to establish a
449        // connection
450        // it should still be valid, even if the map changed
451        for (it2 = info->connections.begin();
452            it2 != info->connections.end();
453            ++it2) {
454            if (!*it2) {
455                info->connections.erase(it2);
456                break;
457            }
458        }
459        info->lastFailedConnectionTimestamp = start;
460        info->condition.broadcast();
461        if (info->connections.empty())
462            m_conns.erase(it);
463        throw;
464    }
465    return result;
466}
467
468void
469ConnectionCache::closeIdleConnections()
470{
471    FiberMutex::ScopedLock lock(m_mutex);
472    MORDOR_LOG_DEBUG(g_cacheLog) << " dropping idle connections";
473    // We don't just clear the list, because there may be a connection in
474    // progress that has an iterator into it
475    CachedConnectionMap::iterator it, extraIt;
476    for (it = m_conns.begin(); it != m_conns.end();) {
477        it->second->condition.broadcast();
478        for (ConnectionList::iterator it2 = it->second->connections.begin();
479            it2 != it->second->connections.end();) {
480            if (*it2) {
481                Stream::ptr connStream = (*it2)->stream();
482                connStream->cancelRead();
483                connStream->cancelWrite();
484                if (m_idleTimeout != ~0ull)
485                    (*it2)->idleTimeout(~0ull, NULL);
486                it2 = it->second->connections.erase(it2);
487            } else {
488                ++it2;
489            }
490        }
491        if (it->second->connections.empty()) {
492            extraIt = it;
493            ++it;
494            m_conns.erase(extraIt);
495        } else {
496            ++it;
497        }
498    }
499}
500
501void
502ConnectionCache::abortConnections()
503{
504    FiberMutex::ScopedLock lock(m_mutex);
505    MORDOR_LOG_DEBUG(g_cacheLog) << " aborting all connections";
506    m_closed = true;
507    CachedConnectionMap::iterator it;
508    for (it = m_conns.begin(); it != m_conns.end(); ++it) {
509        it->second->condition.broadcast();
510        for (ConnectionList::iterator it2 = it->second->connections.begin();
511            it2 != it->second->connections.end();
512            ++it2) {
513            if (*it2) {
514                Stream::ptr connStream = (*it2)->stream();
515                connStream->cancelRead();
516                connStream->cancelWrite();
517                if (m_idleTimeout != ~0ull)
518                    (*it2)->idleTimeout(~0ull, NULL);
519            }
520        }
521    }
522    m_conns.clear();
523    lock.unlock();
524    m_streamBroker->cancelPending();
525}
526
527void
528ConnectionCache::cleanOutDeadConns(CachedConnectionMap &conns)
529{
530    CachedConnectionMap::iterator it, it3;
531    ConnectionList::iterator it2;
532    for (it = conns.begin(); it != conns.end();) {
533        for (it2 = it->second->connections.begin();
534            it2 != it->second->connections.end();) {
535            if (*it2 && !(*it2)->newRequestsAllowed()) {
536                if (m_idleTimeout != ~0ull)
537                    (*it2)->idleTimeout(~0ull, NULL);
538                it2 = it->second->connections.erase(it2);
539            } else {
540                ++it2;
541            }
542        }
543        if (it->second->connections.empty()) {
544            it3 = it;
545            ++it3;
546            conns.erase(it);
547            it = it3;
548        } else {
549            ++it;
550        }
551    }
552}
553
554void
555ConnectionBroker::addSSL(const URI &uri, Stream::ptr &stream)
556{
557    if (uri.schemeDefined() && uri.scheme() == "https") {
558        TimeoutStream::ptr timeoutStream;
559        if (m_timerManager) {
560            timeoutStream.reset(new TimeoutStream(stream, *m_timerManager));
561            timeoutStream->readTimeout(m_sslReadTimeout);
562            timeoutStream->writeTimeout(m_sslWriteTimeout);
563            stream = timeoutStream;
564        }
565        BufferedStream::ptr bufferedStream(new BufferedStream(stream));
566        bufferedStream->allowPartialReads(true);
567        SSLStream::ptr sslStream(new SSLStream(bufferedStream, true, true, m_sslCtx));
568        // Only do SNI when required to verify host name
569        if (m_verifySslCertificateHost)
570            sslStream->serverNameIndication(uri.authority.host());
571        sslStream->connect();
572        if (m_verifySslCertificate)
573            sslStream->verifyPeerCertificate();
574        if (m_verifySslCertificateHost)
575            sslStream->verifyPeerCertificate(uri.authority.host());
576        if (timeoutStream) {
577            bufferedStream->parent(timeoutStream->parent());
578            timeoutStream.reset();
579        }
580        bufferedStream.reset(new BufferedStream(sslStream));
581        // Max data in each SSL record
582        bufferedStream->bufferSize(16384);
583        bufferedStream->flushMultiplesOfBuffer(true);
584        bufferedStream->allowPartialReads(true);
585        stream = bufferedStream;
586    }
587}
588
589namespace {
590struct CompareConn
591{
592    CompareConn(const ClientConnection *conn)
593        : m_conn(conn)
594    {}
595
596    bool operator()(const ClientConnection::ptr &lhs) const
597    {
598        return lhs.get() == m_conn;
599    }
600
601    const ClientConnection *m_conn;
602};
603}
604
605void
606ConnectionCache::dropConnection(weak_ptr self,
607    const URI &uri,
608    const ClientConnection *connection)
609{
610    ptr strongSelf = self.lock();
611    if (!strongSelf) {
612        return;
613    }
614
615    FiberMutex::ScopedLock lock(m_mutex);
616    CachedConnectionMap::iterator it = m_conns.find(uri);
617    if (it == m_conns.end())
618    {
619        MORDOR_LOG_TRACE(g_cacheLog) << this << " Failed to drop connection to " << uri << "not found in our cache ";
620        return;
621    }
622    ConnectionList::iterator it2 = std::find_if(it->second->connections.begin(),
623        it->second->connections.end(), CompareConn(connection));
624    if (it2 != it->second->connections.end()) {
625        MORDOR_LOG_TRACE(g_cacheLog) << this << " dropping connection "
626            << connection << " to " << uri;
627        if (m_idleTimeout != ~0ull)
628            (*it2)->idleTimeout(~0ull, NULL);
629        it->second->connections.erase(it2);
630        if (it->second->connections.empty())
631            m_conns.erase(it);
632    }
633}
634
635// Get the number of active connections
636// Can be used to determine throttling with multiple connections.
637size_t
638ConnectionCache::getActiveConnections()
639{
640    size_t result = 0;
641    size_t tmpCount;
642    FiberMutex::ScopedLock lock(m_mutex);
643    CachedConnectionMap::iterator it;
644    ConnectionList::iterator it2;
645    for (it = m_conns.begin(); it != m_conns.end(); it++) {
646        boost::shared_ptr<ConnectionInfo> info = it->second;
647        tmpCount = 0;
648        for (it2 = info->connections.begin();
649            it2 != info->connections.end();
650            ++it2) {
651            if (*it2 != NULL && (*it2)->outstandingRequests() > 0){
652                tmpCount++;
653            }
654        }
655        MORDOR_LOG_TRACE(g_cacheLog) << this << " Active connections to " << it->first.toString() << " - " << tmpCount;
656        result += tmpCount;
657    }
658    MORDOR_LOG_TRACE(g_cacheLog) << this << " Total connections " << result;
659    return result;
660}
661
662std::pair<ClientConnection::ptr, bool>
663MockConnectionBroker::getConnection(const URI &uri, bool forceNewConnection)
664{
665    URI schemeAndAuthority = uri;
666    schemeAndAuthority.path = URI::Path();
667    schemeAndAuthority.queryDefined(false);
668    schemeAndAuthority.fragmentDefined(false);
669    ConnectionCache::iterator it = m_conns.find(schemeAndAuthority);
670    if (it != m_conns.end() && !it->second->newRequestsAllowed()) {
671        m_conns.erase(it);
672        it = m_conns.end();
673    }
674    if (it == m_conns.end()) {
675        std::pair<Stream::ptr, Stream::ptr> pipes = pipeStream();
676        ClientConnection::ptr client(
677            new ClientConnection(pipes.first, m_timerManager));
678        if (m_timerManager) {
679            client->readTimeout(m_httpReadTimeout);
680            client->writeTimeout(m_httpWriteTimeout);
681        }
682        ServerConnection::ptr server(
683            new ServerConnection(pipes.second, boost::bind(m_dg,
684                schemeAndAuthority, _1)));
685        Scheduler::getThis()->schedule(Fiber::ptr(new Fiber(boost::bind(
686            &ServerConnection::processRequests, server))));
687        m_conns[schemeAndAuthority] = client;
688        return std::make_pair(client, false);
689    }
690    return std::make_pair(it->second, false);
691}
692
693RequestBroker::ptr
694RequestBrokerFilter::parent()
695{
696    if (m_parent)
697        return m_parent;
698    return RequestBroker::ptr(m_weakParent);
699}
700
701static void doBody(ClientRequest::ptr request,
702    boost::function<void (ClientRequest::ptr)> bodyDg,
703    Future<> &future,
704    boost::exception_ptr &exception, bool &exceptionWasHttp)
705{
706    exceptionWasHttp = false;
707    try {
708        bodyDg(request);
709    } catch (boost::exception &ex) {
710        exceptionWasHttp = request->requestState() == ClientRequest::ERROR;
711        if (exceptionWasHttp)
712            ex << errinfo_source(HTTP);
713        // Make sure the request is fully aborted so we don't hang waiting for
714        // a response (since the request object is still in scope by the
715        // caller, it won't do this automatically)
716        request->cancel();
717        exception = boost::current_exception();
718    } catch (...) {
719        exceptionWasHttp = request->requestState() == ClientRequest::ERROR;
720        // Make sure the request is fully aborted so we don't hang waiting for
721        // a response (since the request object is still in scope by the
722        // caller, it won't do this automatically)
723        request->cancel();
724        exception = boost::current_exception();
725    }
726    future.signal();
727}
728
729ClientRequest::ptr
730BaseRequestBroker::request(Request &requestHeaders, bool forceNewConnection,
731                           boost::function<void (ClientRequest::ptr)> bodyDg)
732{
733    URI &currentUri = requestHeaders.requestLine.uri;
734    URI originalUri = currentUri;
735    bool connect = requestHeaders.requestLine.method == CONNECT;
736    MORDOR_ASSERT(originalUri.authority.hostDefined());
737    if (!connect) {
738        requestHeaders.request.host = originalUri.authority.host();
739    } else {
740        MORDOR_ASSERT(originalUri.scheme() == "http");
741        MORDOR_ASSERT(originalUri.path.segments.size() == 2);
742        currentUri = URI();
743        currentUri.authority = originalUri.path.segments[1];
744    }
745    ConnectionBroker::ptr connectionBroker = m_connectionBroker;
746    if (!connectionBroker)
747        connectionBroker = m_weakConnectionBroker.lock();
748    std::pair<ClientConnection::ptr, bool> conn;
749    try {
750        conn = connectionBroker->getConnection(
751            connect ? originalUri : currentUri, forceNewConnection);
752    } catch (boost::exception &ex) {
753        currentUri = originalUri;
754        ex << errinfo_source(CONNECTION);
755        throw;
756    } catch (...) {
757        currentUri = originalUri;
758        throw;
759    }
760    // We use an absolute URI if we're talking to a proxy, or if the path
761    // starts with "//" (path_absolute does not allow an empty first segment;
762    // path_abempty as part of absolute_URI does); otherwise use only the path
763    // (and query)
764    if (!connect && !conn.second && !(originalUri.path.isAbsolute() &&
765        originalUri.path.segments.size() > 2 &&
766        originalUri.path.segments[1].empty())) {
767        currentUri.schemeDefined(false);
768        currentUri.authority.hostDefined(false);
769    }
770    try {
771        ClientRequest::ptr request;
772        try {
773            request = conn.first->request(requestHeaders);
774            if (!bodyDg)
775                request->doRequest();
776        } catch (boost::exception &ex) {
777            ex << errinfo_source(HTTP);
778            throw;
779        }
780        Future<> future;
781        boost::exception_ptr exception;
782        bool exceptionWasHttp = false;
783        if (bodyDg)
784            Scheduler::getThis()->schedule(boost::bind(&doBody,
785                request, bodyDg, boost::ref(future), boost::ref(exception),
786                boost::ref(exceptionWasHttp)));
787        currentUri = originalUri;
788        try {
789            // Force reading the response here to check for connectivity problems
790            request->response();
791        } catch (boost::exception &ex) {
792            ex << errinfo_source(HTTP);
793            if (bodyDg)
794                future.wait();
795            // Prefer to throw an exception from send, if there was one
796            if (exception)
797                Mordor::rethrow_exception(exception);
798            throw;
799        } catch (...) {
800            if (bodyDg)
801                future.wait();
802            // Prefer to throw an exception from send, if there was one
803            if (exception)
804                Mordor::rethrow_exception(exception);
805            throw;
806        }
807        if (bodyDg)
808            future.wait();
809        // Rethrow any exception from bodyDg *if* it didn't come from the
810        // HTTP framework, or directly from a stream within the framework
811        // (i.e. an exception from the user's code)
812        // otherwise, ignore it - we have a response, and we don't really
813        // care that the request didn't complete
814        if (exception && !exceptionWasHttp)
815            Mordor::rethrow_exception(exception);
816        return request;
817    } catch (...) {
818        currentUri = originalUri;
819        throw;
820    }
821}
822
823ClientRequest::ptr
824RetryRequestBroker::request(Request &requestHeaders, bool forceNewConnection,
825                           boost::function<void (ClientRequest::ptr)> bodyDg)
826{
827    size_t localRetries = 0;
828    size_t *retries = mp_retries ? mp_retries : &localRetries;
829    while (true) {
830        try {
831            ClientRequest::ptr request =
832                parent()->request(requestHeaders, forceNewConnection, bodyDg);
833            // Successful request resets shared retry counter
834            *retries = 0;
835            return request;
836        } catch (SocketException &ex) {
837            const ExceptionSource *source = boost::get_error_info<errinfo_source>(ex);
838            if (!source || (*source != HTTP && *source != CONNECTION))
839                throw;
840            if (m_delayDg && !m_delayDg(atomicIncrement(*retries)))
841                throw;
842            continue;
843        } catch (PriorRequestFailedException &ex) {
844            const ExceptionSource *source = boost::get_error_info<errinfo_source>(ex);
845            if (!source || *source != HTTP)
846                throw;
847            if (m_delayDg && !m_delayDg(*retries + 1))
848                throw;
849            continue;
850        } catch (PriorConnectionFailedException &ex) {
851            const ExceptionSource *source = boost::get_error_info<errinfo_source>(ex);
852            if (!source || (*source != HTTP && *source != CONNECTION))
853                throw;
854            if (m_delayDg && !m_delayDg(*retries + 1))
855                throw;
856            continue;
857        } catch (UnexpectedEofException &ex) {
858            const ExceptionSource *source = boost::get_error_info<errinfo_source>(ex);
859            if (!source || *source != HTTP)
860                throw;
861            if (m_delayDg && !m_delayDg(atomicIncrement(*retries)))
862                throw;
863            continue;
864        }
865        MORDOR_NOTREACHED();
866    }
867}
868
869ClientRequest::ptr
870RedirectRequestBroker::request(Request &requestHeaders, bool forceNewConnection,
871                               boost::function<void (ClientRequest::ptr)> bodyDg)
872{
873    URI &currentUri = requestHeaders.requestLine.uri;
874    URI originalUri = currentUri;
875    std::list<URI> uris;
876    uris.push_back(originalUri);
877    size_t redirects = 0;
878    while (true) {
879        try {
880            ClientRequest::ptr request = parent()->request(requestHeaders,
881                forceNewConnection, bodyDg);
882            bool handleRedirect = false;
883            switch (request->response().status.status)
884            {
885            case FOUND:
886                handleRedirect = m_handle302;
887                break;
888            case TEMPORARY_REDIRECT:
889                handleRedirect = m_handle307;
890                break;
891            case MOVED_PERMANENTLY:
892                handleRedirect = m_handle301;
893                break;
894            default:
895                currentUri = originalUri;
896                return request;
897            }
898            if (handleRedirect) {
899                if (++redirects == m_maxRedirects)
900                    MORDOR_THROW_EXCEPTION(CircularRedirectException(originalUri));
901                currentUri = URI::transform(currentUri,
902                    request->response().response.location);
903                if (std::find(uris.begin(), uris.end(), currentUri)
904                    != uris.end())
905                    MORDOR_THROW_EXCEPTION(CircularRedirectException(originalUri));
906                uris.push_back(currentUri);
907                if (request->response().status.status == MOVED_PERMANENTLY)
908                    originalUri = currentUri;
909                request->finish();
910                continue;
911            } else {
912                currentUri = originalUri;
913                return request;
914            }
915        } catch (...) {
916            currentUri = originalUri;
917            throw;
918        }
919        MORDOR_NOTREACHED();
920    }
921}
922
923ClientRequest::ptr
924UserAgentRequestBroker::request(Request &requestHeaders, bool forceNewConnection,
925                               boost::function<void (ClientRequest::ptr)> bodyDg)
926{
927    requestHeaders.request.userAgent = m_userAgent;
928    return parent()->request(requestHeaders, forceNewConnection, bodyDg);
929}
930
931}}