PageRenderTime 83ms CodeModel.GetById 41ms app.highlight 16ms RepoModel.GetById 24ms app.codeStats 0ms

/mordor/streams/transfer.cpp

http://github.com/mozy/mordor
C++ | 123 lines | 109 code | 12 blank | 2 comment | 39 complexity | 6615883bf1ba497fce2a6f08214f2405 MD5 | raw file
  1// Copyright (c) 2009 - Mozy, Inc.
  2
  3#include "transfer.h"
  4
  5#include <boost/bind.hpp>
  6
  7#include "mordor/assert.h"
  8#include "mordor/config.h"
  9#include "mordor/fiber.h"
 10#include "mordor/parallel.h"
 11#include "mordor/streams/buffer.h"
 12#include "mordor/streams/null.h"
 13#include "stream.h"
 14
 15namespace Mordor {
 16
 17static ConfigVar<size_t>::ptr g_chunkSize =
 18    Config::lookup("transferstream.chunksize",
 19                   (size_t)65536,
 20                   "transfer chunk size.");
 21static Logger::ptr g_log = Log::lookup("mordor:stream:transfer");
 22
 23static void readOne(Stream &src, Buffer *&buffer, size_t len, size_t &result)
 24{
 25    result = src.read(*buffer, len);
 26    MORDOR_LOG_TRACE(g_log) << "read " << result << " bytes from " << &src;
 27}
 28
 29static void writeOne(Stream &dst, Buffer *&buffer)
 30{
 31    size_t result;
 32    while (buffer->readAvailable() > 0) {
 33        result = dst.write(*buffer, buffer->readAvailable());
 34        MORDOR_LOG_TRACE(g_log) << "wrote " << result << " bytes to " << &dst;
 35        buffer->consume(result);
 36    }
 37}
 38
 39unsigned long long transferStream(Stream &src, Stream &dst,
 40                                  unsigned long long toTransfer,
 41                                  ExactLength exactLength)
 42{
 43    MORDOR_LOG_DEBUG(g_log) << "transferring " << toTransfer << " bytes from "
 44        << &src << " to " << &dst;
 45    MORDOR_ASSERT(src.supportsRead());
 46    MORDOR_ASSERT(dst.supportsWrite());
 47    Buffer buf1, buf2;
 48    Buffer *readBuffer, *writeBuffer;
 49    size_t chunkSize = g_chunkSize->val();
 50    size_t todo;
 51    size_t readResult;
 52    unsigned long long totalRead = 0;
 53    if (toTransfer == 0)
 54        return 0;
 55    if (exactLength == INFER)
 56        exactLength = (toTransfer == ~0ull ? UNTILEOF : EXACT);
 57    MORDOR_ASSERT(exactLength == EXACT || exactLength == UNTILEOF);
 58
 59    readBuffer = &buf1;
 60    todo = chunkSize;
 61    if (toTransfer - totalRead < (unsigned long long)todo)
 62        todo = (size_t)(toTransfer - totalRead);
 63    readOne(src, readBuffer, todo, readResult);
 64    totalRead += readResult;
 65    if (readResult == 0 && exactLength == EXACT)
 66        MORDOR_THROW_EXCEPTION(UnexpectedEofException());
 67    if (readResult == 0)
 68        return totalRead;
 69
 70    // Optimize transfer to NullStream
 71    if (&dst == &NullStream::get()) {
 72        while (true) {
 73            readBuffer->clear();
 74            todo = chunkSize;
 75            if (toTransfer - totalRead < (unsigned long long)todo)
 76                todo = (size_t)(toTransfer - totalRead);
 77            if (todo == 0)
 78                return totalRead;
 79            readOne(src, readBuffer, todo, readResult);
 80            totalRead += readResult;
 81            if (readResult == 0 && exactLength == EXACT)
 82                MORDOR_THROW_EXCEPTION(UnexpectedEofException());
 83            if (readResult == 0)
 84                return totalRead;
 85        }
 86    }
 87
 88    std::vector<boost::function<void ()> > dgs;
 89    std::vector<Fiber::ptr> fibers;
 90    dgs.resize(2);
 91    fibers.resize(2);
 92    fibers[0].reset(new Fiber(NULL));
 93    fibers[1].reset(new Fiber(NULL));
 94    dgs[0] = boost::bind(&readOne, boost::ref(src), boost::ref(readBuffer),
 95        boost::ref(todo), boost::ref(readResult));
 96    dgs[1] = boost::bind(&writeOne, boost::ref(dst), boost::ref(writeBuffer));
 97    while (totalRead < toTransfer) {
 98        writeBuffer = readBuffer;
 99        if (readBuffer == &buf1)
100            readBuffer = &buf2;
101        else
102            readBuffer = &buf1;
103        todo = chunkSize;
104        if (toTransfer - totalRead < (unsigned long long)todo)
105            todo = (size_t)(toTransfer - totalRead);
106        parallel_do(dgs, fibers);
107        totalRead += readResult;
108        if (readResult == 0 && exactLength == EXACT && totalRead < toTransfer) {
109            MORDOR_LOG_ERROR(g_log) << "only read " << totalRead << "/"
110                << toTransfer << " from " << &src;
111            MORDOR_THROW_EXCEPTION(UnexpectedEofException());
112        }
113        if (readResult == 0)
114            return totalRead;
115    }
116    writeBuffer = readBuffer;
117    writeOne(dst, writeBuffer);
118    MORDOR_LOG_VERBOSE(g_log) << "transferred " << totalRead << "/" << toTransfer
119        << " from " << &src << " to " << &dst;
120    return totalRead;
121}
122
123}