PageRenderTime 41ms CodeModel.GetById 14ms RepoModel.GetById 0ms app.codeStats 0ms

/mordor/streams/transfer.cpp

http://github.com/mozy/mordor
C++ | 123 lines | 109 code | 12 blank | 2 comment | 39 complexity | 6615883bf1ba497fce2a6f08214f2405 MD5 | raw file
Possible License(s): BSD-3-Clause
  1. // Copyright (c) 2009 - Mozy, Inc.
  2. #include "transfer.h"
  3. #include <boost/bind.hpp>
  4. #include "mordor/assert.h"
  5. #include "mordor/config.h"
  6. #include "mordor/fiber.h"
  7. #include "mordor/parallel.h"
  8. #include "mordor/streams/buffer.h"
  9. #include "mordor/streams/null.h"
  10. #include "stream.h"
  11. namespace Mordor {
  12. static ConfigVar<size_t>::ptr g_chunkSize =
  13. Config::lookup("transferstream.chunksize",
  14. (size_t)65536,
  15. "transfer chunk size.");
  16. static Logger::ptr g_log = Log::lookup("mordor:stream:transfer");
  17. static void readOne(Stream &src, Buffer *&buffer, size_t len, size_t &result)
  18. {
  19. result = src.read(*buffer, len);
  20. MORDOR_LOG_TRACE(g_log) << "read " << result << " bytes from " << &src;
  21. }
  22. static void writeOne(Stream &dst, Buffer *&buffer)
  23. {
  24. size_t result;
  25. while (buffer->readAvailable() > 0) {
  26. result = dst.write(*buffer, buffer->readAvailable());
  27. MORDOR_LOG_TRACE(g_log) << "wrote " << result << " bytes to " << &dst;
  28. buffer->consume(result);
  29. }
  30. }
  31. unsigned long long transferStream(Stream &src, Stream &dst,
  32. unsigned long long toTransfer,
  33. ExactLength exactLength)
  34. {
  35. MORDOR_LOG_DEBUG(g_log) << "transferring " << toTransfer << " bytes from "
  36. << &src << " to " << &dst;
  37. MORDOR_ASSERT(src.supportsRead());
  38. MORDOR_ASSERT(dst.supportsWrite());
  39. Buffer buf1, buf2;
  40. Buffer *readBuffer, *writeBuffer;
  41. size_t chunkSize = g_chunkSize->val();
  42. size_t todo;
  43. size_t readResult;
  44. unsigned long long totalRead = 0;
  45. if (toTransfer == 0)
  46. return 0;
  47. if (exactLength == INFER)
  48. exactLength = (toTransfer == ~0ull ? UNTILEOF : EXACT);
  49. MORDOR_ASSERT(exactLength == EXACT || exactLength == UNTILEOF);
  50. readBuffer = &buf1;
  51. todo = chunkSize;
  52. if (toTransfer - totalRead < (unsigned long long)todo)
  53. todo = (size_t)(toTransfer - totalRead);
  54. readOne(src, readBuffer, todo, readResult);
  55. totalRead += readResult;
  56. if (readResult == 0 && exactLength == EXACT)
  57. MORDOR_THROW_EXCEPTION(UnexpectedEofException());
  58. if (readResult == 0)
  59. return totalRead;
  60. // Optimize transfer to NullStream
  61. if (&dst == &NullStream::get()) {
  62. while (true) {
  63. readBuffer->clear();
  64. todo = chunkSize;
  65. if (toTransfer - totalRead < (unsigned long long)todo)
  66. todo = (size_t)(toTransfer - totalRead);
  67. if (todo == 0)
  68. return totalRead;
  69. readOne(src, readBuffer, todo, readResult);
  70. totalRead += readResult;
  71. if (readResult == 0 && exactLength == EXACT)
  72. MORDOR_THROW_EXCEPTION(UnexpectedEofException());
  73. if (readResult == 0)
  74. return totalRead;
  75. }
  76. }
  77. std::vector<boost::function<void ()> > dgs;
  78. std::vector<Fiber::ptr> fibers;
  79. dgs.resize(2);
  80. fibers.resize(2);
  81. fibers[0].reset(new Fiber(NULL));
  82. fibers[1].reset(new Fiber(NULL));
  83. dgs[0] = boost::bind(&readOne, boost::ref(src), boost::ref(readBuffer),
  84. boost::ref(todo), boost::ref(readResult));
  85. dgs[1] = boost::bind(&writeOne, boost::ref(dst), boost::ref(writeBuffer));
  86. while (totalRead < toTransfer) {
  87. writeBuffer = readBuffer;
  88. if (readBuffer == &buf1)
  89. readBuffer = &buf2;
  90. else
  91. readBuffer = &buf1;
  92. todo = chunkSize;
  93. if (toTransfer - totalRead < (unsigned long long)todo)
  94. todo = (size_t)(toTransfer - totalRead);
  95. parallel_do(dgs, fibers);
  96. totalRead += readResult;
  97. if (readResult == 0 && exactLength == EXACT && totalRead < toTransfer) {
  98. MORDOR_LOG_ERROR(g_log) << "only read " << totalRead << "/"
  99. << toTransfer << " from " << &src;
  100. MORDOR_THROW_EXCEPTION(UnexpectedEofException());
  101. }
  102. if (readResult == 0)
  103. return totalRead;
  104. }
  105. writeBuffer = readBuffer;
  106. writeOne(dst, writeBuffer);
  107. MORDOR_LOG_VERBOSE(g_log) << "transferred " << totalRead << "/" << toTransfer
  108. << " from " << &src << " to " << &dst;
  109. return totalRead;
  110. }
  111. }