/mordor/pq/copy.cpp

http://github.com/mozy/mordor · C++ · 444 lines · 411 code · 32 blank · 1 comment · 54 complexity · c08f6524dcfc94ef3941bcb75f18c4dd MD5 · raw file

  1. // Copyright (c) 2010 - Mozy, Inc.
  2. #include "mordor/assert.h"
  3. #include "mordor/iomanager.h"
  4. #include "mordor/log.h"
  5. #include "mordor/streams/buffer.h"
  6. #include "mordor/streams/stream.h"
  7. #include "connection.h"
  8. #include "exception.h"
  9. namespace Mordor {
  10. namespace PQ {
  11. static Logger::ptr g_log = Log::lookup("mordor:pq");
  12. Connection::CopyParams::CopyParams(const std::string &table,
  13. boost::shared_ptr<PGconn> conn, SchedulerType *scheduler)
  14. : m_table(table),
  15. m_scheduler(scheduler),
  16. m_conn(conn),
  17. m_binary(false),
  18. m_csv(false),
  19. m_header(false),
  20. m_delimiter('\0'),
  21. m_quote('\0'),
  22. m_escape('\0')
  23. {}
  24. Connection::CopyParams &
  25. Connection::CopyParams::columns(const std::vector<std::string> &columns)
  26. {
  27. m_columns = columns;
  28. return *this;
  29. }
  30. Connection::CopyParams &
  31. Connection::CopyParams::binary()
  32. {
  33. MORDOR_ASSERT(!m_csv);
  34. m_binary = true;
  35. return *this;
  36. }
  37. Connection::CopyParams &
  38. Connection::CopyParams::csv()
  39. {
  40. MORDOR_ASSERT(!m_binary);
  41. m_csv = true;
  42. return *this;
  43. }
  44. Connection::CopyParams &
  45. Connection::CopyParams::delimiter(char delimiter)
  46. {
  47. MORDOR_ASSERT(!m_binary);
  48. m_delimiter = delimiter;
  49. return *this;
  50. }
  51. Connection::CopyParams &
  52. Connection::CopyParams::nullString(const std::string &nullString)
  53. {
  54. MORDOR_ASSERT(!m_binary);
  55. m_nullString = nullString;
  56. return *this;
  57. }
  58. Connection::CopyParams &
  59. Connection::CopyParams::header()
  60. {
  61. MORDOR_ASSERT(m_csv);
  62. m_header = true;
  63. return *this;
  64. }
  65. Connection::CopyParams &
  66. Connection::CopyParams::quote(char quote)
  67. {
  68. MORDOR_ASSERT(m_csv);
  69. m_quote = quote;
  70. return *this;
  71. }
  72. Connection::CopyParams &
  73. Connection::CopyParams::escape(char escape)
  74. {
  75. MORDOR_ASSERT(m_csv);
  76. m_escape = escape;
  77. return *this;
  78. }
  79. Connection::CopyParams &
  80. Connection::CopyParams::notNullQuoteColumns(const std::vector<std::string> &columns)
  81. {
  82. MORDOR_ASSERT(m_csv);
  83. m_notNullQuoteColumns = columns;
  84. return *this;
  85. }
  86. class CopyInStream : public Stream
  87. {
  88. public:
  89. CopyInStream(boost::shared_ptr<PGconn> conn, SchedulerType *scheduler)
  90. : m_conn(conn),
  91. m_scheduler(scheduler)
  92. {}
  93. ~CopyInStream()
  94. {
  95. boost::shared_ptr<PGconn> sharedConn = m_conn.lock();
  96. if (sharedConn) {
  97. PGconn *conn = sharedConn.get();
  98. try {
  99. putCopyEnd(conn, "COPY IN aborted");
  100. } catch (...) {
  101. }
  102. }
  103. }
  104. bool supportsWrite() { return true; }
  105. void close(CloseType type)
  106. {
  107. MORDOR_ASSERT(type & WRITE);
  108. boost::shared_ptr<PGconn> sharedConn = m_conn.lock();
  109. if (sharedConn) {
  110. PGconn *conn = sharedConn.get();
  111. putCopyEnd(conn, NULL);
  112. m_conn.reset();
  113. }
  114. }
  115. size_t write(const void *buffer, size_t length)
  116. {
  117. boost::shared_ptr<PGconn> sharedConn = m_conn.lock();
  118. MORDOR_ASSERT(sharedConn);
  119. PGconn *conn = sharedConn.get();
  120. int status = 0;
  121. length = std::min<size_t>(length, 0x7fffffff);
  122. #ifdef WINDOWS
  123. SchedulerSwitcher switcher(m_scheduler);
  124. #endif
  125. while (status == 0) {
  126. status = PQputCopyData(conn, (const char *)buffer, (int)length);
  127. switch (status) {
  128. case 1:
  129. return length;
  130. case -1:
  131. throwException(conn);
  132. #ifndef WINDOWS
  133. case 0:
  134. MORDOR_ASSERT(m_scheduler);
  135. m_scheduler->registerEvent(PQsocket(conn),
  136. SchedulerType::WRITE);
  137. Scheduler::yieldTo();
  138. break;
  139. #endif
  140. default:
  141. MORDOR_NOTREACHED();
  142. }
  143. }
  144. MORDOR_NOTREACHED();
  145. }
  146. private:
  147. void putCopyEnd(PGconn *conn, const char *error) {
  148. #ifdef WINDOWS
  149. SchedulerSwitcher switcher(m_scheduler);
  150. #endif
  151. int status = 0;
  152. while (status == 0) {
  153. status = PQputCopyEnd(conn, error);
  154. switch (status) {
  155. case 1:
  156. break;
  157. case -1:
  158. throwException(conn);
  159. #ifndef WINDOWS
  160. case 0:
  161. MORDOR_ASSERT(m_scheduler);
  162. m_scheduler->registerEvent(PQsocket(conn),
  163. SchedulerType::WRITE);
  164. Scheduler::yieldTo();
  165. break;
  166. #endif
  167. default:
  168. MORDOR_NOTREACHED();
  169. }
  170. }
  171. #ifndef WINDOWS
  172. if (m_scheduler)
  173. PQ::flush(conn, m_scheduler);
  174. #endif
  175. boost::shared_ptr<PGresult> result;
  176. #ifndef WINDOWS
  177. if (m_scheduler)
  178. result.reset(nextResult(conn, m_scheduler), &PQclear);
  179. else
  180. #endif
  181. result.reset(PQgetResult(conn), &PQclear);
  182. while (result) {
  183. ExecStatusType status = PQresultStatus(result.get());
  184. MORDOR_LOG_DEBUG(g_log) << conn << " PQresultStatus("
  185. << result.get() << "): " << PQresStatus(status);
  186. if (status != PGRES_COMMAND_OK)
  187. throwException(result.get());
  188. #ifndef WINDOWS
  189. if (m_scheduler)
  190. result.reset(nextResult(conn, m_scheduler), &PQclear);
  191. else
  192. #endif
  193. result.reset(PQgetResult(conn), &PQclear);
  194. }
  195. MORDOR_LOG_VERBOSE(g_log) << conn << " PQputCopyEnd(\""
  196. << (error ? error : "") << "\")";
  197. }
  198. private:
  199. boost::weak_ptr<PGconn> m_conn;
  200. SchedulerType *m_scheduler;
  201. };
  202. class CopyOutStream : public Stream
  203. {
  204. public:
  205. CopyOutStream(boost::shared_ptr<PGconn> conn, SchedulerType *scheduler)
  206. : m_conn(conn),
  207. m_scheduler(scheduler)
  208. {}
  209. bool supportsRead() { return true; }
  210. size_t read(Buffer &buffer, size_t length)
  211. {
  212. if (m_readBuffer.readAvailable()) {
  213. length = (std::min)(m_readBuffer.readAvailable(), length);
  214. buffer.copyIn(m_readBuffer, length);
  215. m_readBuffer.consume(length);
  216. return length;
  217. }
  218. boost::shared_ptr<PGconn> sharedConn = m_conn.lock();
  219. if (!sharedConn)
  220. return 0;
  221. PGconn *conn = sharedConn.get();
  222. #ifndef WINDOWS
  223. SchedulerSwitcher switcher(m_scheduler);
  224. #endif
  225. int status = 0;
  226. do {
  227. char *data = NULL;
  228. status = PQgetCopyData(conn, &data,
  229. #ifdef WINDOWS
  230. 0
  231. #else
  232. m_scheduler ? 1 : 0
  233. #endif
  234. );
  235. switch (status) {
  236. case 0:
  237. #ifdef WINDOWS
  238. MORDOR_NOTREACHED();
  239. #else
  240. MORDOR_ASSERT(m_scheduler);
  241. m_scheduler->registerEvent(PQsocket(conn),
  242. SchedulerType::READ);
  243. Scheduler::yieldTo();
  244. continue;
  245. #endif
  246. case -1:
  247. break;
  248. case -2:
  249. throwException(conn);
  250. default:
  251. MORDOR_ASSERT(status > 0);
  252. try {
  253. m_readBuffer.copyIn(data, status);
  254. } catch (...) {
  255. PQfreemem(data);
  256. throw;
  257. }
  258. PQfreemem(data);
  259. break;
  260. }
  261. } while (false);
  262. if (status == -1) {
  263. m_conn.reset();
  264. boost::shared_ptr<PGresult> result;
  265. #ifndef WINDOWS
  266. if (m_scheduler)
  267. result.reset(nextResult(conn, m_scheduler), &PQclear);
  268. else
  269. #endif
  270. result.reset(PQgetResult(conn), &PQclear);
  271. while (result) {
  272. ExecStatusType status = PQresultStatus(result.get());
  273. MORDOR_LOG_DEBUG(g_log) << conn << " PQresultStatus("
  274. << result.get() << "): " << PQresStatus(status);
  275. if (status != PGRES_COMMAND_OK)
  276. throwException(result.get());
  277. #ifndef WINDOWS
  278. if (m_scheduler)
  279. result.reset(nextResult(conn, m_scheduler), &PQclear);
  280. else
  281. #endif
  282. result.reset(PQgetResult(conn), &PQclear);
  283. }
  284. }
  285. length = (std::min)(m_readBuffer.readAvailable(), length);
  286. buffer.copyIn(m_readBuffer, length);
  287. m_readBuffer.consume(length);
  288. return length;
  289. }
  290. private:
  291. boost::weak_ptr<PGconn> m_conn;
  292. SchedulerType *m_scheduler;
  293. Buffer m_readBuffer;
  294. };
  295. Stream::ptr
  296. Connection::CopyParams::execute(bool out)
  297. {
  298. PGconn *conn = m_conn.get();
  299. std::ostringstream os;
  300. os << "COPY " << m_table << " ";
  301. if (!m_columns.empty()) {
  302. os << "(";
  303. for (std::vector<std::string>::const_iterator it(m_columns.begin());
  304. it != m_columns.end();
  305. ++it) {
  306. if (it != m_columns.begin())
  307. os << ", ";
  308. os << *it;
  309. }
  310. os << ") ";
  311. }
  312. os << (out ? "TO STDOUT" : "FROM STDIN");
  313. if (m_binary) {
  314. os << " BINARY";
  315. } else {
  316. if (m_delimiter != '\0')
  317. os << " DELIMITER '"
  318. << PQ::escape(conn, std::string(1, m_delimiter)) << '\'';
  319. if (!m_nullString.empty())
  320. os << " NULL '" << PQ::escape(conn, m_nullString) << '\'';
  321. if (m_csv) {
  322. os << " CSV";
  323. if (m_header)
  324. os << " HEADER";
  325. if (m_quote != '\0')
  326. os << " QUOTE '"
  327. << PQ::escape(conn, std::string(1, m_quote)) << '\'';
  328. if (m_escape != '\0')
  329. os << " ESCAPE '"
  330. << PQ::escape(conn, std::string(1, m_escape)) << '\'';
  331. if (!m_notNullQuoteColumns.empty()) {
  332. os << (out ? " FORCE QUOTE" : " FORCE NOT NULL ");
  333. for (std::vector<std::string>::const_iterator it(m_notNullQuoteColumns.begin());
  334. it != m_notNullQuoteColumns.end();
  335. ++it) {
  336. if (it != m_notNullQuoteColumns.begin())
  337. os << ", ";
  338. os << *it;
  339. }
  340. }
  341. }
  342. }
  343. boost::shared_ptr<PGresult> result, next;
  344. const char *api = NULL;
  345. #ifdef WINDOWS
  346. SchedulerSwitcher switcher(m_scheduler);
  347. #else
  348. if (m_scheduler) {
  349. api = "PQsendQuery";
  350. if (!PQsendQuery(conn, os.str().c_str()))
  351. throwException(conn);
  352. flush(conn, m_scheduler);
  353. next.reset(nextResult(conn, m_scheduler), &PQclear);
  354. while (next) {
  355. result = next;
  356. if (PQresultStatus(result.get()) ==
  357. (out ? PGRES_COPY_OUT : PGRES_COPY_IN))
  358. break;
  359. next.reset(nextResult(conn, m_scheduler), &PQclear);
  360. if (next) {
  361. ExecStatusType status = PQresultStatus(next.get());
  362. MORDOR_LOG_VERBOSE(g_log) << conn << "PQresultStatus(" <<
  363. next.get() << "): " << PQresStatus(status);
  364. switch (status) {
  365. case PGRES_COMMAND_OK:
  366. case PGRES_TUPLES_OK:
  367. break;
  368. default:
  369. throwException(next.get());
  370. MORDOR_NOTREACHED();
  371. }
  372. }
  373. }
  374. } else
  375. #endif
  376. {
  377. api = "PQexec";
  378. result.reset(PQexec(conn, os.str().c_str()), &PQclear);
  379. }
  380. if (!result)
  381. throwException(conn);
  382. ExecStatusType status = PQresultStatus(result.get());
  383. MORDOR_ASSERT(api);
  384. MORDOR_LOG_VERBOSE(g_log) << conn << " " << api << "(\"" << os.str()
  385. << "\"), PQresultStatus(" << result.get() << "): "
  386. << PQresStatus(status);
  387. switch (status) {
  388. case PGRES_COMMAND_OK:
  389. case PGRES_TUPLES_OK:
  390. MORDOR_NOTREACHED();
  391. case PGRES_COPY_IN:
  392. MORDOR_ASSERT(!out);
  393. return Stream::ptr(new CopyInStream(m_conn, m_scheduler));
  394. case PGRES_COPY_OUT:
  395. MORDOR_ASSERT(out);
  396. return Stream::ptr(new CopyOutStream(m_conn, m_scheduler));
  397. default:
  398. throwException(result.get());
  399. MORDOR_NOTREACHED();
  400. }
  401. }
  402. Stream::ptr
  403. Connection::CopyInParams::operator()()
  404. {
  405. return execute(false);
  406. }
  407. Stream::ptr
  408. Connection::CopyOutParams::operator()()
  409. {
  410. return execute(true);
  411. }
  412. }}