/dbcrossbar/tests/cli/cp/combined.rs

https://github.com/dbcrossbar/dbcrossbar · Rust · 160 lines · 130 code · 14 blank · 16 comment · 0 complexity · 9dc90fd24de7b6077dd32b28617240df MD5 · raw file

  1. //! Tests that affect multiple backends.
  2. use cli_test_dir::*;
  3. use difference::assert_diff;
  4. use std::{fs, process::Stdio};
  5. use super::*;
  6. #[test]
  7. #[ignore]
  8. fn cp_csv_to_postgres_to_gs_to_csv() {
  9. let _ = env_logger::try_init();
  10. let testdir = TestDir::new("dbcrossbar", "cp_csv_to_postgres_to_gs_to_csv");
  11. let src = testdir.src_path("fixtures/many_types.csv");
  12. let schema = testdir.src_path("fixtures/many_types.sql");
  13. let expected_schema = testdir.src_path("fixtures/many_types_expected.sql");
  14. let pg_table = post_test_table_url("testme1.cp_csv_to_postgres_to_gs_to_csv");
  15. let gs_dir = gs_test_dir_url("cp_csv_to_postgres_to_gs_to_csv");
  16. let bq_table = bq_test_table("cp_csv_to_postgres_to_gs_to_csv");
  17. let gs_dir_2 = gs_test_dir_url("cp_csv_to_postgres_to_gs_to_csv_2");
  18. let pg_table_2 = post_test_table_url("cp_csv_to_postgres_to_gs_to_csv_2");
  19. // CSV to Postgres.
  20. testdir
  21. .cmd()
  22. .args(&[
  23. "cp",
  24. "--if-exists=overwrite",
  25. "--max-streams=8",
  26. &format!("--schema=postgres-sql:{}", schema.display()),
  27. &format!("csv:{}", src.display()),
  28. &pg_table,
  29. ])
  30. .tee_output()
  31. .expect_success();
  32. // (Check PostgreSQL schema extraction now, so we know that we aren't
  33. // messing up later tests.)
  34. testdir
  35. .cmd()
  36. .args(&["schema", "conv", &pg_table, "postgres-sql:pg.sql"])
  37. .stdout(Stdio::piped())
  38. .tee_output()
  39. .expect_success();
  40. let postgres_sql = fs::read_to_string(&expected_schema).unwrap().replace(
  41. "\"many_types\"",
  42. "\"testme1\".\"cp_csv_to_postgres_to_gs_to_csv\"",
  43. );
  44. testdir.expect_file_contents("pg.sql", &postgres_sql);
  45. // Postgres to gs://.
  46. testdir
  47. .cmd()
  48. .args(&["cp", "--if-exists=overwrite", &pg_table, &gs_dir])
  49. .tee_output()
  50. .expect_success();
  51. // gs:// to BigQuery.
  52. testdir
  53. .cmd()
  54. .args(&[
  55. "cp",
  56. "--if-exists=overwrite",
  57. &format!("--schema=postgres-sql:{}", schema.display()),
  58. &gs_dir,
  59. &bq_table,
  60. ])
  61. .tee_output()
  62. .expect_success();
  63. // BigQuery to gs://.
  64. testdir
  65. .cmd()
  66. .args(&[
  67. "cp",
  68. "--if-exists=overwrite",
  69. &format!("--schema=postgres-sql:{}", schema.display()),
  70. &bq_table,
  71. &gs_dir_2,
  72. ])
  73. .tee_output()
  74. .expect_success();
  75. // gs:// back to PostgreSQL. (Mostly because we'll need a PostgreSQL-generated
  76. // CSV file for the final comparison below.)
  77. testdir
  78. .cmd()
  79. .args(&[
  80. "cp",
  81. "--if-exists=overwrite",
  82. &format!("--schema=postgres-sql:{}", schema.display()),
  83. &gs_dir_2,
  84. &pg_table_2,
  85. ])
  86. .tee_output()
  87. .expect_success();
  88. // PostgreSQL back to CSV for the final comparison below.
  89. testdir
  90. .cmd()
  91. .args(&[
  92. "cp",
  93. &format!("--schema=postgres-sql:{}", schema.display()),
  94. &pg_table_2,
  95. "csv:out/",
  96. ])
  97. .tee_output()
  98. .expect_success();
  99. let expected = fs::read_to_string(&src).unwrap();
  100. let actual =
  101. fs::read_to_string(testdir.path("out/cp_csv_to_postgres_to_gs_to_csv_2.csv"))
  102. .unwrap();
  103. assert_diff!(&expected, &actual, ",", 0);
  104. }
  105. #[test]
  106. #[ignore]
  107. fn cp_tricky_column_names_fails() {
  108. let _ = env_logger::try_init();
  109. let testdir = TestDir::new("dbcrossbar", "cp_tricky_column_names");
  110. let src = testdir.src_path("fixtures/tricky_column_names.csv");
  111. let schema = testdir.src_path("fixtures/tricky_column_names.sql");
  112. let pg_table = post_test_table_url("testme1.cp_tricky_column_names");
  113. let bq_table = bq_test_table("cp_tricky_column_names");
  114. let gs_temp_dir = gs_test_dir_url("cp_from_bigquery_with_where");
  115. let bq_temp_ds = bq_temp_dataset();
  116. // CSV to Postgres.
  117. testdir
  118. .cmd()
  119. .args(&[
  120. "cp",
  121. "--if-exists=overwrite",
  122. &format!("--schema=postgres-sql:{}", schema.display()),
  123. &format!("csv:{}", src.display()),
  124. &pg_table,
  125. ])
  126. .tee_output()
  127. .expect_success();
  128. // Postgres to BigQuery.
  129. //
  130. // This is now expected to fail, because editing column names automagically
  131. // ends in tears and policy is now firmly against it. Instead, copy to a
  132. // local machine and run `scrubcsv --clean-column-names`.
  133. testdir
  134. .cmd()
  135. .args(&[
  136. "cp",
  137. "--if-exists=overwrite",
  138. &format!("--schema=postgres-sql:{}", schema.display()),
  139. &format!("--temporary={}", gs_temp_dir),
  140. &format!("--temporary={}", bq_temp_ds),
  141. &pg_table,
  142. &bq_table,
  143. ])
  144. .tee_output()
  145. .expect_failure();
  146. }