/examples/treasury_yield/src/test/java/com/mongodb/hadoop/TestStandalone.java

http://github.com/mongodb/mongo-hadoop · Java · 120 lines · 103 code · 17 blank · 0 comment · 3 complexity · 958d7f39d9bcc39b53451c156fcafe27 MD5 · raw file

  1. package com.mongodb.hadoop;
  2. import com.mongodb.MongoClientURI;
  3. import com.mongodb.hadoop.examples.treasury.TreasuryYieldXMLConfig;
  4. import com.mongodb.hadoop.mapred.output.MongoOutputCommitter;
  5. import com.mongodb.hadoop.splitter.MultiMongoCollectionSplitter;
  6. import com.mongodb.hadoop.splitter.SingleMongoSplitter;
  7. import com.mongodb.hadoop.testutils.MapReduceJob;
  8. import com.mongodb.hadoop.util.MongoClientURIBuilder;
  9. import org.apache.commons.logging.Log;
  10. import org.apache.commons.logging.LogFactory;
  11. import org.codehaus.jackson.JsonNode;
  12. import org.codehaus.jackson.node.ArrayNode;
  13. import org.codehaus.jackson.node.JsonNodeFactory;
  14. import org.codehaus.jackson.node.ObjectNode;
  15. import org.junit.Before;
  16. import org.junit.Test;
  17. import static com.mongodb.hadoop.splitter.MultiMongoCollectionSplitter.MULTI_COLLECTION_CONF_KEY;
  18. import static com.mongodb.hadoop.util.MongoConfigUtil.INPUT_NOTIMEOUT;
  19. import static com.mongodb.hadoop.util.MongoConfigUtil.INPUT_URI;
  20. import static com.mongodb.hadoop.util.MongoConfigUtil.MONGO_SPLITTER_CLASS;
  21. import static com.mongodb.hadoop.util.MongoConfigUtil.SPLITS_USE_RANGEQUERY;
  22. import static org.junit.Assume.assumeFalse;
  23. public class TestStandalone extends TreasuryTest {
  24. private static final Log LOG = LogFactory.getLog(TestStandalone.class);
  25. private final MongoClientURI inputUri2;
  26. public TestStandalone() {
  27. inputUri2 = authCheck(new MongoClientURIBuilder()
  28. .collection("mongo_hadoop", "yield_historical.in2"))
  29. .build();
  30. }
  31. @Before
  32. public void checkConfiguration() {
  33. assumeFalse(isSharded(getInputUri()));
  34. }
  35. @Test
  36. public void testBasicInputSource() {
  37. LOG.info("testing basic input source");
  38. MapReduceJob treasuryJob =
  39. new MapReduceJob(TreasuryYieldXMLConfig.class.getName())
  40. .jar(JOBJAR_PATH)
  41. .param("mongo.input.notimeout", "true")
  42. .inputUris(getInputUri())
  43. .outputUri(getOutputUri());
  44. if (isHadoopV1()) {
  45. treasuryJob.outputCommitter(MongoOutputCommitter.class);
  46. }
  47. treasuryJob.execute(isRunTestInVm());
  48. compareResults(getClient(getInputUri()).getDB(getOutputUri().getDatabase()).getCollection(getOutputUri().getCollection()),
  49. getReference());
  50. }
  51. @Test
  52. public void testTreasuryJsonConfig() {
  53. mongoImport("yield_historical.in3", TREASURY_JSON_PATH);
  54. MapReduceJob treasuryJob =
  55. new MapReduceJob("com.mongodb.hadoop.examples.treasury.TreasuryYieldXMLConfig")
  56. .jar(JOBJAR_PATH)
  57. .param(MONGO_SPLITTER_CLASS, MultiMongoCollectionSplitter.class.getName())
  58. .param(MULTI_COLLECTION_CONF_KEY, collectionSettings().toString())
  59. .outputUri(getOutputUri());
  60. if (isHadoopV1()) {
  61. treasuryJob.outputCommitter(MongoOutputCommitter.class);
  62. }
  63. treasuryJob.execute(isRunTestInVm());
  64. compareDoubled(getClient(getInputUri()).getDB(getOutputUri().getDatabase()).getCollection(getOutputUri().getCollection()));
  65. }
  66. @Test
  67. public void testMultipleCollectionSupport() {
  68. mongoImport(getInputUri().getCollection(), TREASURY_JSON_PATH);
  69. mongoImport(inputUri2.getCollection(), TREASURY_JSON_PATH);
  70. MapReduceJob treasuryJob =
  71. new MapReduceJob("com.mongodb.hadoop.examples.treasury.TreasuryYieldXMLConfig")
  72. .jar(JOBJAR_PATH)
  73. .param(MONGO_SPLITTER_CLASS, MultiMongoCollectionSplitter.class.getName())
  74. .inputUris(getInputUri(), inputUri2)
  75. .outputUri(getOutputUri());
  76. if (isHadoopV1()) {
  77. treasuryJob.outputCommitter(MongoOutputCommitter.class);
  78. }
  79. treasuryJob.execute(isRunTestInVm());
  80. compareDoubled(getClient(getInputUri()).getDB(getOutputUri().getDatabase()).getCollection(getOutputUri().getCollection()));
  81. }
  82. private JsonNode collectionSettings() {
  83. ArrayNode settings = new ArrayNode(JsonNodeFactory.instance);
  84. ObjectNode node = new ObjectNode(JsonNodeFactory.instance);
  85. node.put(INPUT_URI, getInputUri().toString());
  86. ObjectNode dow = new ObjectNode(JsonNodeFactory.instance);
  87. dow.put("dayOfWeek", "FRIDAY");
  88. node.put("query", dow);
  89. node.put(MONGO_SPLITTER_CLASS, SingleMongoSplitter.class.getName());
  90. node.put(SPLITS_USE_RANGEQUERY, true);
  91. node.put(INPUT_NOTIMEOUT, true);
  92. settings.add(node);
  93. MongoClientURI inputUri3 = authCheck(new MongoClientURIBuilder()
  94. .collection("mongo_hadoop", "yield_historical.in3"))
  95. .build();
  96. node = new ObjectNode(JsonNodeFactory.instance);
  97. node.put(INPUT_URI, inputUri3.toString());
  98. node.put(SPLITS_USE_RANGEQUERY, true);
  99. node.put(INPUT_NOTIMEOUT, true);
  100. settings.add(node);
  101. return settings;
  102. }
  103. }