/examples/treasury_yield/src/test/java/com/mongodb/hadoop/TestStandalone.java
http://github.com/mongodb/mongo-hadoop · Java · 120 lines · 103 code · 17 blank · 0 comment · 3 complexity · 958d7f39d9bcc39b53451c156fcafe27 MD5 · raw file
- package com.mongodb.hadoop;
- import com.mongodb.MongoClientURI;
- import com.mongodb.hadoop.examples.treasury.TreasuryYieldXMLConfig;
- import com.mongodb.hadoop.mapred.output.MongoOutputCommitter;
- import com.mongodb.hadoop.splitter.MultiMongoCollectionSplitter;
- import com.mongodb.hadoop.splitter.SingleMongoSplitter;
- import com.mongodb.hadoop.testutils.MapReduceJob;
- import com.mongodb.hadoop.util.MongoClientURIBuilder;
- import org.apache.commons.logging.Log;
- import org.apache.commons.logging.LogFactory;
- import org.codehaus.jackson.JsonNode;
- import org.codehaus.jackson.node.ArrayNode;
- import org.codehaus.jackson.node.JsonNodeFactory;
- import org.codehaus.jackson.node.ObjectNode;
- import org.junit.Before;
- import org.junit.Test;
- import static com.mongodb.hadoop.splitter.MultiMongoCollectionSplitter.MULTI_COLLECTION_CONF_KEY;
- import static com.mongodb.hadoop.util.MongoConfigUtil.INPUT_NOTIMEOUT;
- import static com.mongodb.hadoop.util.MongoConfigUtil.INPUT_URI;
- import static com.mongodb.hadoop.util.MongoConfigUtil.MONGO_SPLITTER_CLASS;
- import static com.mongodb.hadoop.util.MongoConfigUtil.SPLITS_USE_RANGEQUERY;
- import static org.junit.Assume.assumeFalse;
- public class TestStandalone extends TreasuryTest {
- private static final Log LOG = LogFactory.getLog(TestStandalone.class);
- private final MongoClientURI inputUri2;
- public TestStandalone() {
- inputUri2 = authCheck(new MongoClientURIBuilder()
- .collection("mongo_hadoop", "yield_historical.in2"))
- .build();
- }
- @Before
- public void checkConfiguration() {
- assumeFalse(isSharded(getInputUri()));
- }
- @Test
- public void testBasicInputSource() {
- LOG.info("testing basic input source");
- MapReduceJob treasuryJob =
- new MapReduceJob(TreasuryYieldXMLConfig.class.getName())
- .jar(JOBJAR_PATH)
- .param("mongo.input.notimeout", "true")
- .inputUris(getInputUri())
- .outputUri(getOutputUri());
- if (isHadoopV1()) {
- treasuryJob.outputCommitter(MongoOutputCommitter.class);
- }
- treasuryJob.execute(isRunTestInVm());
- compareResults(getClient(getInputUri()).getDB(getOutputUri().getDatabase()).getCollection(getOutputUri().getCollection()),
- getReference());
- }
- @Test
- public void testTreasuryJsonConfig() {
- mongoImport("yield_historical.in3", TREASURY_JSON_PATH);
- MapReduceJob treasuryJob =
- new MapReduceJob("com.mongodb.hadoop.examples.treasury.TreasuryYieldXMLConfig")
- .jar(JOBJAR_PATH)
- .param(MONGO_SPLITTER_CLASS, MultiMongoCollectionSplitter.class.getName())
- .param(MULTI_COLLECTION_CONF_KEY, collectionSettings().toString())
- .outputUri(getOutputUri());
- if (isHadoopV1()) {
- treasuryJob.outputCommitter(MongoOutputCommitter.class);
- }
- treasuryJob.execute(isRunTestInVm());
- compareDoubled(getClient(getInputUri()).getDB(getOutputUri().getDatabase()).getCollection(getOutputUri().getCollection()));
- }
- @Test
- public void testMultipleCollectionSupport() {
- mongoImport(getInputUri().getCollection(), TREASURY_JSON_PATH);
- mongoImport(inputUri2.getCollection(), TREASURY_JSON_PATH);
- MapReduceJob treasuryJob =
- new MapReduceJob("com.mongodb.hadoop.examples.treasury.TreasuryYieldXMLConfig")
- .jar(JOBJAR_PATH)
- .param(MONGO_SPLITTER_CLASS, MultiMongoCollectionSplitter.class.getName())
- .inputUris(getInputUri(), inputUri2)
- .outputUri(getOutputUri());
- if (isHadoopV1()) {
- treasuryJob.outputCommitter(MongoOutputCommitter.class);
- }
- treasuryJob.execute(isRunTestInVm());
- compareDoubled(getClient(getInputUri()).getDB(getOutputUri().getDatabase()).getCollection(getOutputUri().getCollection()));
- }
- private JsonNode collectionSettings() {
- ArrayNode settings = new ArrayNode(JsonNodeFactory.instance);
- ObjectNode node = new ObjectNode(JsonNodeFactory.instance);
- node.put(INPUT_URI, getInputUri().toString());
- ObjectNode dow = new ObjectNode(JsonNodeFactory.instance);
- dow.put("dayOfWeek", "FRIDAY");
- node.put("query", dow);
- node.put(MONGO_SPLITTER_CLASS, SingleMongoSplitter.class.getName());
- node.put(SPLITS_USE_RANGEQUERY, true);
- node.put(INPUT_NOTIMEOUT, true);
- settings.add(node);
- MongoClientURI inputUri3 = authCheck(new MongoClientURIBuilder()
- .collection("mongo_hadoop", "yield_historical.in3"))
- .build();
- node = new ObjectNode(JsonNodeFactory.instance);
- node.put(INPUT_URI, inputUri3.toString());
- node.put(SPLITS_USE_RANGEQUERY, true);
- node.put(INPUT_NOTIMEOUT, true);
- settings.add(node);
- return settings;
- }
- }