PageRenderTime 20ms CodeModel.GetById 11ms app.highlight 6ms RepoModel.GetById 1ms app.codeStats 0ms

/tags/release-0.0.0-rc0/hive/external/contrib/src/test/org/apache/hadoop/hive/contrib/mr/TestGenericMR.java

#
Java | 137 lines | 91 code | 24 blank | 22 comment | 4 complexity | 94eed74efbcfdb08aeb4b4c0e8593d87 MD5 | raw file
  1/**
  2 * Licensed to the Apache Software Foundation (ASF) under one
  3 * or more contributor license agreements.  See the NOTICE file
  4 * distributed with this work for additional information
  5 * regarding copyright ownership.  The ASF licenses this file
  6 * to you under the Apache License, Version 2.0 (the
  7 * "License"); you may not use this file except in compliance
  8 * with the License.  You may obtain a copy of the License at
  9 *
 10 *     http://www.apache.org/licenses/LICENSE-2.0
 11 *
 12 * Unless required by applicable law or agreed to in writing, software
 13 * distributed under the License is distributed on an "AS IS" BASIS,
 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 15 * See the License for the specific language governing permissions and
 16 * limitations under the License.
 17 */
 18package org.apache.hadoop.hive.contrib.mr;
 19
 20import java.io.StringReader;
 21import java.io.StringWriter;
 22import java.util.Iterator;
 23import java.util.NoSuchElementException;
 24
 25import junit.framework.TestCase;
 26
 27/**
 28 * TestGenericMR.
 29 *
 30 */
 31public final class TestGenericMR extends TestCase {
 32  public void testReduceTooFar() throws Exception {
 33    try {
 34      new GenericMR().reduce(new StringReader("a\tb\tc"), new StringWriter(),
 35          new Reducer() {
 36        public void reduce(String key, Iterator<String[]> records,
 37            Output output) throws Exception {
 38          while (true) {
 39            records.next();
 40          }
 41        }
 42      });
 43    } catch (final NoSuchElementException nsee) {
 44      // expected
 45      return;
 46    }
 47
 48    fail("Expected NoSuchElementException");
 49  }
 50
 51  public void testEmptyMap() throws Exception {
 52    final StringWriter out = new StringWriter();
 53
 54    new GenericMR().map(new StringReader(""), out, identityMapper());
 55
 56    assertEquals(0, out.toString().length());
 57  }
 58
 59  public void testIdentityMap() throws Exception {
 60    final String in = "a\tb\nc\td";
 61    final StringWriter out = new StringWriter();
 62
 63    new GenericMR().map(new StringReader(in), out, identityMapper());
 64
 65    assertEquals(in + "\n", out.toString());
 66  }
 67
 68  public void testKVSplitMap() throws Exception {
 69    final String in = "k1=v1,k2=v2\nk1=v2,k2=v3";
 70    final String expected = "k1\tv1\nk2\tv2\nk1\tv2\nk2\tv3\n";
 71    final StringWriter out = new StringWriter();
 72
 73    new GenericMR().map(new StringReader(in), out, new Mapper() {
 74      public void map(String[] record, Output output) throws Exception {
 75        for (final String kvs : record[0].split(",")) {
 76          final String[] kv = kvs.split("=");
 77          output.collect(new String[] {kv[0], kv[1]});
 78        }
 79      }
 80    });
 81
 82    assertEquals(expected, out.toString());
 83  }
 84
 85  public void testIdentityReduce() throws Exception {
 86    final String in = "a\tb\nc\td";
 87    final StringWriter out = new StringWriter();
 88
 89    new GenericMR().reduce(new StringReader(in), out, identityReducer());
 90
 91    assertEquals(in + "\n", out.toString());
 92  }
 93
 94  public void testWordCountReduce() throws Exception {
 95    final String in = "hello\t1\nhello\t2\nokay\t4\nokay\t6\nokay\t2";
 96    final StringWriter out = new StringWriter();
 97
 98    new GenericMR().reduce(new StringReader(in), out, new Reducer() {
 99      @Override
100      public void reduce(String key, Iterator<String[]> records, Output output)
101          throws Exception {
102        int count = 0;
103
104        while (records.hasNext()) {
105          count += Integer.parseInt(records.next()[1]);
106        }
107
108        output.collect(new String[] {key, String.valueOf(count)});
109      }
110    });
111
112    final String expected = "hello\t3\nokay\t12\n";
113
114    assertEquals(expected, out.toString());
115  }
116
117  private Mapper identityMapper() {
118    return new Mapper() {
119      @Override
120      public void map(String[] record, Output output) throws Exception {
121        output.collect(record);
122      }
123    };
124  }
125
126  private Reducer identityReducer() {
127    return new Reducer() {
128      @Override
129      public void reduce(String key, Iterator<String[]> records, Output output)
130          throws Exception {
131        while (records.hasNext()) {
132          output.collect(records.next());
133        }
134      }
135    };
136  }
137}