PageRenderTime 19ms CodeModel.GetById 1ms app.highlight 15ms RepoModel.GetById 1ms app.codeStats 0ms

/tags/release-0.1-rc2/hive/external/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java

#
Java | 192 lines | 118 code | 25 blank | 49 comment | 3 complexity | fae5dbec67ec125eae1f33c4c887534f MD5 | raw file
  1/**
  2 * Licensed to the Apache Software Foundation (ASF) under one
  3 * or more contributor license agreements.  See the NOTICE file
  4 * distributed with this work for additional information
  5 * regarding copyright ownership.  The ASF licenses this file
  6 * to you under the Apache License, Version 2.0 (the
  7 * "License"); you may not use this file except in compliance
  8 * with the License.  You may obtain a copy of the License at
  9 *
 10 *     http://www.apache.org/licenses/LICENSE-2.0
 11 *
 12 * Unless required by applicable law or agreed to in writing, software
 13 * distributed under the License is distributed on an "AS IS" BASIS,
 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 15 * See the License for the specific language governing permissions and
 16 * limitations under the License.
 17 */
 18package org.apache.hadoop.hive.serde2.lazy;
 19
 20import java.util.List;
 21import java.util.Properties;
 22
 23import junit.framework.TestCase;
 24
 25import org.apache.hadoop.conf.Configuration;
 26import org.apache.hadoop.hive.serde.Constants;
 27import org.apache.hadoop.hive.serde2.SerDeException;
 28import org.apache.hadoop.hive.serde2.io.ByteWritable;
 29import org.apache.hadoop.hive.serde2.io.DoubleWritable;
 30import org.apache.hadoop.hive.serde2.io.ShortWritable;
 31import org.apache.hadoop.hive.serde2.objectinspector.StructField;
 32import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
 33import org.apache.hadoop.io.IntWritable;
 34import org.apache.hadoop.io.LongWritable;
 35import org.apache.hadoop.io.Text;
 36
 37/**
 38 * TestLazySimpleSerDe.
 39 *
 40 */
 41public class TestLazySimpleSerDe extends TestCase {
 42
 43  /**
 44   * Test the LazySimpleSerDe class.
 45   */
 46  public void testLazySimpleSerDe() throws Throwable {
 47    try {
 48      // Create the SerDe
 49      LazySimpleSerDe serDe = new LazySimpleSerDe();
 50      Configuration conf = new Configuration();
 51      Properties tbl = createProperties();
 52      serDe.initialize(conf, tbl);
 53
 54      // Data
 55      Text t = new Text("123\t456\t789\t1000\t5.3\thive and hadoop\t1.\tNULL");
 56      String s = "123\t456\t789\t1000\t5.3\thive and hadoop\tNULL\tNULL";
 57      Object[] expectedFieldsData = {new ByteWritable((byte) 123),
 58          new ShortWritable((short) 456), new IntWritable(789),
 59          new LongWritable(1000), new DoubleWritable(5.3),
 60          new Text("hive and hadoop"), null, null};
 61
 62      // Test
 63      deserializeAndSerialize(serDe, t, s, expectedFieldsData);
 64
 65    } catch (Throwable e) {
 66      e.printStackTrace();
 67      throw e;
 68    }
 69  }
 70
 71  private void deserializeAndSerialize(LazySimpleSerDe serDe, Text t, String s,
 72      Object[] expectedFieldsData) throws SerDeException {
 73    // Get the row structure
 74    StructObjectInspector oi = (StructObjectInspector) serDe
 75        .getObjectInspector();
 76    List<? extends StructField> fieldRefs = oi.getAllStructFieldRefs();
 77    assertEquals(8, fieldRefs.size());
 78
 79    // Deserialize
 80    Object row = serDe.deserialize(t);
 81    for (int i = 0; i < fieldRefs.size(); i++) {
 82      Object fieldData = oi.getStructFieldData(row, fieldRefs.get(i));
 83      if (fieldData != null) {
 84        fieldData = ((LazyPrimitive) fieldData).getWritableObject();
 85      }
 86      assertEquals("Field " + i, expectedFieldsData[i], fieldData);
 87    }
 88    // Serialize
 89    assertEquals(Text.class, serDe.getSerializedClass());
 90    Text serializedText = (Text) serDe.serialize(row, oi);
 91    assertEquals("Serialized data", s, serializedText.toString());
 92  }
 93
 94  private Properties createProperties() {
 95    Properties tbl = new Properties();
 96
 97    // Set the configuration parameters
 98    tbl.setProperty(Constants.SERIALIZATION_FORMAT, "9");
 99    tbl.setProperty("columns",
100        "abyte,ashort,aint,along,adouble,astring,anullint,anullstring");
101    tbl.setProperty("columns.types",
102        "tinyint:smallint:int:bigint:double:string:int:string");
103    tbl.setProperty(Constants.SERIALIZATION_NULL_FORMAT, "NULL");
104    return tbl;
105  }
106
107  /**
108   * Test the LazySimpleSerDe class with LastColumnTakesRest option.
109   */
110  public void testLazySimpleSerDeLastColumnTakesRest() throws Throwable {
111    try {
112      // Create the SerDe
113      LazySimpleSerDe serDe = new LazySimpleSerDe();
114      Configuration conf = new Configuration();
115      Properties tbl = createProperties();
116      tbl.setProperty(Constants.SERIALIZATION_LAST_COLUMN_TAKES_REST, "true");
117      serDe.initialize(conf, tbl);
118
119      // Data
120      Text t = new Text("123\t456\t789\t1000\t5.3\thive and hadoop\t1.\ta\tb\t");
121      String s = "123\t456\t789\t1000\t5.3\thive and hadoop\tNULL\ta\tb\t";
122      Object[] expectedFieldsData = {new ByteWritable((byte) 123),
123          new ShortWritable((short) 456), new IntWritable(789),
124          new LongWritable(1000), new DoubleWritable(5.3),
125          new Text("hive and hadoop"), null, new Text("a\tb\t")};
126
127      // Test
128      deserializeAndSerialize(serDe, t, s, expectedFieldsData);
129
130    } catch (Throwable e) {
131      e.printStackTrace();
132      throw e;
133    }
134  }
135
136  /**
137   * Test the LazySimpleSerDe class with extra columns.
138   */
139  public void testLazySimpleSerDeExtraColumns() throws Throwable {
140    try {
141      // Create the SerDe
142      LazySimpleSerDe serDe = new LazySimpleSerDe();
143      Configuration conf = new Configuration();
144      Properties tbl = createProperties();
145      serDe.initialize(conf, tbl);
146
147      // Data
148      Text t = new Text("123\t456\t789\t1000\t5.3\thive and hadoop\t1.\ta\tb\t");
149      String s = "123\t456\t789\t1000\t5.3\thive and hadoop\tNULL\ta";
150      Object[] expectedFieldsData = {new ByteWritable((byte) 123),
151          new ShortWritable((short) 456), new IntWritable(789),
152          new LongWritable(1000), new DoubleWritable(5.3),
153          new Text("hive and hadoop"), null, new Text("a")};
154
155      // Test
156      deserializeAndSerialize(serDe, t, s, expectedFieldsData);
157
158    } catch (Throwable e) {
159      e.printStackTrace();
160      throw e;
161    }
162  }
163
164  /**
165   * Test the LazySimpleSerDe class with missing columns.
166   */
167  public void testLazySimpleSerDeMissingColumns() throws Throwable {
168    try {
169      // Create the SerDe
170      LazySimpleSerDe serDe = new LazySimpleSerDe();
171      Configuration conf = new Configuration();
172      Properties tbl = createProperties();
173      serDe.initialize(conf, tbl);
174
175      // Data
176      Text t = new Text("123\t456\t789\t1000\t5.3\t");
177      String s = "123\t456\t789\t1000\t5.3\t\tNULL\tNULL";
178      Object[] expectedFieldsData = {new ByteWritable((byte) 123),
179          new ShortWritable((short) 456), new IntWritable(789),
180          new LongWritable(1000), new DoubleWritable(5.3), new Text(""), null,
181          null};
182
183      // Test
184      deserializeAndSerialize(serDe, t, s, expectedFieldsData);
185
186    } catch (Throwable e) {
187      e.printStackTrace();
188      throw e;
189    }
190  }
191
192}