PageRenderTime 75ms CodeModel.GetById 18ms app.highlight 53ms RepoModel.GetById 1ms app.codeStats 0ms

/tags/release-0.0.0-rc0/hive/external/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java

#
Java | 314 lines | 237 code | 38 blank | 39 comment | 0 complexity | c75036783d5b6568ba5c7ea770ef1d72 MD5 | raw file
  1/**
  2 * Licensed to the Apache Software Foundation (ASF) under one
  3 * or more contributor license agreements.  See the NOTICE file
  4 * distributed with this work for additional information
  5 * regarding copyright ownership.  The ASF licenses this file
  6 * to you under the Apache License, Version 2.0 (the
  7 * "License"); you may not use this file except in compliance
  8 * with the License.  You may obtain a copy of the License at
  9 *
 10 *     http://www.apache.org/licenses/LICENSE-2.0
 11 *
 12 * Unless required by applicable law or agreed to in writing, software
 13 * distributed under the License is distributed on an "AS IS" BASIS,
 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 15 * See the License for the specific language governing permissions and
 16 * limitations under the License.
 17 */
 18package org.apache.hadoop.hive.serde2.lazy;
 19
 20import java.util.ArrayList;
 21import java.util.Arrays;
 22import java.util.List;
 23
 24import junit.framework.TestCase;
 25
 26import org.apache.hadoop.hive.serde2.SerDeUtils;
 27import org.apache.hadoop.hive.serde2.io.ByteWritable;
 28import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 29import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
 30import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
 31import org.apache.hadoop.io.IntWritable;
 32import org.apache.hadoop.io.Text;
 33
 34/**
 35 * Tests LazyArray, LazyMap, LazyStruct and LazyUnion
 36 *
 37 */
 38public class TestLazyArrayMapStruct extends TestCase {
 39
 40  /**
 41   * Test the LazyArray class.
 42   */
 43  public void testLazyArray() throws Throwable {
 44    try {
 45      // Array of Byte
 46      Text nullSequence = new Text("\\N");
 47      ObjectInspector oi = LazyFactory.createLazyObjectInspector(TypeInfoUtils
 48          .getTypeInfosFromTypeString("array<tinyint>").get(0),
 49          new byte[] {(byte) 1}, 0, nullSequence, false, (byte) 0);
 50      LazyArray b = (LazyArray) LazyFactory.createLazyObject(oi);
 51      byte[] data = new byte[] {'-', '1', 1, '\\', 'N', 1, '8'};
 52      TestLazyPrimitive.initLazyObject(b, data, 0, data.length);
 53
 54      assertNull(b.getListElementObject(-1));
 55      assertEquals(new ByteWritable((byte) -1), ((LazyByte) b
 56          .getListElementObject(0)).getWritableObject());
 57      assertEquals(new ByteWritable((byte) -1), ((LazyByte) b.getList().get(0))
 58          .getWritableObject());
 59      assertNull(b.getListElementObject(1));
 60      assertNull(b.getList().get(1));
 61      assertEquals(new ByteWritable((byte) 8), ((LazyByte) b
 62          .getListElementObject(2)).getWritableObject());
 63      assertEquals(new ByteWritable((byte) 8), ((LazyByte) b.getList().get(2))
 64          .getWritableObject());
 65      assertNull(b.getListElementObject(3));
 66      assertEquals(3, b.getList().size());
 67
 68      // Array of String
 69      oi = LazyFactory.createLazyObjectInspector(TypeInfoUtils
 70          .getTypeInfosFromTypeString("array<string>").get(0),
 71          new byte[] {(byte) '\t'}, 0, nullSequence, false, (byte) 0);
 72      b = (LazyArray) LazyFactory.createLazyObject(oi);
 73      data = new byte[] {'a', 'b', '\t', 'c', '\t', '\\', 'N', '\t', '\t', 'd'};
 74      // Note: the first and last element of the byte[] are NOT used
 75      TestLazyPrimitive.initLazyObject(b, data, 1, data.length - 2);
 76      assertNull(b.getListElementObject(-1));
 77      assertEquals(new Text("b"), ((LazyString) b.getListElementObject(0))
 78          .getWritableObject());
 79      assertEquals(new Text("b"), ((LazyString) b.getList().get(0))
 80          .getWritableObject());
 81      assertEquals(new Text("c"), ((LazyString) b.getListElementObject(1))
 82          .getWritableObject());
 83      assertEquals(new Text("c"), ((LazyString) b.getList().get(1))
 84          .getWritableObject());
 85      assertNull((b.getListElementObject(2)));
 86      assertNull((b.getList().get(2)));
 87      assertEquals(new Text(""), ((LazyString) b.getListElementObject(3))
 88          .getWritableObject());
 89      assertEquals(new Text(""), ((LazyString) b.getList().get(3))
 90          .getWritableObject());
 91      assertEquals(new Text(""), ((LazyString) b.getListElementObject(4))
 92          .getWritableObject());
 93      assertEquals(new Text(""), ((LazyString) b.getList().get(4))
 94          .getWritableObject());
 95      assertNull((b.getListElementObject(5)));
 96      assertEquals(5, b.getList().size());
 97
 98    } catch (Throwable e) {
 99      e.printStackTrace();
100      throw e;
101    }
102  }
103
104  /**
105   * Test the LazyMap class.
106   */
107  public void testLazyMap() throws Throwable {
108    try {
109      {
110        // Map of Integer to String
111        Text nullSequence = new Text("\\N");
112        ObjectInspector oi = LazyFactory
113            .createLazyObjectInspector(TypeInfoUtils
114            .getTypeInfosFromTypeString("map<int,string>").get(0),
115            new byte[] {(byte) 1, (byte) 2}, 0, nullSequence, false,
116            (byte) 0);
117        LazyMap b = (LazyMap) LazyFactory.createLazyObject(oi);
118        byte[] data = new byte[] {'2', 2, 'd', 'e', 'f', 1, '-', '1', 2, '\\',
119            'N', 1, '0', 2, '0', 1, '8', 2, 'a', 'b', 'c'};
120        TestLazyPrimitive.initLazyObject(b, data, 0, data.length);
121
122        assertEquals(new Text("def"), ((LazyString) b
123            .getMapValueElement(new IntWritable(2))).getWritableObject());
124        assertNull(b.getMapValueElement(new IntWritable(-1)));
125        assertEquals(new Text("0"), ((LazyString) b
126            .getMapValueElement(new IntWritable(0))).getWritableObject());
127        assertEquals(new Text("abc"), ((LazyString) b
128            .getMapValueElement(new IntWritable(8))).getWritableObject());
129        assertNull(b.getMapValueElement(new IntWritable(12345)));
130
131        assertEquals("{2:'def',-1:null,0:'0',8:'abc'}".replace('\'', '\"'),
132            SerDeUtils.getJSONString(b, oi));
133      }
134
135      {
136        // Map of String to String
137        Text nullSequence = new Text("\\N");
138        ObjectInspector oi = LazyFactory.createLazyObjectInspector(
139            TypeInfoUtils.getTypeInfosFromTypeString("map<string,string>").get(
140            0), new byte[] {(byte) '#', (byte) '\t'}, 0, nullSequence,
141            false, (byte) 0);
142        LazyMap b = (LazyMap) LazyFactory.createLazyObject(oi);
143        byte[] data = new byte[] {'2', '\t', 'd', '\t', 'f', '#', '2', '\t',
144            'd', '#', '-', '1', '#', '0', '\t', '0', '#', '8', '\t', 'a', 'b', 'c'};
145        TestLazyPrimitive.initLazyObject(b, data, 0, data.length);
146
147        assertEquals(new Text("d\tf"), ((LazyString) b
148            .getMapValueElement(new Text("2"))).getWritableObject());
149        assertNull(b.getMapValueElement(new Text("-1")));
150        assertEquals(new Text("0"), ((LazyString) b
151            .getMapValueElement(new Text("0"))).getWritableObject());
152        assertEquals(new Text("abc"), ((LazyString) b
153            .getMapValueElement(new Text("8"))).getWritableObject());
154        assertNull(b.getMapValueElement(new Text("-")));
155
156        assertEquals("{'2':'d\\tf','2':'d','-1':null,'0':'0','8':'abc'}"
157            .replace('\'', '\"'), SerDeUtils.getJSONString(b, oi));
158      }
159
160    } catch (Throwable e) {
161      e.printStackTrace();
162      throw e;
163    }
164  }
165
166  /**
167   * Test the LazyStruct class.
168   */
169  public void testLazyStruct() throws Throwable {
170    try {
171      {
172        ArrayList<TypeInfo> fieldTypeInfos = TypeInfoUtils
173            .getTypeInfosFromTypeString("int,array<string>,map<string,string>,string");
174        List<String> fieldNames = Arrays.asList(new String[] {"a", "b", "c",
175            "d"});
176        Text nullSequence = new Text("\\N");
177
178        ObjectInspector oi = LazyFactory.createLazyStructInspector(fieldNames,
179            fieldTypeInfos, new byte[] {' ', ':', '='}, nullSequence, false,
180            false, (byte) 0);
181        LazyStruct o = (LazyStruct) LazyFactory.createLazyObject(oi);
182
183        Text data;
184
185        data = new Text("123 a:b:c d=e:f=g hi");
186        TestLazyPrimitive.initLazyObject(o, data.getBytes(), 0, data
187            .getLength());
188        assertEquals(
189            "{'a':123,'b':['a','b','c'],'c':{'d':'e','f':'g'},'d':'hi'}"
190            .replace("'", "\""), SerDeUtils.getJSONString(o, oi));
191
192        data = new Text("123 \\N d=e:f=g \\N");
193        TestLazyPrimitive.initLazyObject(o, data.getBytes(), 0, data
194            .getLength());
195        assertEquals("{'a':123,'b':null,'c':{'d':'e','f':'g'},'d':null}"
196            .replace("'", "\""), SerDeUtils.getJSONString(o, oi));
197
198        data = new Text("\\N a d=\\N:f=g:h no tail");
199        TestLazyPrimitive.initLazyObject(o, data.getBytes(), 0, data
200            .getLength());
201        assertEquals(
202            "{'a':null,'b':['a'],'c':{'d':null,'f':'g','h':null},'d':'no'}"
203            .replace("'", "\""), SerDeUtils.getJSONString(o, oi));
204
205        data = new Text("\\N :a:: \\N no tail");
206        TestLazyPrimitive.initLazyObject(o, data.getBytes(), 0, data
207            .getLength());
208        assertEquals("{'a':null,'b':['','a','',''],'c':null,'d':'no'}".replace(
209            "'", "\""), SerDeUtils.getJSONString(o, oi));
210
211        data = new Text("123   ");
212        TestLazyPrimitive.initLazyObject(o, data.getBytes(), 0, data
213            .getLength());
214        assertEquals("{'a':123,'b':[],'c':{},'d':''}".replace("'", "\""),
215            SerDeUtils.getJSONString(o, oi));
216
217        data = new Text(": : : :");
218        TestLazyPrimitive.initLazyObject(o, data.getBytes(), 0, data
219            .getLength());
220        assertEquals("{'a':null,'b':['',''],'c':{'':null,'':null},'d':':'}"
221            .replace("'", "\""), SerDeUtils.getJSONString(o, oi));
222
223        data = new Text("= = = =");
224        TestLazyPrimitive.initLazyObject(o, data.getBytes(), 0, data
225            .getLength());
226        assertEquals("{'a':null,'b':['='],'c':{'':''},'d':'='}".replace("'",
227            "\""), SerDeUtils.getJSONString(o, oi));
228
229        // test LastColumnTakesRest
230        oi = LazyFactory.createLazyStructInspector(Arrays.asList(new String[] {
231            "a", "b", "c", "d"}), fieldTypeInfos,
232            new byte[] {' ', ':', '='}, nullSequence, true, false, (byte) 0);
233        o = (LazyStruct) LazyFactory.createLazyObject(oi);
234        data = new Text("\\N a d=\\N:f=g:h has tail");
235        TestLazyPrimitive.initLazyObject(o, data.getBytes(), 0, data
236            .getLength());
237        assertEquals(
238            "{'a':null,'b':['a'],'c':{'d':null,'f':'g','h':null},'d':'has tail'}"
239            .replace("'", "\""), SerDeUtils.getJSONString(o, oi));
240      }
241    } catch (Throwable e) {
242      e.printStackTrace();
243      throw e;
244    }
245  }
246
247  /**
248   * Test the LazyUnion class.
249   */
250  public void testLazyUnion() throws Throwable {
251    try {
252      {
253        TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(
254            "uniontype<int,array<string>,map<string,string>,string>");
255        Text nullSequence = new Text("\\N");
256
257        ObjectInspector oi = LazyFactory.createLazyObjectInspector(typeInfo,
258            new byte[] {'^', ':', '='}, 0, nullSequence, false, (byte) 0);
259        LazyUnion o = (LazyUnion) LazyFactory.createLazyObject(oi);
260
261        Text data;
262
263        data = new Text("0^123");
264        TestLazyPrimitive.initLazyObject(o, data.getBytes(), 0,
265            data.getLength());
266        assertEquals("{0:123}", SerDeUtils.getJSONString(o, oi));
267
268        data = new Text("1^a:b:c");
269        TestLazyPrimitive.initLazyObject(o, data.getBytes(), 0,
270            data.getLength());
271        assertEquals(
272            "{1:[\"a\",\"b\",\"c\"]}", SerDeUtils.getJSONString(o, oi));
273
274        data = new Text("2^d=e:f=g");
275        TestLazyPrimitive.initLazyObject(o, data.getBytes(), 0,
276            data.getLength());
277        assertEquals(
278            "{2:{\"d\":\"e\",\"f\":\"g\"}}", SerDeUtils.getJSONString(o, oi));
279
280        data = new Text("3^hi");
281        TestLazyPrimitive.initLazyObject(o, data.getBytes(), 0,
282            data.getLength());
283        assertEquals("{3:\"hi\"}", SerDeUtils.getJSONString(o, oi));
284
285
286        data = new Text("0^\\N");
287        TestLazyPrimitive.initLazyObject(o, data.getBytes(), 0,
288            data.getLength());
289        assertEquals("{0:null}", SerDeUtils.getJSONString(o, oi));
290
291        data = new Text("1^ :a::");
292        TestLazyPrimitive.initLazyObject(o, data.getBytes(), 0,
293            data.getLength());
294        assertEquals(
295            "{1:[\" \",\"a\",\"\",\"\"]}", SerDeUtils.getJSONString(o, oi));
296
297        data = new Text("2^d=\\N:f=g:h");
298        TestLazyPrimitive.initLazyObject(o, data.getBytes(), 0,
299            data.getLength());
300        assertEquals(
301            "{2:{\"d\":null,\"f\":\"g\",\"h\":null}}",
302            SerDeUtils.getJSONString(o, oi));
303
304        data = new Text("2^= ");
305        TestLazyPrimitive.initLazyObject(o, data.getBytes(), 0,
306            data.getLength());
307        assertEquals("{2:{\"\":\" \"}}", SerDeUtils.getJSONString(o, oi));
308      }
309    } catch (Throwable e) {
310      e.printStackTrace();
311      throw e;
312    }
313  }
314}