PageRenderTime 40ms CodeModel.GetById 14ms RepoModel.GetById 0ms app.codeStats 0ms

/tags/release-0.2.0-rc0/src/test/org/apache/hcatalog/mapreduce/TestHCatOutputFormat.java

#
Java | 174 lines | 126 code | 29 blank | 19 comment | 1 complexity | e6b827fd70360b5d8e687f94c1c6ea94 MD5 | raw file
Possible License(s): Apache-2.0, BSD-3-Clause, JSON, CPL-1.0
  1. /**
  2. * Licensed to the Apache Software Foundation (ASF) under one
  3. * or more contributor license agreements. See the NOTICE file
  4. * distributed with this work for additional information
  5. * regarding copyright ownership. The ASF licenses this file
  6. * to you under the Apache License, Version 2.0 (the
  7. * "License"); you may not use this file except in compliance
  8. * with the License. You may obtain a copy of the License at
  9. *
  10. * http://www.apache.org/licenses/LICENSE-2.0
  11. *
  12. * Unless required by applicable law or agreed to in writing, software
  13. * distributed under the License is distributed on an "AS IS" BASIS,
  14. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. * See the License for the specific language governing permissions and
  16. * limitations under the License.
  17. */
  18. package org.apache.hcatalog.mapreduce;
  19. import java.util.ArrayList;
  20. import java.util.Arrays;
  21. import java.util.HashMap;
  22. import java.util.List;
  23. import java.util.Map;
  24. import junit.framework.TestCase;
  25. import org.apache.hadoop.conf.Configuration;
  26. import org.apache.hadoop.fs.Path;
  27. import org.apache.hadoop.hive.conf.HiveConf;
  28. import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
  29. import org.apache.hadoop.hive.metastore.api.Database;
  30. import org.apache.hadoop.hive.metastore.api.FieldSchema;
  31. import org.apache.hadoop.hive.metastore.api.Partition;
  32. import org.apache.hadoop.hive.metastore.api.SerDeInfo;
  33. import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
  34. import org.apache.hadoop.hive.metastore.api.Table;
  35. import org.apache.hadoop.hive.serde.Constants;
  36. import org.apache.hadoop.mapreduce.Job;
  37. import org.apache.hadoop.mapreduce.OutputCommitter;
  38. import org.apache.hadoop.util.StringUtils;
  39. import org.apache.hcatalog.common.HCatConstants;
  40. import org.apache.hcatalog.mapreduce.HCatOutputCommitter;
  41. import org.apache.hcatalog.mapreduce.HCatOutputFormat;
  42. import org.apache.hcatalog.mapreduce.HCatTableInfo;
  43. import org.apache.hcatalog.mapreduce.InitializeInput;
  44. import org.apache.hcatalog.mapreduce.OutputJobInfo;
  45. import org.apache.hcatalog.mapreduce.StorerInfo;
  46. import org.apache.hcatalog.rcfile.RCFileOutputDriver;
  47. public class TestHCatOutputFormat extends TestCase {
  48. private HiveMetaStoreClient client;
  49. private HiveConf hiveConf;
  50. private static final String dbName = "hcatOutputFormatTestDB";
  51. private static final String tblName = "hcatOutputFormatTestTable";
  52. @Override
  53. protected void setUp() throws Exception {
  54. super.setUp();
  55. hiveConf = new HiveConf(this.getClass());
  56. try {
  57. client = new HiveMetaStoreClient(hiveConf, null);
  58. initTable();
  59. } catch (Throwable e) {
  60. System.err.println("Unable to open the metastore");
  61. System.err.println(StringUtils.stringifyException(e));
  62. throw new Exception(e);
  63. }
  64. }
  65. @Override
  66. protected void tearDown() throws Exception {
  67. try {
  68. super.tearDown();
  69. client.dropTable(dbName, tblName);
  70. client.dropDatabase(dbName);
  71. client.close();
  72. } catch (Throwable e) {
  73. System.err.println("Unable to close metastore");
  74. System.err.println(StringUtils.stringifyException(e));
  75. throw new Exception(e);
  76. }
  77. }
  78. private void initTable() throws Exception {
  79. try {
  80. client.dropTable(dbName, tblName);
  81. } catch(Exception e) {}
  82. try {
  83. client.dropDatabase(dbName);
  84. } catch(Exception e) {}
  85. client.createDatabase(new Database(dbName, "", null,null));
  86. assertNotNull((client.getDatabase(dbName).getLocationUri()));
  87. List<FieldSchema> fields = new ArrayList<FieldSchema>();
  88. fields.add(new FieldSchema("colname", Constants.STRING_TYPE_NAME, ""));
  89. Table tbl = new Table();
  90. tbl.setDbName(dbName);
  91. tbl.setTableName(tblName);
  92. StorageDescriptor sd = new StorageDescriptor();
  93. sd.setCols(fields);
  94. tbl.setSd(sd);
  95. //sd.setLocation("hdfs://tmp");
  96. sd.setParameters(new HashMap<String, String>());
  97. sd.getParameters().put("test_param_1", "Use this for comments etc");
  98. sd.setBucketCols(new ArrayList<String>(2));
  99. sd.getBucketCols().add("name");
  100. sd.setSerdeInfo(new SerDeInfo());
  101. sd.getSerdeInfo().setName(tbl.getTableName());
  102. sd.getSerdeInfo().setParameters(new HashMap<String, String>());
  103. sd.getSerdeInfo().getParameters().put(
  104. org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT, "1");
  105. sd.getSerdeInfo().setSerializationLib(
  106. org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName());
  107. tbl.setPartitionKeys(fields);
  108. Map<String, String> tableParams = new HashMap<String, String>();
  109. tableParams.put(HCatConstants.HCAT_OSD_CLASS, RCFileOutputDriver.class.getName());
  110. tableParams.put(HCatConstants.HCAT_ISD_CLASS, "testInputClass");
  111. tableParams.put("hcat.testarg", "testArgValue");
  112. tbl.setParameters(tableParams);
  113. client.createTable(tbl);
  114. Path tblPath = new Path(client.getTable(dbName, tblName).getSd().getLocation());
  115. assertTrue(tblPath.getFileSystem(hiveConf).mkdirs(new Path(tblPath,"colname=p1")));
  116. }
  117. public void testSetOutput() throws Exception {
  118. Configuration conf = new Configuration();
  119. Job job = new Job(conf, "test outputformat");
  120. Map<String, String> partitionValues = new HashMap<String, String>();
  121. partitionValues.put("colname", "p1");
  122. //null server url means local mode
  123. HCatTableInfo info = HCatTableInfo.getOutputTableInfo(null, null, dbName, tblName, partitionValues);
  124. HCatOutputFormat.setOutput(job, info);
  125. OutputJobInfo jobInfo = HCatOutputFormat.getJobInfo(job);
  126. assertNotNull(jobInfo.getTableInfo());
  127. assertEquals(1, jobInfo.getTableInfo().getPartitionValues().size());
  128. assertEquals("p1", jobInfo.getTableInfo().getPartitionValues().get("colname"));
  129. assertEquals(1, jobInfo.getTableSchema().getFields().size());
  130. assertEquals("colname", jobInfo.getTableSchema().getFields().get(0).getName());
  131. StorerInfo storer = jobInfo.getStorerInfo();
  132. assertEquals(RCFileOutputDriver.class.getName(), storer.getOutputSDClass());
  133. publishTest(job);
  134. }
  135. public void publishTest(Job job) throws Exception {
  136. OutputCommitter committer = new HCatOutputCommitter(job,null);
  137. committer.cleanupJob(job);
  138. Partition part = client.getPartition(dbName, tblName, Arrays.asList("p1"));
  139. assertNotNull(part);
  140. StorerInfo storer = InitializeInput.extractStorerInfo(part.getSd(),part.getParameters());
  141. assertEquals(storer.getInputSDClass(), "testInputClass");
  142. assertEquals(storer.getProperties().get("hcat.testarg"), "testArgValue");
  143. assertTrue(part.getSd().getLocation().indexOf("p1") != -1);
  144. }
  145. }