PageRenderTime 23ms CodeModel.GetById 11ms app.highlight 9ms RepoModel.GetById 1ms app.codeStats 0ms

/tags/release-0.2.0-rc0/src/test/org/apache/hcatalog/mapreduce/TestHCatOutputFormat.java

#
Java | 174 lines | 126 code | 29 blank | 19 comment | 1 complexity | e6b827fd70360b5d8e687f94c1c6ea94 MD5 | raw file
  1/**
  2 * Licensed to the Apache Software Foundation (ASF) under one
  3 * or more contributor license agreements.  See the NOTICE file
  4 * distributed with this work for additional information
  5 * regarding copyright ownership.  The ASF licenses this file
  6 * to you under the Apache License, Version 2.0 (the
  7 * "License"); you may not use this file except in compliance
  8 * with the License.  You may obtain a copy of the License at
  9 *
 10 *     http://www.apache.org/licenses/LICENSE-2.0
 11 *
 12 * Unless required by applicable law or agreed to in writing, software
 13 * distributed under the License is distributed on an "AS IS" BASIS,
 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 15 * See the License for the specific language governing permissions and
 16 * limitations under the License.
 17 */
 18
 19package org.apache.hcatalog.mapreduce;
 20
 21import java.util.ArrayList;
 22import java.util.Arrays;
 23import java.util.HashMap;
 24import java.util.List;
 25import java.util.Map;
 26
 27import junit.framework.TestCase;
 28
 29import org.apache.hadoop.conf.Configuration;
 30import org.apache.hadoop.fs.Path;
 31import org.apache.hadoop.hive.conf.HiveConf;
 32import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
 33import org.apache.hadoop.hive.metastore.api.Database;
 34import org.apache.hadoop.hive.metastore.api.FieldSchema;
 35import org.apache.hadoop.hive.metastore.api.Partition;
 36import org.apache.hadoop.hive.metastore.api.SerDeInfo;
 37import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
 38import org.apache.hadoop.hive.metastore.api.Table;
 39import org.apache.hadoop.hive.serde.Constants;
 40import org.apache.hadoop.mapreduce.Job;
 41import org.apache.hadoop.mapreduce.OutputCommitter;
 42import org.apache.hadoop.util.StringUtils;
 43import org.apache.hcatalog.common.HCatConstants;
 44import org.apache.hcatalog.mapreduce.HCatOutputCommitter;
 45import org.apache.hcatalog.mapreduce.HCatOutputFormat;
 46import org.apache.hcatalog.mapreduce.HCatTableInfo;
 47import org.apache.hcatalog.mapreduce.InitializeInput;
 48import org.apache.hcatalog.mapreduce.OutputJobInfo;
 49import org.apache.hcatalog.mapreduce.StorerInfo;
 50import org.apache.hcatalog.rcfile.RCFileOutputDriver;
 51
 52public class TestHCatOutputFormat extends TestCase {
 53  private HiveMetaStoreClient client;
 54  private HiveConf hiveConf;
 55
 56  private static final String dbName = "hcatOutputFormatTestDB";
 57  private static final String tblName = "hcatOutputFormatTestTable";
 58
 59  @Override
 60  protected void setUp() throws Exception {
 61    super.setUp();
 62    hiveConf = new HiveConf(this.getClass());
 63
 64    try {
 65      client = new HiveMetaStoreClient(hiveConf, null);
 66
 67      initTable();
 68    } catch (Throwable e) {
 69      System.err.println("Unable to open the metastore");
 70      System.err.println(StringUtils.stringifyException(e));
 71      throw new Exception(e);
 72    }
 73  }
 74
 75  @Override
 76  protected void tearDown() throws Exception {
 77    try {
 78      super.tearDown();
 79      client.dropTable(dbName, tblName);
 80      client.dropDatabase(dbName);
 81
 82      client.close();
 83    } catch (Throwable e) {
 84      System.err.println("Unable to close metastore");
 85      System.err.println(StringUtils.stringifyException(e));
 86      throw new Exception(e);
 87    }
 88  }
 89
 90  private void initTable() throws Exception {
 91
 92    try {
 93      client.dropTable(dbName, tblName);
 94    } catch(Exception e) {}
 95    try {
 96      client.dropDatabase(dbName);
 97    } catch(Exception e) {}
 98    client.createDatabase(new Database(dbName, "", null,null));
 99    assertNotNull((client.getDatabase(dbName).getLocationUri()));
100
101    List<FieldSchema> fields = new ArrayList<FieldSchema>();
102    fields.add(new FieldSchema("colname", Constants.STRING_TYPE_NAME, ""));
103
104    Table tbl = new Table();
105    tbl.setDbName(dbName);
106    tbl.setTableName(tblName);
107    StorageDescriptor sd = new StorageDescriptor();
108    sd.setCols(fields);
109    tbl.setSd(sd);
110
111    //sd.setLocation("hdfs://tmp");
112    sd.setParameters(new HashMap<String, String>());
113    sd.getParameters().put("test_param_1", "Use this for comments etc");
114    sd.setBucketCols(new ArrayList<String>(2));
115    sd.getBucketCols().add("name");
116    sd.setSerdeInfo(new SerDeInfo());
117    sd.getSerdeInfo().setName(tbl.getTableName());
118    sd.getSerdeInfo().setParameters(new HashMap<String, String>());
119    sd.getSerdeInfo().getParameters().put(
120        org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT, "1");
121    sd.getSerdeInfo().setSerializationLib(
122        org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName());
123    tbl.setPartitionKeys(fields);
124
125    Map<String, String> tableParams = new HashMap<String, String>();
126    tableParams.put(HCatConstants.HCAT_OSD_CLASS, RCFileOutputDriver.class.getName());
127    tableParams.put(HCatConstants.HCAT_ISD_CLASS, "testInputClass");
128    tableParams.put("hcat.testarg", "testArgValue");
129
130    tbl.setParameters(tableParams);
131
132    client.createTable(tbl);
133    Path tblPath = new Path(client.getTable(dbName, tblName).getSd().getLocation());
134    assertTrue(tblPath.getFileSystem(hiveConf).mkdirs(new Path(tblPath,"colname=p1")));
135
136  }
137
138  public void testSetOutput() throws Exception {
139    Configuration conf = new Configuration();
140    Job job = new Job(conf, "test outputformat");
141
142    Map<String, String> partitionValues = new HashMap<String, String>();
143    partitionValues.put("colname", "p1");
144    //null server url means local mode
145    HCatTableInfo info = HCatTableInfo.getOutputTableInfo(null, null, dbName, tblName, partitionValues);
146
147    HCatOutputFormat.setOutput(job, info);
148    OutputJobInfo jobInfo = HCatOutputFormat.getJobInfo(job);
149
150    assertNotNull(jobInfo.getTableInfo());
151    assertEquals(1, jobInfo.getTableInfo().getPartitionValues().size());
152    assertEquals("p1", jobInfo.getTableInfo().getPartitionValues().get("colname"));
153    assertEquals(1, jobInfo.getTableSchema().getFields().size());
154    assertEquals("colname", jobInfo.getTableSchema().getFields().get(0).getName());
155
156    StorerInfo storer = jobInfo.getStorerInfo();
157    assertEquals(RCFileOutputDriver.class.getName(), storer.getOutputSDClass());
158
159    publishTest(job);
160  }
161
162  public void publishTest(Job job) throws Exception {
163    OutputCommitter committer = new HCatOutputCommitter(job,null);
164    committer.cleanupJob(job);
165
166    Partition part = client.getPartition(dbName, tblName, Arrays.asList("p1"));
167    assertNotNull(part);
168
169    StorerInfo storer = InitializeInput.extractStorerInfo(part.getSd(),part.getParameters());
170    assertEquals(storer.getInputSDClass(), "testInputClass");
171    assertEquals(storer.getProperties().get("hcat.testarg"), "testArgValue");
172    assertTrue(part.getSd().getLocation().indexOf("p1") != -1);
173  }
174}