/tags/release-0.2.0-rc0/src/test/org/apache/hcatalog/mapreduce/TestHCatOutputFormat.java
Java | 174 lines | 126 code | 29 blank | 19 comment | 1 complexity | e6b827fd70360b5d8e687f94c1c6ea94 MD5 | raw file
Possible License(s): Apache-2.0, BSD-3-Clause, JSON, CPL-1.0
- /**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
- package org.apache.hcatalog.mapreduce;
- import java.util.ArrayList;
- import java.util.Arrays;
- import java.util.HashMap;
- import java.util.List;
- import java.util.Map;
- import junit.framework.TestCase;
- import org.apache.hadoop.conf.Configuration;
- import org.apache.hadoop.fs.Path;
- import org.apache.hadoop.hive.conf.HiveConf;
- import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
- import org.apache.hadoop.hive.metastore.api.Database;
- import org.apache.hadoop.hive.metastore.api.FieldSchema;
- import org.apache.hadoop.hive.metastore.api.Partition;
- import org.apache.hadoop.hive.metastore.api.SerDeInfo;
- import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
- import org.apache.hadoop.hive.metastore.api.Table;
- import org.apache.hadoop.hive.serde.Constants;
- import org.apache.hadoop.mapreduce.Job;
- import org.apache.hadoop.mapreduce.OutputCommitter;
- import org.apache.hadoop.util.StringUtils;
- import org.apache.hcatalog.common.HCatConstants;
- import org.apache.hcatalog.mapreduce.HCatOutputCommitter;
- import org.apache.hcatalog.mapreduce.HCatOutputFormat;
- import org.apache.hcatalog.mapreduce.HCatTableInfo;
- import org.apache.hcatalog.mapreduce.InitializeInput;
- import org.apache.hcatalog.mapreduce.OutputJobInfo;
- import org.apache.hcatalog.mapreduce.StorerInfo;
- import org.apache.hcatalog.rcfile.RCFileOutputDriver;
- public class TestHCatOutputFormat extends TestCase {
- private HiveMetaStoreClient client;
- private HiveConf hiveConf;
- private static final String dbName = "hcatOutputFormatTestDB";
- private static final String tblName = "hcatOutputFormatTestTable";
- @Override
- protected void setUp() throws Exception {
- super.setUp();
- hiveConf = new HiveConf(this.getClass());
- try {
- client = new HiveMetaStoreClient(hiveConf, null);
- initTable();
- } catch (Throwable e) {
- System.err.println("Unable to open the metastore");
- System.err.println(StringUtils.stringifyException(e));
- throw new Exception(e);
- }
- }
- @Override
- protected void tearDown() throws Exception {
- try {
- super.tearDown();
- client.dropTable(dbName, tblName);
- client.dropDatabase(dbName);
- client.close();
- } catch (Throwable e) {
- System.err.println("Unable to close metastore");
- System.err.println(StringUtils.stringifyException(e));
- throw new Exception(e);
- }
- }
- private void initTable() throws Exception {
- try {
- client.dropTable(dbName, tblName);
- } catch(Exception e) {}
- try {
- client.dropDatabase(dbName);
- } catch(Exception e) {}
- client.createDatabase(new Database(dbName, "", null,null));
- assertNotNull((client.getDatabase(dbName).getLocationUri()));
- List<FieldSchema> fields = new ArrayList<FieldSchema>();
- fields.add(new FieldSchema("colname", Constants.STRING_TYPE_NAME, ""));
- Table tbl = new Table();
- tbl.setDbName(dbName);
- tbl.setTableName(tblName);
- StorageDescriptor sd = new StorageDescriptor();
- sd.setCols(fields);
- tbl.setSd(sd);
- //sd.setLocation("hdfs://tmp");
- sd.setParameters(new HashMap<String, String>());
- sd.getParameters().put("test_param_1", "Use this for comments etc");
- sd.setBucketCols(new ArrayList<String>(2));
- sd.getBucketCols().add("name");
- sd.setSerdeInfo(new SerDeInfo());
- sd.getSerdeInfo().setName(tbl.getTableName());
- sd.getSerdeInfo().setParameters(new HashMap<String, String>());
- sd.getSerdeInfo().getParameters().put(
- org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT, "1");
- sd.getSerdeInfo().setSerializationLib(
- org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName());
- tbl.setPartitionKeys(fields);
- Map<String, String> tableParams = new HashMap<String, String>();
- tableParams.put(HCatConstants.HCAT_OSD_CLASS, RCFileOutputDriver.class.getName());
- tableParams.put(HCatConstants.HCAT_ISD_CLASS, "testInputClass");
- tableParams.put("hcat.testarg", "testArgValue");
- tbl.setParameters(tableParams);
- client.createTable(tbl);
- Path tblPath = new Path(client.getTable(dbName, tblName).getSd().getLocation());
- assertTrue(tblPath.getFileSystem(hiveConf).mkdirs(new Path(tblPath,"colname=p1")));
- }
- public void testSetOutput() throws Exception {
- Configuration conf = new Configuration();
- Job job = new Job(conf, "test outputformat");
- Map<String, String> partitionValues = new HashMap<String, String>();
- partitionValues.put("colname", "p1");
- //null server url means local mode
- HCatTableInfo info = HCatTableInfo.getOutputTableInfo(null, null, dbName, tblName, partitionValues);
- HCatOutputFormat.setOutput(job, info);
- OutputJobInfo jobInfo = HCatOutputFormat.getJobInfo(job);
- assertNotNull(jobInfo.getTableInfo());
- assertEquals(1, jobInfo.getTableInfo().getPartitionValues().size());
- assertEquals("p1", jobInfo.getTableInfo().getPartitionValues().get("colname"));
- assertEquals(1, jobInfo.getTableSchema().getFields().size());
- assertEquals("colname", jobInfo.getTableSchema().getFields().get(0).getName());
- StorerInfo storer = jobInfo.getStorerInfo();
- assertEquals(RCFileOutputDriver.class.getName(), storer.getOutputSDClass());
- publishTest(job);
- }
- public void publishTest(Job job) throws Exception {
- OutputCommitter committer = new HCatOutputCommitter(job,null);
- committer.cleanupJob(job);
- Partition part = client.getPartition(dbName, tblName, Arrays.asList("p1"));
- assertNotNull(part);
- StorerInfo storer = InitializeInput.extractStorerInfo(part.getSd(),part.getParameters());
- assertEquals(storer.getInputSDClass(), "testInputClass");
- assertEquals(storer.getProperties().get("hcat.testarg"), "testArgValue");
- assertTrue(part.getSd().getLocation().indexOf("p1") != -1);
- }
- }