PageRenderTime 26ms CodeModel.GetById 11ms app.highlight 10ms RepoModel.GetById 1ms app.codeStats 1ms

/tags/release-0.1-rc2/hive/external/ql/src/java/org/apache/hadoop/hive/ql/io/HiveSequenceFileOutputFormat.java

#
Java | 79 lines | 34 code | 10 blank | 35 comment | 0 complexity | 493d9982b8210ec798e4924ad16f4227 MD5 | raw file
 1/**
 2 * Licensed to the Apache Software Foundation (ASF) under one
 3 * or more contributor license agreements.  See the NOTICE file
 4 * distributed with this work for additional information
 5 * regarding copyright ownership.  The ASF licenses this file
 6 * to you under the Apache License, Version 2.0 (the
 7 * "License"); you may not use this file except in compliance
 8 * with the License.  You may obtain a copy of the License at
 9 *
10 *     http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18
19package org.apache.hadoop.hive.ql.io;
20
21import java.io.IOException;
22import java.util.Properties;
23
24import org.apache.hadoop.fs.FileSystem;
25import org.apache.hadoop.fs.Path;
26import org.apache.hadoop.hive.ql.exec.Utilities;
27import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter;
28import org.apache.hadoop.io.BytesWritable;
29import org.apache.hadoop.io.SequenceFile;
30import org.apache.hadoop.io.Writable;
31import org.apache.hadoop.io.WritableComparable;
32import org.apache.hadoop.mapred.JobConf;
33import org.apache.hadoop.mapred.SequenceFileOutputFormat;
34import org.apache.hadoop.util.Progressable;
35
36/** A {@link HiveOutputFormat} that writes {@link SequenceFile}s. */
37public class HiveSequenceFileOutputFormat extends SequenceFileOutputFormat
38    implements HiveOutputFormat<WritableComparable, Writable> {
39
40  BytesWritable EMPTY_KEY = new BytesWritable();
41
42  /**
43   * create the final out file, and output an empty key as the key.
44   * 
45   * @param jc
46   *          the job configuration file
47   * @param finalOutPath
48   *          the final output file to be created
49   * @param valueClass
50   *          the value class used for create
51   * @param isCompressed
52   *          whether the content is compressed or not
53   * @param tableProperties
54   *          the tableInfo of this file's corresponding table
55   * @param progress
56   *          progress used for status report
57   * @return the RecordWriter for the output file
58   */
59  @Override
60  public RecordWriter getHiveRecordWriter(JobConf jc, Path finalOutPath,
61      Class<? extends Writable> valueClass, boolean isCompressed,
62      Properties tableProperties, Progressable progress) throws IOException {
63
64    FileSystem fs = finalOutPath.getFileSystem(jc);
65    final SequenceFile.Writer outStream = Utilities.createSequenceWriter(jc,
66        fs, finalOutPath, BytesWritable.class, valueClass, isCompressed);
67
68    return new RecordWriter() {
69      public void write(Writable r) throws IOException {
70        outStream.append(EMPTY_KEY, r);
71      }
72
73      public void close(boolean abort) throws IOException {
74        outStream.close();
75      }
76    };
77  }
78
79}