PageRenderTime 28ms CodeModel.GetById 15ms app.highlight 10ms RepoModel.GetById 1ms app.codeStats 0ms

/tags/release-0.1-rc2/hive/external/ql/src/java/org/apache/hadoop/hive/ql/io/HiveIgnoreKeyTextOutputFormat.java

#
Java | 129 lines | 73 code | 15 blank | 41 comment | 2 complexity | 0ee675d659d7865d73890695c52e342a MD5 | raw file
  1/**
  2 * Licensed to the Apache Software Foundation (ASF) under one
  3 * or more contributor license agreements.  See the NOTICE file
  4 * distributed with this work for additional information
  5 * regarding copyright ownership.  The ASF licenses this file
  6 * to you under the Apache License, Version 2.0 (the
  7 * "License"); you may not use this file except in compliance
  8 * with the License.  You may obtain a copy of the License at
  9 *
 10 *     http://www.apache.org/licenses/LICENSE-2.0
 11 *
 12 * Unless required by applicable law or agreed to in writing, software
 13 * distributed under the License is distributed on an "AS IS" BASIS,
 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 15 * See the License for the specific language governing permissions and
 16 * limitations under the License.
 17 */
 18
 19package org.apache.hadoop.hive.ql.io;
 20
 21import java.io.IOException;
 22import java.io.OutputStream;
 23import java.util.Properties;
 24
 25import org.apache.hadoop.fs.FileSystem;
 26import org.apache.hadoop.fs.Path;
 27import org.apache.hadoop.hive.ql.exec.Utilities;
 28import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter;
 29import org.apache.hadoop.hive.serde.Constants;
 30import org.apache.hadoop.io.BytesWritable;
 31import org.apache.hadoop.io.Text;
 32import org.apache.hadoop.io.Writable;
 33import org.apache.hadoop.io.WritableComparable;
 34import org.apache.hadoop.mapred.JobConf;
 35import org.apache.hadoop.mapred.Reporter;
 36import org.apache.hadoop.mapred.TextOutputFormat;
 37import org.apache.hadoop.util.Progressable;
 38
 39/**
 40 * HiveIgnoreKeyTextOutputFormat replaces key with null before feeding the <key,
 41 * value> to TextOutputFormat.RecordWriter.
 42 * 
 43 */
 44public class HiveIgnoreKeyTextOutputFormat<K extends WritableComparable, V extends Writable>
 45    extends TextOutputFormat<K, V> implements HiveOutputFormat<K, V> {
 46
 47  /**
 48   * create the final out file, and output row by row. After one row is
 49   * appended, a configured row separator is appended
 50   * 
 51   * @param jc
 52   *          the job configuration file
 53   * @param outPath
 54   *          the final output file to be created
 55   * @param valueClass
 56   *          the value class used for create
 57   * @param isCompressed
 58   *          whether the content is compressed or not
 59   * @param tableProperties
 60   *          the tableProperties of this file's corresponding table
 61   * @param progress
 62   *          progress used for status report
 63   * @return the RecordWriter
 64   */
 65  @Override
 66  public RecordWriter getHiveRecordWriter(JobConf jc, Path outPath,
 67      Class<? extends Writable> valueClass, boolean isCompressed,
 68      Properties tableProperties, Progressable progress) throws IOException {
 69    int rowSeparator = 0;
 70    String rowSeparatorString = tableProperties.getProperty(
 71        Constants.LINE_DELIM, "\n");
 72    try {
 73      rowSeparator = Byte.parseByte(rowSeparatorString);
 74    } catch (NumberFormatException e) {
 75      rowSeparator = rowSeparatorString.charAt(0);
 76    }
 77
 78    final int finalRowSeparator = rowSeparator;
 79    FileSystem fs = outPath.getFileSystem(jc);
 80    final OutputStream outStream = Utilities.createCompressedStream(jc, fs
 81        .create(outPath), isCompressed);
 82    return new RecordWriter() {
 83      public void write(Writable r) throws IOException {
 84        if (r instanceof Text) {
 85          Text tr = (Text) r;
 86          outStream.write(tr.getBytes(), 0, tr.getLength());
 87          outStream.write(finalRowSeparator);
 88        } else {
 89          // DynamicSerDe always writes out BytesWritable
 90          BytesWritable bw = (BytesWritable) r;
 91          outStream.write(bw.get(), 0, bw.getSize());
 92          outStream.write(finalRowSeparator);
 93        }
 94      }
 95
 96      public void close(boolean abort) throws IOException {
 97        outStream.close();
 98      }
 99    };
100  }
101
102  protected static class IgnoreKeyWriter<K extends WritableComparable, V extends Writable>
103      implements org.apache.hadoop.mapred.RecordWriter<K, V> {
104
105    private final org.apache.hadoop.mapred.RecordWriter<K, V> mWriter;
106
107    public IgnoreKeyWriter(org.apache.hadoop.mapred.RecordWriter<K, V> writer) {
108      this.mWriter = writer;
109    }
110
111    public synchronized void write(K key, V value) throws IOException {
112      this.mWriter.write(null, value);
113    }
114
115    public void close(Reporter reporter) throws IOException {
116      this.mWriter.close(reporter);
117    }
118  }
119
120  @Override
121  public org.apache.hadoop.mapred.RecordWriter<K, V> getRecordWriter(
122      FileSystem ignored, JobConf job, String name, Progressable progress)
123      throws IOException {
124
125    return new IgnoreKeyWriter<K, V>(super.getRecordWriter(ignored, job, name,
126        progress));
127  }
128
129}