PageRenderTime 40ms CodeModel.GetById 12ms app.highlight 22ms RepoModel.GetById 1ms app.codeStats 0ms

/tags/release-0.1-rc2/hive/external/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileRecordReader.java

#
Java | 139 lines | 88 code | 23 blank | 28 comment | 8 complexity | 2dda00a07d8ccf3d141805d764a8d4ff MD5 | raw file
  1/**
  2 * Licensed to the Apache Software Foundation (ASF) under one
  3 * or more contributor license agreements.  See the NOTICE file
  4 * distributed with this work for additional information
  5 * regarding copyright ownership.  The ASF licenses this file
  6 * to you under the Apache License, Version 2.0 (the
  7 * "License"); you may not use this file except in compliance
  8 * with the License.  You may obtain a copy of the License at
  9 *
 10 *     http://www.apache.org/licenses/LICENSE-2.0
 11 *
 12 * Unless required by applicable law or agreed to in writing, software
 13 * distributed under the License is distributed on an "AS IS" BASIS,
 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 15 * See the License for the specific language governing permissions and
 16 * limitations under the License.
 17 */
 18
 19package org.apache.hadoop.hive.ql.io;
 20
 21import java.io.IOException;
 22
 23import org.apache.hadoop.conf.Configuration;
 24import org.apache.hadoop.fs.FileSystem;
 25import org.apache.hadoop.fs.Path;
 26import org.apache.hadoop.hive.ql.io.RCFile.Reader;
 27import org.apache.hadoop.hive.serde2.columnar.BytesRefArrayWritable;
 28import org.apache.hadoop.io.LongWritable;
 29import org.apache.hadoop.mapred.FileSplit;
 30import org.apache.hadoop.mapred.RecordReader;
 31import org.apache.hadoop.util.ReflectionUtils;
 32
 33/**
 34 * RCFileRecordReader.
 35 *
 36 * @param <K>
 37 * @param <V>
 38 */
 39public class RCFileRecordReader<K extends LongWritable, V extends BytesRefArrayWritable>
 40    implements RecordReader<LongWritable, BytesRefArrayWritable> {
 41
 42  private final Reader in;
 43  private final long start;
 44  private final long end;
 45  private boolean more = true;
 46  protected Configuration conf;
 47
 48  public RCFileRecordReader(Configuration conf, FileSplit split)
 49      throws IOException {
 50    Path path = split.getPath();
 51    FileSystem fs = path.getFileSystem(conf);
 52    this.in = new RCFile.Reader(fs, path, conf);
 53    this.end = split.getStart() + split.getLength();
 54    this.conf = conf;
 55
 56    if (split.getStart() > in.getPosition()) {
 57      in.sync(split.getStart()); // sync to start
 58    }
 59
 60    this.start = in.getPosition();
 61    more = start < end;
 62  }
 63
 64  public Class<?> getKeyClass() {
 65    return LongWritable.class;
 66  }
 67
 68  public Class<?> getValueClass() {
 69    return BytesRefArrayWritable.class;
 70  }
 71
 72  public LongWritable createKey() {
 73    return (LongWritable) ReflectionUtils.newInstance(getKeyClass(), conf);
 74  }
 75
 76  public BytesRefArrayWritable createValue() {
 77    return (BytesRefArrayWritable) ReflectionUtils.newInstance(getValueClass(),
 78        conf);
 79  }
 80
 81  @Override
 82  public boolean next(LongWritable key, BytesRefArrayWritable value)
 83      throws IOException {
 84
 85    more = next(key);
 86
 87    if (more) {
 88      in.getCurrentRow(value);
 89    }
 90    return more;
 91  }
 92
 93  protected boolean next(LongWritable key) throws IOException {
 94    if (!more) {
 95      return false;
 96    }
 97
 98    more = in.next(key);
 99    if (!more) {
100      return false;
101    }
102
103    long lastSeenSyncPos = in.lastSeenSyncPos();
104    if (lastSeenSyncPos >= end) {
105      more = false;
106      return more;
107    }
108    return more;
109  }
110
111  /**
112   * Return the progress within the input split.
113   *
114   * @return 0.0 to 1.0 of the input byte range
115   */
116  public float getProgress() throws IOException {
117    if (end == start) {
118      return 0.0f;
119    } else {
120      return Math.min(1.0f, (in.getPosition() - start) / (float) (end - start));
121    }
122  }
123
124  public long getPos() throws IOException {
125    return in.getPosition();
126  }
127
128  protected void seek(long pos) throws IOException {
129    in.seek(pos);
130  }
131
132  public long getStart() {
133    return start;
134  }
135
136  public void close() throws IOException {
137    in.close();
138  }
139}