PageRenderTime 40ms CodeModel.GetById 34ms app.highlight 4ms RepoModel.GetById 1ms app.codeStats 0ms

/tags/release-0.2.0-rc0/src/java/org/apache/hcatalog/pig/HCatBaseLoader.java

#
Java | 117 lines | 70 code | 19 blank | 28 comment | 0 complexity | 7501871d5894c53a69e77cfa5175c538 MD5 | raw file
  1/*
  2 * Licensed to the Apache Software Foundation (ASF) under one
  3 * or more contributor license agreements.  See the NOTICE file
  4 * distributed with this work for additional information
  5 * regarding copyright ownership.  The ASF licenses this file
  6 * to you under the Apache License, Version 2.0 (the
  7 * "License"); you may not use this file except in compliance
  8 * with the License.  You may obtain a copy of the License at
  9 *
 10 *     http://www.apache.org/licenses/LICENSE-2.0
 11 *
 12 * Unless required by applicable law or agreed to in writing, software
 13 * distributed under the License is distributed on an "AS IS" BASIS,
 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 15 * See the License for the specific language governing permissions and
 16 * limitations under the License.
 17 */
 18package org.apache.hcatalog.pig;
 19
 20import java.io.IOException;
 21import java.util.Arrays;
 22import java.util.List;
 23import java.util.Properties;
 24
 25import org.apache.hadoop.mapreduce.Job;
 26import org.apache.hadoop.mapreduce.RecordReader;
 27import org.apache.hcatalog.data.HCatRecord;
 28import org.apache.hcatalog.data.schema.HCatSchema;
 29import org.apache.pig.LoadFunc;
 30import org.apache.pig.LoadMetadata;
 31import org.apache.pig.LoadPushDown;
 32import org.apache.pig.PigException;
 33import org.apache.pig.ResourceStatistics;
 34import org.apache.pig.backend.executionengine.ExecException;
 35import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit;
 36import org.apache.pig.data.Tuple;
 37import org.apache.pig.impl.logicalLayer.FrontendException;
 38import org.apache.pig.impl.util.UDFContext;
 39
 40/**
 41 * Base class for HCatLoader and HCatEximLoader
 42 */
 43
 44public abstract class HCatBaseLoader extends LoadFunc implements LoadMetadata, LoadPushDown {
 45
 46  protected static final String PRUNE_PROJECTION_INFO = "prune.projection.info";
 47
 48  private RecordReader<?, ?> reader;
 49  protected String signature;
 50
 51  HCatSchema outputSchema = null;
 52
 53
 54  @Override
 55  public Tuple getNext() throws IOException {
 56    try {
 57      HCatRecord hr =  (HCatRecord) (reader.nextKeyValue() ? reader.getCurrentValue() : null);
 58      Tuple t = PigHCatUtil.transformToTuple(hr,outputSchema);
 59      // TODO : we were discussing an iter interface, and also a LazyTuple
 60      // change this when plans for that solidifies.
 61      return t;
 62    } catch (ExecException e) {
 63      int errCode = 6018;
 64      String errMsg = "Error while reading input";
 65      throw new ExecException(errMsg, errCode,
 66          PigException.REMOTE_ENVIRONMENT, e);
 67    } catch (Exception eOther){
 68      int errCode = 6018;
 69      String errMsg = "Error converting read value to tuple";
 70      throw new ExecException(errMsg, errCode,
 71          PigException.REMOTE_ENVIRONMENT, eOther);
 72    }
 73
 74  }
 75
 76  @Override
 77  public void prepareToRead(RecordReader reader, PigSplit arg1) throws IOException {
 78    this.reader = reader;
 79  }
 80
 81  @Override
 82  public ResourceStatistics getStatistics(String location, Job job) throws IOException {
 83    // statistics not implemented currently
 84    return null;
 85  }
 86
 87  @Override
 88  public List<OperatorSet> getFeatures() {
 89    return Arrays.asList(LoadPushDown.OperatorSet.PROJECTION);
 90  }
 91
 92  @Override
 93  public RequiredFieldResponse pushProjection(RequiredFieldList requiredFieldsInfo) throws FrontendException {
 94    // Store the required fields information in the UDFContext so that we
 95    // can retrieve it later.
 96    storeInUDFContext(signature, PRUNE_PROJECTION_INFO, requiredFieldsInfo);
 97
 98    // HCat will always prune columns based on what we ask of it - so the
 99    // response is true
100    return new RequiredFieldResponse(true);
101  }
102
103  @Override
104  public void setUDFContextSignature(String signature) {
105    this.signature = signature;
106  }
107
108
109  // helper methods
110  protected void storeInUDFContext(String signature, String key, Object value) {
111    UDFContext udfContext = UDFContext.getUDFContext();
112    Properties props = udfContext.getUDFProperties(
113        this.getClass(), new String[] {signature});
114    props.put(key, value);
115  }
116
117}