PageRenderTime 23ms CodeModel.GetById 11ms app.highlight 9ms RepoModel.GetById 1ms app.codeStats 0ms

/tags/release-0.0.0-rc0/hive/external/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java

#
Java | 117 lines | 71 code | 10 blank | 36 comment | 11 complexity | 6b2d51a855fa54d18eeee53b27cf8389 MD5 | raw file
  1/**
  2 * Licensed to the Apache Software Foundation (ASF) under one
  3 * or more contributor license agreements.  See the NOTICE file
  4 * distributed with this work for additional information
  5 * regarding copyright ownership.  The ASF licenses this file
  6 * to you under the Apache License, Version 2.0 (the
  7 * "License"); you may not use this file except in compliance
  8 * with the License.  You may obtain a copy of the License at
  9 *
 10 *     http://www.apache.org/licenses/LICENSE-2.0
 11 *
 12 * Unless required by applicable law or agreed to in writing, software
 13 * distributed under the License is distributed on an "AS IS" BASIS,
 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 15 * See the License for the specific language governing permissions and
 16 * limitations under the License.
 17 */
 18
 19package org.apache.hadoop.hive.ql.optimizer;
 20
 21import java.io.Serializable;
 22import java.util.HashSet;
 23import java.util.Map;
 24import java.util.Set;
 25import java.util.Stack;
 26
 27import org.apache.hadoop.hive.ql.exec.Operator;
 28import org.apache.hadoop.hive.ql.exec.TableScanOperator;
 29import org.apache.hadoop.hive.ql.exec.Task;
 30import org.apache.hadoop.hive.ql.exec.TaskFactory;
 31import org.apache.hadoop.hive.ql.lib.Node;
 32import org.apache.hadoop.hive.ql.lib.NodeProcessor;
 33import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
 34import org.apache.hadoop.hive.ql.metadata.Partition;
 35import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx;
 36import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.tableSpec;
 37import org.apache.hadoop.hive.ql.parse.ParseContext;
 38import org.apache.hadoop.hive.ql.parse.PrunedPartitionList;
 39import org.apache.hadoop.hive.ql.parse.QBParseInfo;
 40import org.apache.hadoop.hive.ql.parse.SemanticException;
 41import org.apache.hadoop.hive.ql.plan.MapredWork;
 42import org.apache.hadoop.hive.ql.plan.StatsWork;
 43/**
 44 * Processor for the rule - table scan.
 45 */
 46public class GenMRTableScan1 implements NodeProcessor {
 47  public GenMRTableScan1() {
 48  }
 49
 50  /**
 51   * Table Sink encountered.
 52   *
 53   * @param nd
 54   *          the table sink operator encountered
 55   * @param opProcCtx
 56   *          context
 57   */
 58  public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx opProcCtx,
 59      Object... nodeOutputs) throws SemanticException {
 60    TableScanOperator op = (TableScanOperator) nd;
 61    GenMRProcContext ctx = (GenMRProcContext) opProcCtx;
 62    ParseContext parseCtx = ctx.getParseCtx();
 63    Map<Operator<? extends Serializable>, GenMapRedCtx> mapCurrCtx = ctx.getMapCurrCtx();
 64
 65    // create a dummy MapReduce task
 66    MapredWork currWork = GenMapRedUtils.getMapRedWork(parseCtx.getConf());
 67    Task<? extends Serializable> currTask = TaskFactory.get(currWork, parseCtx.getConf());
 68    Operator<? extends Serializable> currTopOp = op;
 69    ctx.setCurrTask(currTask);
 70    ctx.setCurrTopOp(currTopOp);
 71
 72    for (String alias : parseCtx.getTopOps().keySet()) {
 73      Operator<? extends Serializable> currOp = parseCtx.getTopOps().get(alias);
 74      if (currOp == op) {
 75        String currAliasId = alias;
 76        ctx.setCurrAliasId(currAliasId);
 77        mapCurrCtx.put(op, new GenMapRedCtx(currTask, currTopOp, currAliasId));
 78
 79        QBParseInfo parseInfo = parseCtx.getQB().getParseInfo();
 80        if (parseInfo.isAnalyzeCommand()) {
 81
 82          //   ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS;
 83          // The plan consists of a simple MapRedTask followed by a StatsTask.
 84          // The MR task is just a simple TableScanOperator
 85
 86          StatsWork statsWork = new StatsWork(parseCtx.getQB().getParseInfo().getTableSpec());
 87          statsWork.setAggKey(op.getConf().getStatsAggPrefix());
 88          Task<StatsWork> statsTask = TaskFactory.get(statsWork, parseCtx.getConf());
 89          currTask.addDependentTask(statsTask);
 90          ctx.getRootTasks().add(currTask);
 91          currWork.setGatheringStats(true);
 92          // NOTE: here we should use the new partition predicate pushdown API to get a list of pruned list,
 93          // and pass it to setTaskPlan as the last parameter
 94          Set<Partition> confirmedPartns = new HashSet<Partition>();
 95          tableSpec tblSpec = parseInfo.getTableSpec();
 96          if (tblSpec.specType == tableSpec.SpecType.STATIC_PARTITION) {
 97            // static partition
 98            confirmedPartns.add(tblSpec.partHandle);
 99          } else if (tblSpec.specType == tableSpec.SpecType.DYNAMIC_PARTITION) {
100            // dynamic partition
101            confirmedPartns.addAll(tblSpec.partitions);
102          }
103          if (confirmedPartns.size() > 0) {
104            PrunedPartitionList partList = new PrunedPartitionList(confirmedPartns, new HashSet<Partition>(), null);
105            GenMapRedUtils.setTaskPlan(currAliasId, currTopOp, currWork, false, ctx, partList);
106          } else { // non-partitioned table
107            GenMapRedUtils.setTaskPlan(currAliasId, currTopOp, currWork, false, ctx);
108          }
109        }
110        return null;
111      }
112    }
113    assert false;
114    return null;
115  }
116
117}