PageRenderTime 22ms CodeModel.GetById 11ms app.highlight 8ms RepoModel.GetById 1ms app.codeStats 0ms

/tags/release-0.0.0-rc0/hive/external/ql/src/java/org/apache/hadoop/hive/ql/lib/TaskGraphWalker.java

#
Java | 204 lines | 120 code | 23 blank | 61 comment | 25 complexity | 82eea22664343523cedf347f1c91eb72 MD5 | raw file
  1/**
  2 * Licensed to the Apache Software Foundation (ASF) under one
  3 * or more contributor license agreements.  See the NOTICE file
  4 * distributed with this work for additional information
  5 * regarding copyright ownership.  The ASF licenses this file
  6 * to you under the Apache License, Version 2.0 (the
  7 * "License"); you may not use this file except in compliance
  8 * with the License.  You may obtain a copy of the License at
  9 *
 10 *     http://www.apache.org/licenses/LICENSE-2.0
 11 *
 12 * Unless required by applicable law or agreed to in writing, software
 13 * distributed under the License is distributed on an "AS IS" BASIS,
 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 15 * See the License for the specific language governing permissions and
 16 * limitations under the License.
 17 */
 18
 19package org.apache.hadoop.hive.ql.lib;
 20
 21import java.io.Serializable;
 22import java.util.ArrayList;
 23import java.util.Collection;
 24import java.util.HashMap;
 25import java.util.HashSet;
 26import java.util.List;
 27import java.util.Set;
 28import java.util.Stack;
 29
 30import org.apache.hadoop.hive.ql.exec.ConditionalTask;
 31import org.apache.hadoop.hive.ql.exec.Task;
 32import org.apache.hadoop.hive.ql.parse.SemanticException;
 33
 34/**
 35 * base class for operator graph walker this class takes list of starting ops
 36 * and walks them one by one. it maintains list of walked operators
 37 * (dispatchedList) and a list of operators that are discovered but not yet
 38 * dispatched
 39 */
 40public class TaskGraphWalker implements GraphWalker {
 41
 42
 43  public class TaskGraphWalkerContext{
 44    private final HashMap<Node, Object> reMap;
 45
 46    public TaskGraphWalkerContext(HashMap<Node, Object> reMap){
 47      this.reMap = reMap;
 48    }
 49    public void addToDispatchList(Node dispatchedObj){
 50      if(dispatchedObj != null) {
 51        retMap.put(dispatchedObj, null);
 52      }
 53    }
 54  }
 55
 56  protected Stack<Node> opStack;
 57  private final List<Node> toWalk = new ArrayList<Node>();
 58  private final HashMap<Node, Object> retMap = new HashMap<Node, Object>();
 59  private final Dispatcher dispatcher;
 60  private final  TaskGraphWalkerContext walkerCtx;
 61
 62  /**
 63   * Constructor.
 64   *
 65   * @param disp
 66   *          dispatcher to call for each op encountered
 67   */
 68  public TaskGraphWalker(Dispatcher disp) {
 69    dispatcher = disp;
 70    opStack = new Stack<Node>();
 71    walkerCtx = new TaskGraphWalkerContext(retMap);
 72  }
 73
 74  /**
 75   * @return the toWalk
 76   */
 77  public List<Node> getToWalk() {
 78    return toWalk;
 79  }
 80
 81  /**
 82   * @return the doneList
 83   */
 84  public Set<Node> getDispatchedList() {
 85    return retMap.keySet();
 86  }
 87
 88  /**
 89   * Dispatch the current operator.
 90   *
 91   * @param nd
 92   *          node being walked
 93   * @param ndStack
 94   *          stack of nodes encountered
 95   * @throws SemanticException
 96   */
 97  public void dispatch(Node nd, Stack<Node> ndStack,TaskGraphWalkerContext walkerCtx) throws SemanticException {
 98    Object[] nodeOutputs = null;
 99    if (nd.getChildren() != null) {
100      nodeOutputs = new Object[nd.getChildren().size()+1];
101      nodeOutputs[0] = walkerCtx;
102      int i = 1;
103      for (Node child : nd.getChildren()) {
104        nodeOutputs[i++] = retMap.get(child);
105      }
106    }else{
107      nodeOutputs = new Object[1];
108      nodeOutputs[0] = walkerCtx;
109    }
110
111    Object retVal = dispatcher.dispatch(nd, ndStack, nodeOutputs);
112    retMap.put(nd, retVal);
113  }
114
115  public void dispatch(Node nd, Stack<Node> ndStack) throws SemanticException {
116    Object[] nodeOutputs = null;
117    if (nd.getChildren() != null) {
118      nodeOutputs = new Object[nd.getChildren().size()];
119      int i = 1;
120      for (Node child : nd.getChildren()) {
121        nodeOutputs[i++] = retMap.get(child);
122      }
123    }
124
125    Object retVal = dispatcher.dispatch(nd, ndStack, nodeOutputs);
126    retMap.put(nd, retVal);
127  }
128
129  /**
130   * starting point for walking.
131   *
132   * @throws SemanticException
133   */
134  public void startWalking(Collection<Node> startNodes,
135      HashMap<Node, Object> nodeOutput) throws SemanticException {
136    toWalk.addAll(startNodes);
137    while (toWalk.size() > 0) {
138      Node nd = toWalk.remove(0);
139      walk(nd);
140      if (nodeOutput != null) {
141        nodeOutput.put(nd, retMap.get(nd));
142      }
143    }
144  }
145
146  /**
147   * walk the current operator and its descendants.
148   *
149   * @param nd
150   *          current operator in the graph
151   * @throws SemanticException
152   */
153  public void walk(Node nd) throws SemanticException {
154      if(!(nd instanceof Task)){
155        throw new SemanticException("Task Graph Walker only walks for Task Graph");
156      }
157
158      if (getDispatchedList().contains(nd)) {
159        return;
160      }
161      if (opStack.empty() || nd != opStack.peek()) {
162        opStack.push(nd);
163      }
164
165      List<Task<? extends Serializable>> nextTaskList = null;
166      Set<Task<? extends Serializable>> nextTaskSet = new HashSet<Task<? extends Serializable>>();
167      List<Task<? extends Serializable>> taskListInConditionalTask = null;
168
169
170      if(nd instanceof ConditionalTask ){
171        //for conditional task, next task list should return the children tasks of each task, which
172        //is contained in the conditional task.
173        taskListInConditionalTask = ((ConditionalTask) nd).getListTasks();
174        for(Task<? extends Serializable> tsk: taskListInConditionalTask){
175          List<Task<? extends Serializable>> childTask = tsk.getChildTasks();
176          if(childTask != null){
177            nextTaskSet.addAll(tsk.getChildTasks());
178          }
179        }
180        //convert the set into list
181        if(nextTaskSet.size()>0){
182          nextTaskList = new ArrayList<Task<? extends Serializable>>();
183          for(Task<? extends Serializable> tsk:nextTaskSet ){
184            nextTaskList.add(tsk);
185          }
186        }
187      }else{
188        //for other tasks, just return its children tasks
189        nextTaskList = ((Task<? extends Serializable>)nd).getChildTasks();
190      }
191
192      if ((nextTaskList == null)
193          || getDispatchedList().containsAll(nextTaskList)) {
194        dispatch(nd, opStack,this.walkerCtx);
195        opStack.pop();
196        return;
197      }
198      // add children, self to the front of the queue in that order
199      getToWalk().add(0, nd);
200      getToWalk().removeAll(nextTaskList);
201      getToWalk().addAll(0, nextTaskList);
202
203  }
204}