PageRenderTime 40ms CodeModel.GetById 21ms app.highlight 16ms RepoModel.GetById 1ms app.codeStats 0ms

/tags/release-0.1-rc2/hive/external/ql/src/java/org/apache/hadoop/hive/ql/plan/MapredLocalWork.java

#
Java | 271 lines | 183 code | 53 blank | 35 comment | 16 complexity | 4624db6571b29b7aa941cf151c1946fe MD5 | raw file
  1/**
  2 * Licensed to the Apache Software Foundation (ASF) under one
  3 * or more contributor license agreements.  See the NOTICE file
  4 * distributed with this work for additional information
  5 * regarding copyright ownership.  The ASF licenses this file
  6 * to you under the Apache License, Version 2.0 (the
  7 * "License"); you may not use this file except in compliance
  8 * with the License.  You may obtain a copy of the License at
  9 *
 10 *     http://www.apache.org/licenses/LICENSE-2.0
 11 *
 12 * Unless required by applicable law or agreed to in writing, software
 13 * distributed under the License is distributed on an "AS IS" BASIS,
 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 15 * See the License for the specific language governing permissions and
 16 * limitations under the License.
 17 */
 18
 19package org.apache.hadoop.hive.ql.plan;
 20
 21import java.io.File;
 22import java.io.Serializable;
 23import java.util.ArrayList;
 24import java.util.Iterator;
 25import java.util.LinkedHashMap;
 26import java.util.List;
 27import java.util.Map.Entry;
 28
 29import org.apache.hadoop.fs.Path;
 30import org.apache.hadoop.hive.ql.exec.BucketMatcher;
 31import org.apache.hadoop.hive.ql.exec.Operator;
 32
 33/**
 34 * MapredLocalWork.
 35 *
 36 */
 37@Explain(displayName = "Map Reduce Local Work")
 38public class MapredLocalWork implements Serializable {
 39  private static final long serialVersionUID = 1L;
 40
 41  private LinkedHashMap<String, Operator<? extends Serializable>> aliasToWork;
 42  private LinkedHashMap<String, FetchWork> aliasToFetchWork;
 43  private boolean inputFileChangeSensitive;
 44  private BucketMapJoinContext bucketMapjoinContext;
 45  private String tmpFileURI;
 46  private String stageID;
 47
 48  private List<Operator<? extends Serializable>> dummyParentOp ;
 49
 50  public MapredLocalWork() {
 51
 52  }
 53
 54  public MapredLocalWork(
 55      final LinkedHashMap<String, Operator<? extends Serializable>> aliasToWork,
 56      final LinkedHashMap<String, FetchWork> aliasToFetchWork) {
 57    this.aliasToWork = aliasToWork;
 58    this.aliasToFetchWork = aliasToFetchWork;
 59
 60  }
 61
 62  public MapredLocalWork(MapredLocalWork clone){
 63    this.tmpFileURI = clone.tmpFileURI;
 64    this.inputFileChangeSensitive=clone.inputFileChangeSensitive;
 65
 66  }
 67
 68
 69  public void setDummyParentOp(List<Operator<? extends Serializable>> op){
 70    this.dummyParentOp=op;
 71  }
 72
 73
 74  public List<Operator<? extends Serializable>> getDummyParentOp(){
 75    return this.dummyParentOp;
 76  }
 77
 78
 79  @Explain(displayName = "Alias -> Map Local Operator Tree")
 80  public LinkedHashMap<String, Operator<? extends Serializable>> getAliasToWork() {
 81    return aliasToWork;
 82  }
 83
 84  public String getStageID() {
 85    return stageID;
 86  }
 87
 88  public void setStageID(String stageID) {
 89    this.stageID = stageID;
 90  }
 91
 92  public void setAliasToWork(
 93      final LinkedHashMap<String, Operator<? extends Serializable>> aliasToWork) {
 94    this.aliasToWork = aliasToWork;
 95  }
 96
 97  /**
 98   * @return the aliasToFetchWork
 99   */
100  @Explain(displayName = "Alias -> Map Local Tables")
101  public LinkedHashMap<String, FetchWork> getAliasToFetchWork() {
102    return aliasToFetchWork;
103  }
104
105  /**
106   * @param aliasToFetchWork
107   *          the aliasToFetchWork to set
108   */
109  public void setAliasToFetchWork(
110      final LinkedHashMap<String, FetchWork> aliasToFetchWork) {
111    this.aliasToFetchWork = aliasToFetchWork;
112  }
113
114  public boolean getInputFileChangeSensitive() {
115    return inputFileChangeSensitive;
116  }
117
118  public void setInputFileChangeSensitive(boolean inputFileChangeSensitive) {
119    this.inputFileChangeSensitive = inputFileChangeSensitive;
120  }
121
122
123
124  public void deriveExplainAttributes() {
125    if (bucketMapjoinContext != null) {
126      bucketMapjoinContext.deriveBucketMapJoinMapping();
127    }
128    for (FetchWork fetchWork : aliasToFetchWork.values()) {
129      if (fetchWork.getTblDesc() == null) {
130        continue;
131      }
132      PlanUtils.configureTableJobPropertiesForStorageHandler(
133        fetchWork.getTblDesc());
134    }
135  }
136
137  @Explain(displayName = "Bucket Mapjoin Context", normalExplain = false)
138  public BucketMapJoinContext getBucketMapjoinContext() {
139    return bucketMapjoinContext;
140  }
141
142  public void setBucketMapjoinContext(BucketMapJoinContext bucketMapjoinContext) {
143    this.bucketMapjoinContext = bucketMapjoinContext;
144  }
145
146  public void setTmpFileURI(String tmpFileURI) {
147    this.tmpFileURI = tmpFileURI;
148  }
149
150  public String getTmpFileURI() {
151    return tmpFileURI;
152  }
153
154  public static class BucketMapJoinContext implements Serializable {
155
156    private static final long serialVersionUID = 1L;
157
158    // used for bucket map join
159    private LinkedHashMap<String, LinkedHashMap<String, ArrayList<String>>> aliasBucketFileNameMapping;
160    private String mapJoinBigTableAlias;
161    private Class<? extends BucketMatcher> bucketMatcherClass;
162
163    private LinkedHashMap<String, LinkedHashMap<String, ArrayList<String>>> aliasBucketBaseFileNameMapping;
164    private LinkedHashMap<String, Integer> bucketFileNameMapping;
165
166    public void setMapJoinBigTableAlias(String bigTableAlias) {
167      this.mapJoinBigTableAlias = bigTableAlias;
168    }
169
170
171    public void deriveBucketMapJoinMapping() {
172      if (aliasBucketFileNameMapping != null) {
173        Iterator<Entry<String, LinkedHashMap<String, ArrayList<String>>>> iter =
174          aliasBucketFileNameMapping.entrySet().iterator();
175        aliasBucketBaseFileNameMapping = new LinkedHashMap<String, LinkedHashMap<String, ArrayList<String>>>();
176
177        while (iter.hasNext()) {
178          Entry<String, LinkedHashMap<String, ArrayList<String>>> old = iter.next();
179
180          LinkedHashMap<String, ArrayList<String>> newBucketBaseFileNameMapping = new LinkedHashMap<String, ArrayList<String>>();
181          Iterator<Entry<String, ArrayList<String>>> oldAliasFileNameMappingIter = old.getValue().entrySet().iterator();
182          while (oldAliasFileNameMappingIter.hasNext()) {
183            //For a give table and its bucket full file path list, only keep the base file name (remove file path etc).
184            //And put the new list into the new mapping.
185            Entry<String, ArrayList<String>> oldTableBucketFileNames =  oldAliasFileNameMappingIter.next();
186            ArrayList<String> oldTableBucketNames = oldTableBucketFileNames.getValue();
187            ArrayList<String> newTableBucketFileBaseName = new ArrayList<String> (oldTableBucketNames.size());
188            //for each bucket file, only keep its base files and store into a new list.
189            if (oldTableBucketNames != null) {
190              for (String bucketFName : oldTableBucketNames) {
191                newTableBucketFileBaseName.add(getBaseFileName(bucketFName));
192              }
193            }
194            String bigTblBucketFileName = getBaseFileName(oldTableBucketFileNames.getKey());
195            if(newBucketBaseFileNameMapping.containsKey(bigTblBucketFileName)) {
196              String fullPath = oldTableBucketFileNames.getKey();
197              String dir = getBaseFileName(fullPath.substring(0, fullPath.lastIndexOf(bigTblBucketFileName)));
198              bigTblBucketFileName = dir + File.separator + bigTblBucketFileName;
199            }
200            //put the new mapping
201            newBucketBaseFileNameMapping.put(bigTblBucketFileName, newTableBucketFileBaseName);
202          }
203          String tableAlias = old.getKey();
204          aliasBucketBaseFileNameMapping.put(tableAlias, newBucketBaseFileNameMapping);
205        }
206      }
207    }
208
209    private String getBaseFileName (String path) {
210      try {
211	return ((new Path(path)).getName());
212      } catch (Exception ex) {
213        // This could be due to either URI syntax error or File constructor
214        // illegal arg; we don't really care which one it is.
215        return path;
216      }
217    }
218
219    public String getMapJoinBigTableAlias() {
220      return mapJoinBigTableAlias;
221    }
222
223    public Class<? extends BucketMatcher> getBucketMatcherClass() {
224      return bucketMatcherClass;
225    }
226
227    public void setBucketMatcherClass(
228        Class<? extends BucketMatcher> bucketMatcherClass) {
229      this.bucketMatcherClass = bucketMatcherClass;
230    }
231
232    @Explain(displayName = "Alias Bucket File Name Mapping", normalExplain = false)
233    public LinkedHashMap<String, LinkedHashMap<String, ArrayList<String>>> getAliasBucketFileNameMapping() {
234      return aliasBucketFileNameMapping;
235    }
236
237    public void setAliasBucketFileNameMapping(
238        LinkedHashMap<String, LinkedHashMap<String, ArrayList<String>>> aliasBucketFileNameMapping) {
239      this.aliasBucketFileNameMapping = aliasBucketFileNameMapping;
240    }
241
242    @Override
243    public String toString() {
244      if (aliasBucketFileNameMapping != null) {
245        return "Mapping:" + aliasBucketFileNameMapping.toString();
246      } else {
247        return "";
248      }
249    }
250
251    @Explain(displayName = "Alias Bucket Base File Name Mapping", normalExplain = false)
252    public LinkedHashMap<String, LinkedHashMap<String, ArrayList<String>>> getAliasBucketBaseFileNameMapping() {
253      return aliasBucketBaseFileNameMapping;
254    }
255
256    public void setAliasBucketBaseFileNameMapping(
257        LinkedHashMap<String, LinkedHashMap<String, ArrayList<String>>> aliasBucketBaseFileNameMapping) {
258      this.aliasBucketBaseFileNameMapping = aliasBucketBaseFileNameMapping;
259    }
260
261    @Explain(displayName = "Alias Bucket Output File Name Mapping", normalExplain = false)
262    public LinkedHashMap<String, Integer> getBucketFileNameMapping() {
263      return bucketFileNameMapping;
264    }
265
266    public void setBucketFileNameMapping(LinkedHashMap<String, Integer> bucketFileNameMapping) {
267      this.bucketFileNameMapping = bucketFileNameMapping;
268    }
269
270  }
271}