PageRenderTime 29ms CodeModel.GetById 13ms app.highlight 12ms RepoModel.GetById 2ms app.codeStats 0ms

/tags/release-0.0.0-rc0/hive/external/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java

#
Java | 250 lines | 147 code | 37 blank | 66 comment | 2 complexity | f1bfcf84fc17b35b2a5894b75665f2bf MD5 | raw file
  1/**
  2 * Licensed to the Apache Software Foundation (ASF) under one
  3 * or more contributor license agreements.  See the NOTICE file
  4 * distributed with this work for additional information
  5 * regarding copyright ownership.  The ASF licenses this file
  6 * to you under the Apache License, Version 2.0 (the
  7 * "License"); you may not use this file except in compliance
  8 * with the License.  You may obtain a copy of the License at
  9 *
 10 *     http://www.apache.org/licenses/LICENSE-2.0
 11 *
 12 * Unless required by applicable law or agreed to in writing, software
 13 * distributed under the License is distributed on an "AS IS" BASIS,
 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 15 * See the License for the specific language governing permissions and
 16 * limitations under the License.
 17 */
 18
 19package org.apache.hadoop.hive.ql.plan;
 20
 21import java.io.Serializable;
 22import java.util.ArrayList;
 23
 24import org.apache.hadoop.fs.Path;
 25
 26/**
 27 * FileSinkDesc.
 28 *
 29 */
 30@Explain(displayName = "File Output Operator")
 31public class FileSinkDesc implements Serializable {
 32  private static final long serialVersionUID = 1L;
 33  private String dirName;
 34  // normally statsKeyPref will be the same as dirName, but the latter
 35  // could be changed in local execution optimization
 36  private String statsKeyPref;
 37  private TableDesc tableInfo;
 38  private boolean compressed;
 39  private int destTableId;
 40  private String compressCodec;
 41  private String compressType;
 42  private boolean multiFileSpray;
 43  private int     totalFiles;
 44  private ArrayList<ExprNodeDesc> partitionCols;
 45  private int     numFiles;
 46  private DynamicPartitionCtx dpCtx;
 47  private String staticSpec; // static partition spec ends with a '/'
 48  private boolean gatherStats;
 49
 50  public FileSinkDesc() {
 51  }
 52
 53  public FileSinkDesc(final String dirName, final TableDesc tableInfo,
 54      final boolean compressed, final int destTableId, final boolean multiFileSpray,
 55      final int numFiles, final int totalFiles, final ArrayList<ExprNodeDesc> partitionCols,
 56      final DynamicPartitionCtx dpCtx) {
 57
 58    this.dirName = dirName;
 59    this.tableInfo = tableInfo;
 60    this.compressed = compressed;
 61    this.destTableId = destTableId;
 62    this.multiFileSpray = multiFileSpray;
 63    this.numFiles = numFiles;
 64    this.totalFiles = totalFiles;
 65    this.partitionCols = partitionCols;
 66    this.dpCtx = dpCtx;
 67  }
 68
 69  public FileSinkDesc(final String dirName, final TableDesc tableInfo,
 70      final boolean compressed) {
 71
 72    this.dirName = dirName;
 73    this.tableInfo = tableInfo;
 74    this.compressed = compressed;
 75    destTableId = 0;
 76    this.multiFileSpray = false;
 77    this.numFiles = 1;
 78    this.totalFiles = 1;
 79    this.partitionCols = null;
 80  }
 81
 82  @Explain(displayName = "directory", normalExplain = false)
 83  public String getDirName() {
 84    return dirName;
 85  }
 86
 87  public void setDirName(final String dirName) {
 88    this.dirName = dirName;
 89  }
 90
 91  @Explain(displayName = "table")
 92  public TableDesc getTableInfo() {
 93    return tableInfo;
 94  }
 95
 96  public void setTableInfo(final TableDesc tableInfo) {
 97    this.tableInfo = tableInfo;
 98  }
 99
100  @Explain(displayName = "compressed")
101  public boolean getCompressed() {
102    return compressed;
103  }
104
105  public void setCompressed(boolean compressed) {
106    this.compressed = compressed;
107  }
108
109  @Explain(displayName = "GlobalTableId")
110  public int getDestTableId() {
111    return destTableId;
112  }
113
114  public void setDestTableId(int destTableId) {
115    this.destTableId = destTableId;
116  }
117
118  public String getCompressCodec() {
119    return compressCodec;
120  }
121
122  public void setCompressCodec(String intermediateCompressorCodec) {
123    compressCodec = intermediateCompressorCodec;
124  }
125
126  public String getCompressType() {
127    return compressType;
128  }
129
130  public void setCompressType(String intermediateCompressType) {
131    compressType = intermediateCompressType;
132  }
133
134  /**
135   * @return the multiFileSpray
136   */
137  @Explain(displayName = "MultiFileSpray", normalExplain = false)
138  public boolean isMultiFileSpray() {
139    return multiFileSpray;
140  }
141
142  /**
143   * @param multiFileSpray the multiFileSpray to set
144   */
145  public void setMultiFileSpray(boolean multiFileSpray) {
146    this.multiFileSpray = multiFileSpray;
147  }
148
149  /**
150   * @return the totalFiles
151   */
152  @Explain(displayName = "TotalFiles", normalExplain = false)
153  public int getTotalFiles() {
154    return totalFiles;
155  }
156
157  /**
158   * @param totalFiles the totalFiles to set
159   */
160  public void setTotalFiles(int totalFiles) {
161    this.totalFiles = totalFiles;
162  }
163
164  /**
165   * @return the partitionCols
166   */
167  public ArrayList<ExprNodeDesc> getPartitionCols() {
168    return partitionCols;
169  }
170
171  /**
172   * @param partitionCols the partitionCols to set
173   */
174  public void setPartitionCols(ArrayList<ExprNodeDesc> partitionCols) {
175    this.partitionCols = partitionCols;
176  }
177
178  /**
179   * @return the numFiles
180   */
181  @Explain(displayName = "NumFilesPerFileSink", normalExplain = false)
182  public int getNumFiles() {
183    return numFiles;
184  }
185
186  /**
187   * @param numFiles the numFiles to set
188   */
189  public void setNumFiles(int numFiles) {
190    this.numFiles = numFiles;
191  }
192
193  public void setDynPartCtx(DynamicPartitionCtx dpc) {
194    this.dpCtx = dpc;
195  }
196
197  public DynamicPartitionCtx getDynPartCtx() {
198    return this.dpCtx;
199  }
200
201  public void setStaticSpec(String staticSpec) {
202    this.staticSpec = staticSpec;
203  }
204
205  @Explain(displayName = "Static Partition Specification", normalExplain = false)
206  public String getStaticSpec() {
207    return staticSpec;
208  }
209
210  public void setGatherStats(boolean gatherStats) {
211    this.gatherStats = gatherStats;
212  }
213
214  @Explain(displayName = "GatherStats", normalExplain = false)
215  public boolean isGatherStats() {
216    return gatherStats;
217  }
218
219  /**
220   * Construct the key prefix used as (intermediate) statistics publishing
221   * and aggregation. During stats publishing phase, this key prefix will be
222   * appended with the optional dynamic partition spec and the task ID. The
223   * whole key uniquely identifies the output of a task for this job. In the
224   * stats aggregation phase, all rows with the same prefix plus dynamic partition
225   * specs (obtained at run-time after MR job finishes) will be serving as the
226   * prefix: all rows with the same prefix (output of all tasks for this job)
227   * will be aggregated.
228   * @return key prefix used for stats publishing and aggregation.
229   */
230  @Explain(displayName = "Stats Publishing Key Prefix", normalExplain = false)
231  public String getStatsAggPrefix() {
232    // dirName uniquely identifies destination directory of a FileSinkOperator.
233    // If more than one FileSinkOperator write to the same partition, this dirName
234    // should be different.
235    return statsKeyPref;
236  }
237
238  /**
239   * Set the stats aggregation key. If the input string is not terminated by Path.SEPARATOR
240   * aggregation key will add one to make it as a directory name.
241   * @param k input directory name.
242   */
243  public void setStatsAggPrefix(String k) {
244    if (k.endsWith(Path.SEPARATOR)) {
245      statsKeyPref = k;
246    } else {
247      statsKeyPref = k + Path.SEPARATOR;
248    }
249  }
250}