PageRenderTime 34ms CodeModel.GetById 8ms app.highlight 20ms RepoModel.GetById 2ms app.codeStats 0ms

/tags/release-0.2.0-rc0/src/java/org/apache/hcatalog/mapreduce/HCatBaseOutputCommitter.java

#
Java | 116 lines | 78 code | 18 blank | 20 comment | 11 complexity | 05ee2601e451e0f7c6412af0df774309 MD5 | raw file
  1/*
  2 * Licensed to the Apache Software Foundation (ASF) under one
  3 * or more contributor license agreements.  See the NOTICE file
  4 * distributed with this work for additional information
  5 * regarding copyright ownership.  The ASF licenses this file
  6 * to you under the Apache License, Version 2.0 (the
  7 * "License"); you may not use this file except in compliance
  8 * with the License.  You may obtain a copy of the License at
  9 *
 10 *     http://www.apache.org/licenses/LICENSE-2.0
 11 *
 12 * Unless required by applicable law or agreed to in writing, software
 13 * distributed under the License is distributed on an "AS IS" BASIS,
 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 15 * See the License for the specific language governing permissions and
 16 * limitations under the License.
 17 */
 18
 19package org.apache.hcatalog.mapreduce;
 20
 21import java.io.IOException;
 22
 23import org.apache.hadoop.conf.Configuration;
 24import org.apache.hadoop.fs.FileSystem;
 25import org.apache.hadoop.fs.Path;
 26import org.apache.hadoop.mapreduce.JobContext;
 27import org.apache.hadoop.mapreduce.JobStatus.State;
 28import org.apache.hadoop.mapreduce.OutputCommitter;
 29import org.apache.hadoop.mapreduce.TaskAttemptContext;
 30import org.apache.hcatalog.common.HCatException;
 31
 32public abstract class HCatBaseOutputCommitter extends OutputCommitter {
 33
 34  /** The underlying output committer */
 35  protected final OutputCommitter baseCommitter;
 36
 37  public HCatBaseOutputCommitter(JobContext context, OutputCommitter baseCommitter) {
 38    this.baseCommitter = baseCommitter;
 39  }
 40
 41  @Override
 42  public void abortTask(TaskAttemptContext context) throws IOException {
 43      baseCommitter.abortTask(context);
 44  }
 45
 46  @Override
 47  public void commitTask(TaskAttemptContext context) throws IOException {
 48      baseCommitter.commitTask(context);
 49  }
 50
 51  @Override
 52  public boolean needsTaskCommit(TaskAttemptContext context) throws IOException {
 53      return baseCommitter.needsTaskCommit(context);
 54  }
 55
 56  @Override
 57  public void setupJob(JobContext context) throws IOException {
 58    if( baseCommitter != null ) {
 59      baseCommitter.setupJob(context);
 60    }
 61  }
 62
 63  @Override
 64  public void setupTask(TaskAttemptContext context) throws IOException {
 65      baseCommitter.setupTask(context);
 66  }
 67
 68  @Override
 69  public void abortJob(JobContext jobContext, State state) throws IOException {
 70    if(baseCommitter != null) {
 71      baseCommitter.abortJob(jobContext, state);
 72    }
 73    OutputJobInfo jobInfo = HCatOutputFormat.getJobInfo(jobContext);
 74
 75    doAbortJob(jobContext, jobInfo);
 76
 77    Path src = new Path(jobInfo.getLocation());
 78    FileSystem fs = src.getFileSystem(jobContext.getConfiguration());
 79    fs.delete(src, true);
 80  }
 81
 82  protected void doAbortJob(JobContext jobContext, OutputJobInfo jobInfo) throws HCatException {
 83  }
 84
 85  public static final String SUCCEEDED_FILE_NAME = "_SUCCESS";
 86  static final String SUCCESSFUL_JOB_OUTPUT_DIR_MARKER =
 87    "mapreduce.fileoutputcommitter.marksuccessfuljobs";
 88
 89  private static boolean getOutputDirMarking(Configuration conf) {
 90    return conf.getBoolean(SUCCESSFUL_JOB_OUTPUT_DIR_MARKER,
 91                           false);
 92  }
 93
 94  @Override
 95  public void commitJob(JobContext jobContext) throws IOException {
 96    if(baseCommitter != null) {
 97      baseCommitter.commitJob(jobContext);
 98    }
 99    // create _SUCCESS FILE if so requested.
100    OutputJobInfo jobInfo = HCatOutputFormat.getJobInfo(jobContext);
101    if(getOutputDirMarking(jobContext.getConfiguration())) {
102      Path outputPath = new Path(jobInfo.getLocation());
103      if (outputPath != null) {
104        FileSystem fileSys = outputPath.getFileSystem(jobContext.getConfiguration());
105        // create a file in the folder to mark it
106        if (fileSys.exists(outputPath)) {
107          Path filePath = new Path(outputPath, SUCCEEDED_FILE_NAME);
108          if(!fileSys.exists(filePath)) { // may have been created by baseCommitter.commitJob()
109            fileSys.create(filePath).close();
110          }
111        }
112      }
113    }
114    cleanupJob(jobContext);
115  }
116}