PageRenderTime 28ms CodeModel.GetById 11ms app.highlight 13ms RepoModel.GetById 1ms app.codeStats 0ms

/tags/release-0.0.0-rc0/hive/external/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStatsPublisher.java

#
Java | 140 lines | 87 code | 20 blank | 33 comment | 5 complexity | 3c75f4a15cc7c20ab3581169045b3b72 MD5 | raw file
  1/**
  2 * Licensed to the Apache Software Foundation (ASF) under one
  3 * or more contributor license agreements.  See the NOTICE file
  4 * distributed with this work for additional information
  5 * regarding copyright ownership.  The ASF licenses this file
  6 * to you under the Apache License, Version 2.0 (the
  7 * "License"); you may not use this file except in compliance
  8 * with the License.  You may obtain a copy of the License at
  9 *
 10 *     http://www.apache.org/licenses/LICENSE-2.0
 11 *
 12 * Unless required by applicable law or agreed to in writing, software
 13 * distributed under the License is distributed on an "AS IS" BASIS,
 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 15 * See the License for the specific language governing permissions and
 16 * limitations under the License.
 17 */
 18
 19package org.apache.hadoop.hive.hbase;
 20
 21import java.io.IOException;
 22
 23import org.apache.commons.logging.Log;
 24import org.apache.commons.logging.LogFactory;
 25import org.apache.hadoop.conf.Configuration;
 26import org.apache.hadoop.hbase.HBaseConfiguration;
 27import org.apache.hadoop.hbase.HColumnDescriptor;
 28import org.apache.hadoop.hbase.HTableDescriptor;
 29import org.apache.hadoop.hbase.client.Get;
 30import org.apache.hadoop.hbase.client.HBaseAdmin;
 31import org.apache.hadoop.hbase.client.HTable;
 32import org.apache.hadoop.hbase.client.Put;
 33import org.apache.hadoop.hbase.client.Result;
 34import org.apache.hadoop.hbase.client.RowLock;
 35import org.apache.hadoop.hbase.util.Bytes;
 36import org.apache.hadoop.hive.ql.stats.*;
 37
 38/**
 39 * A class that implements the StatsPublisher interface through HBase.
 40 */
 41public class HBaseStatsPublisher implements StatsPublisher {
 42
 43  private HTable htable;
 44  private byte[] rowCountFamily, rowCountColumn;
 45  private final Log LOG = LogFactory.getLog(this.getClass().getName());
 46
 47  /**
 48   * Does the necessary HBase initializations.
 49   */
 50  public boolean connect(Configuration hiveconf) {
 51
 52    try {
 53      HBaseConfiguration hbaseConf = new HBaseConfiguration(hiveconf);
 54      HBaseAdmin hbase = new HBaseAdmin(hbaseConf);
 55      rowCountFamily = Bytes.toBytes(HBaseStatsSetupConstants.PART_STAT_ROW_COUNT_COLUMN_FAMILY);
 56      rowCountColumn = Bytes.toBytes(HBaseStatsSetupConstants.PART_STAT_ROW_COUNT_COLUMN_NAME);
 57      htable = new HTable(HBaseStatsSetupConstants.PART_STAT_TABLE_NAME);
 58      // for performance reason, defer update until the closeConnection
 59      htable.setAutoFlush(false);
 60    } catch (IOException e) {
 61      LOG.error("Error during HBase connection. " + e);
 62      return false;
 63    }
 64
 65    return true;
 66  }
 67
 68  /**
 69   * Writes temporary statistics into HBase;
 70   */
 71  public boolean publishStat(String rowID, String key, String value) {
 72
 73    if (key != StatsSetupConst.ROW_COUNT) {
 74      LOG.warn("Warning. Invalid statistic. Currently " +
 75      "row count is the only supported statistic");
 76      return false;
 77    }
 78
 79    // Write in HBase
 80    try {
 81      Get get = new Get(Bytes.toBytes(rowID));
 82      Result result = htable.get(get);
 83      int val = Integer.parseInt(value);
 84      int oldVal = 0;
 85      if (!result.isEmpty()) {
 86        oldVal = Integer.parseInt(Bytes.toString(result.getValue(rowCountFamily, rowCountColumn)));
 87      }
 88      if (oldVal < val) {
 89        Put row = new Put(Bytes.toBytes(rowID));
 90        row.add(rowCountFamily, rowCountColumn, Bytes.toBytes(Integer.toString(val)));
 91        htable.put(row);
 92      }
 93      return true;
 94
 95    } catch (IOException e) {
 96      LOG.error("Error during publishing statistics. " + e);
 97      return false;
 98    }
 99  }
100
101  public boolean closeConnection() {
102    // batch update
103    try {
104      htable.flushCommits();
105      return true;
106    } catch (IOException e) {
107      LOG.error("Cannot commit changes in stats publishing.", e);
108      return false;
109    }
110  }
111
112
113  /**
114   * Does the necessary HBase initializations.
115   */
116  public boolean init(Configuration hiveconf) {
117    try {
118      HBaseConfiguration hbaseConf = new HBaseConfiguration(hiveconf);
119      HBaseAdmin hbase = new HBaseAdmin(hbaseConf);
120
121      rowCountFamily = Bytes.toBytes(HBaseStatsSetupConstants.PART_STAT_ROW_COUNT_COLUMN_FAMILY);
122      rowCountColumn = Bytes.toBytes(HBaseStatsSetupConstants.PART_STAT_ROW_COUNT_COLUMN_NAME);
123
124      // Creating table if not exists
125      if (!hbase.tableExists(HBaseStatsSetupConstants.PART_STAT_TABLE_NAME)) {
126        HTableDescriptor table = new HTableDescriptor(HBaseStatsSetupConstants.PART_STAT_TABLE_NAME);
127
128        HColumnDescriptor rowCount = new HColumnDescriptor(rowCountFamily);
129        table.addFamily(rowCount);
130
131        hbase.createTable(table);
132      }
133    } catch (IOException e) {
134      LOG.error("Error during HBase initialization. " + e);
135      return false;
136    }
137
138    return true;
139  }  
140}