/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClientHMSImpl.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.hive.hcatalog.api;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import javax.annotation.Nullable;

import org.apache.commons.lang3.tuple.Pair;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.classification.InterfaceAudience;
import org.apache.hadoop.hive.common.classification.InterfaceStability;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.metastore.api.NotificationEvent;
import org.apache.hadoop.hive.metastore.api.NotificationEventResponse;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.PartitionEventType;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.api.UnknownDBException;
import org.apache.hadoop.hive.metastore.api.UnknownTableException;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.exec.SerializationUtilities;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import org.apache.hive.hcatalog.api.repl.HCatReplicationTaskIterator;
import org.apache.hive.hcatalog.api.repl.ReplicationTask;
import org.apache.hive.hcatalog.common.HCatConstants;
import org.apache.hive.hcatalog.common.HCatException;
import org.apache.hive.hcatalog.common.HCatUtil;
import org.apache.hive.hcatalog.data.schema.HCatFieldSchema;
import org.apache.hive.hcatalog.data.schema.HCatSchemaUtils;
import org.apache.thrift.TException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.base.Function;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;

/**
 * The HCatClientHMSImpl is the Hive Metastore client based implementation of
 * HCatClient.
 */
public class HCatClientHMSImpl extends HCatClient {

  private static final Logger LOG = LoggerFactory.getLogger(HCatClientHMSImpl.class);
  private IMetaStoreClient hmsClient;
  private Configuration config;
  private HiveConf hiveConfig;

  @Override
  public List<String> listDatabaseNamesByPattern(String pattern)
    throws HCatException {
    List<String> dbNames = null;
    try {
      dbNames = hmsClient.getDatabases(pattern);
    } catch (MetaException exp) {
      throw new HCatException("MetaException while listing db names. " + exp.getMessage(), exp);
    } catch (TException e) {
      throw new HCatException("Transport Exception while listing db names. " + e.getMessage(), e);
    }
    return dbNames;
  }

  @Override
  public HCatDatabase getDatabase(String dbName) throws HCatException {
    HCatDatabase db = null;
    try {
      Database hiveDB = hmsClient.getDatabase(checkDB(dbName));
      if (hiveDB != null) {
        db = new HCatDatabase(hiveDB);
      }
    } catch (NoSuchObjectException exp) {
      throw new ObjectNotFoundException(
        "NoSuchObjectException while fetching database", exp);
    } catch (MetaException exp) {
      throw new HCatException("MetaException while fetching database",
        exp);
    } catch (TException exp) {
      throw new ConnectionFailureException(
        "TException while fetching database", exp);
    }
    return db;
  }

  @Override
  public void createDatabase(HCatCreateDBDesc dbInfo) throws HCatException {
    try {
      hmsClient.createDatabase(dbInfo.toHiveDb());
    } catch (AlreadyExistsException exp) {
      if (!dbInfo.getIfNotExists()) {
        throw new HCatException(
          "AlreadyExistsException while creating database", exp);
      }
    } catch (InvalidObjectException exp) {
      throw new HCatException(
        "InvalidObjectException while creating database", exp);
    } catch (MetaException exp) {
      throw new HCatException("MetaException while creating database",
        exp);
    } catch (TException exp) {
      throw new ConnectionFailureException(
        "TException while creating database", exp);
    }
  }

  @Override
  public void dropDatabase(String dbName, boolean ifExists, DropDBMode mode)
    throws HCatException {
    boolean isCascade = mode.toString().equalsIgnoreCase("cascade");
    try {
      hmsClient.dropDatabase(checkDB(dbName), true, ifExists, isCascade);
    } catch (NoSuchObjectException e) {
      if (!ifExists) {
        throw new ObjectNotFoundException(
          "NoSuchObjectException while dropping db.", e);
      }
    } catch (InvalidOperationException e) {
      throw new HCatException(
        "InvalidOperationException while dropping db.", e);
    } catch (MetaException e) {
      throw new HCatException("MetaException while dropping db.", e);
    } catch (TException e) {
      throw new ConnectionFailureException("TException while dropping db.",
        e);
    }
  }

  @Override
  public List<String> listTableNamesByPattern(String dbName,
                        String tablePattern) throws HCatException {
    List<String> tableNames = null;
    try {
      tableNames = hmsClient.getTables(checkDB(dbName), tablePattern);
    } catch (MetaException e) {
      throw new HCatException("MetaException while fetching table names. " + e.getMessage(), e);
    } catch (UnknownDBException e) {
      throw new HCatException("UnknownDB " + dbName + " while fetching table names.", e);
    } catch (TException e) {
      throw new HCatException("Transport exception while fetching table names. "
          + e.getMessage(), e);
    }
    return tableNames;
  }

  @Override
  public HCatTable getTable(String dbName, String tableName)
    throws HCatException {
    HCatTable table = null;
    try {
      Table hiveTable = hmsClient.getTable(checkDB(dbName), tableName);
      if (hiveTable != null) {
        table = new HCatTable(hiveTable);
      }
    } catch (MetaException e) {
      throw new HCatException("MetaException while fetching table.", e);
    } catch (NoSuchObjectException e) {
      throw new ObjectNotFoundException(
        "NoSuchObjectException while fetching table.", e);
    } catch (TException e) {
      throw new ConnectionFailureException(
          "TException while fetching table.", e);
    }
    return table;
  }

  @Override
  public void createTable(HCatCreateTableDesc createTableDesc)
    throws HCatException {
    try {
      hmsClient.createTable(createTableDesc.getHCatTable().toHiveTable());
    } catch (AlreadyExistsException e) {
      if (!createTableDesc.getIfNotExists()) {
        throw new HCatException(
          "AlreadyExistsException while creating table.", e);
      }
    } catch (InvalidObjectException e) {
      throw new HCatException(
        "InvalidObjectException while creating table.", e);
    } catch (MetaException e) {
      throw new HCatException("MetaException while creating table.", e);
    } catch (NoSuchObjectException e) {
      throw new ObjectNotFoundException(
        "NoSuchObjectException while creating table.", e);
    } catch (TException e) {
      throw new ConnectionFailureException(
        "TException while creating table.", e);
    } catch (IOException e) {
      throw new HCatException("IOException while creating hive conf.", e);
    }

  }

  @Override
  public void updateTableSchema(String dbName, String tableName, List<HCatFieldSchema> columnSchema)
    throws HCatException {
    try {
      Table table = hmsClient.getTable(dbName, tableName);
      table.getSd().setCols(HCatSchemaUtils.getFieldSchemas(columnSchema));
      hmsClient.alter_table(dbName, tableName, table);
    }
    catch (InvalidOperationException e) {
      throw new HCatException("InvalidOperationException while updating table schema.", e);
    }
    catch (MetaException e) {
      throw new HCatException("MetaException while updating table schema.", e);
    }
    catch (NoSuchObjectException e) {
      throw new ObjectNotFoundException(
          "NoSuchObjectException while updating table schema.", e);
    }
    catch (TException e) {
      throw new ConnectionFailureException(
          "TException while updating table schema.", e);
    }
  }

  @Override
  public void updateTableSchema(String dbName, String tableName, HCatTable newTableDefinition) throws HCatException {
    try {
      hmsClient.alter_table(dbName, tableName, newTableDefinition.toHiveTable());
    }
    catch (InvalidOperationException e) {
      throw new HCatException("InvalidOperationException while updating table schema.", e);
    }
    catch (MetaException e) {
      throw new HCatException("MetaException while updating table schema.", e);
    }
    catch (NoSuchObjectException e) {
      throw new ObjectNotFoundException(
          "NoSuchObjectException while updating table schema.", e);
    }
    catch (TException e) {
      throw new ConnectionFailureException(
          "TException while updating table schema.", e);
    }
  }

  @Override
  public void createTableLike(String dbName, String existingTblName,
                String newTableName, boolean ifNotExists, boolean isExternal,
                String location) throws HCatException {

    Table hiveTable = getHiveTableLike(checkDB(dbName), existingTblName,
      newTableName, ifNotExists, location);
    if (hiveTable != null) {
      try {
        hmsClient.createTable(hiveTable);
      } catch (AlreadyExistsException e) {
        if (!ifNotExists) {
          throw new HCatException(
            "A table already exists with the name "
              + newTableName, e);
        }
      } catch (InvalidObjectException e) {
        throw new HCatException(
          "InvalidObjectException in create table like command.",
          e);
      } catch (MetaException e) {
        throw new HCatException(
          "MetaException in create table like command.", e);
      } catch (NoSuchObjectException e) {
        throw new ObjectNotFoundException(
          "NoSuchObjectException in create table like command.",
          e);
      } catch (TException e) {
        throw new ConnectionFailureException(
          "TException in create table like command.", e);
      }
    }
  }

  @Override
  public void dropTable(String dbName, String tableName, boolean ifExists)
    throws HCatException {
    try {
      hmsClient.dropTable(checkDB(dbName), tableName, true, ifExists);
    } catch (NoSuchObjectException e) {
      if (!ifExists) {
        throw new ObjectNotFoundException(
          "NoSuchObjectException while dropping table.", e);
      }
    } catch (MetaException e) {
      throw new HCatException("MetaException while dropping table.", e);
    } catch (TException e) {
      throw new ConnectionFailureException(
        "TException while dropping table.", e);
    }
  }

  @Override
  public void renameTable(String dbName, String oldName, String newName)
    throws HCatException {
    Table tbl;
    try {
      Table oldtbl = hmsClient.getTable(checkDB(dbName), oldName);
      if (oldtbl != null) {
        // TODO : Should be moved out.
        if (oldtbl
          .getParameters()
          .get(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE) != null) {
          throw new HCatException(
            "Cannot use rename command on a non-native table");
        }
        tbl = new Table(oldtbl);
        tbl.setTableName(newName);
        hmsClient.alter_table(checkDB(dbName), oldName, tbl);
      }
    } catch (MetaException e) {
      throw new HCatException("MetaException while renaming table", e);
    } catch (NoSuchObjectException e) {
      throw new ObjectNotFoundException(
        "NoSuchObjectException while renaming table", e);
    } catch (InvalidOperationException e) {
      throw new HCatException(
        "InvalidOperationException while renaming table", e);
    } catch (TException e) {
      throw new ConnectionFailureException(
          "TException while renaming table", e);
    }
  }

  @Override
  public List<HCatPartition> getPartitions(String dbName, String tblName)
    throws HCatException {
    List<HCatPartition> hcatPtns = new ArrayList<HCatPartition>();
    try {
      HCatTable hcatTable = getTable(dbName, tblName);
      List<Partition> hivePtns = hmsClient.listPartitions(
        checkDB(dbName), tblName, (short) -1);
      for (Partition ptn : hivePtns) {
        hcatPtns.add(new HCatPartition(hcatTable, ptn));
      }
    } catch (NoSuchObjectException e) {
      throw new ObjectNotFoundException(
        "NoSuchObjectException while retrieving partition.", e);
    } catch (MetaException e) {
      throw new HCatException(
        "MetaException while retrieving partition.", e);
    } catch (TException e) {
      throw new ConnectionFailureException(
        "TException while retrieving partition.", e);
    }
    return hcatPtns;
  }

  @Override
  public List<HCatPartition> getPartitions(String dbName, String tblName, Map<String, String> partitionSpec) throws HCatException {
    return listPartitionsByFilter(dbName, tblName, getFilterString(partitionSpec));
  }

  @Override
  @InterfaceAudience.LimitedPrivate({"Hive"})
  @InterfaceStability.Evolving
  public HCatPartitionSpec getPartitionSpecs(String dbName, String tableName, int maxPartitions) throws HCatException {
    try {
      return new HCatPartitionSpec(getTable(dbName, tableName),
                                   hmsClient.listPartitionSpecs(dbName, tableName, maxPartitions));
    }
    catch (NoSuchObjectException e) {
      throw new ObjectNotFoundException(
          "NoSuchObjectException while retrieving partition.", e);
    } catch (MetaException e) {
      throw new HCatException(
          "MetaException while retrieving partition.", e);
    } catch (TException e) {
      throw new ConnectionFailureException(
          "TException while retrieving partition.", e);
    }
  }

  @Override
  public HCatPartitionSpec getPartitionSpecs(String dbName, String tableName, Map<String, String> partitionSelector, int maxPartitions) throws HCatException {
    return listPartitionSpecsByFilter(dbName, tableName, getFilterString(partitionSelector), maxPartitions);
  }

  private static String getFilterString(Map<String, String> partitionSpec) {
    final String AND = " AND ";

    StringBuilder filter = new StringBuilder();
    for (Map.Entry<String, String> entry : partitionSpec.entrySet()) {
      filter.append(entry.getKey()).append("=").append("\"").append(entry.getValue()).append("\"").append(AND);
    }

    int length = filter.toString().length();
    if (length > 0)
      filter.delete(length - AND.length(), length);

    return filter.toString();
  }

  @Override
  public HCatPartition getPartition(String dbName, String tableName,
                    Map<String, String> partitionSpec) throws HCatException {
    HCatPartition partition = null;
    try {
      HCatTable hcatTable = getTable(dbName, tableName);
      List<HCatFieldSchema> partitionColumns = hcatTable.getPartCols();
      if (partitionColumns.size() != partitionSpec.size()) {
        throw new HCatException("Partition-spec doesn't have the right number of partition keys.");
      }

      ArrayList<String> ptnValues = new ArrayList<String>();
      for (HCatFieldSchema partitionColumn : partitionColumns) {
        String partKey = partitionColumn.getName();
        if (partitionSpec.containsKey(partKey)) {
          ptnValues.add(partitionSpec.get(partKey)); // Partition-keys added in order.
        }
        else {
          throw new HCatException("Invalid partition-key specified: " + partKey);
        }
      }
      Partition hivePartition = hmsClient.getPartition(checkDB(dbName),
        tableName, ptnValues);
      if (hivePartition != null) {
        partition = new HCatPartition(hcatTable, hivePartition);
      }
    } catch (MetaException e) {
      throw new HCatException(
        "MetaException while retrieving partition.", e);
    } catch (NoSuchObjectException e) {
      throw new ObjectNotFoundException(
        "NoSuchObjectException while retrieving partition.", e);
    } catch (TException e) {
      throw new ConnectionFailureException(
          "TException while retrieving partition.", e);
    }
    return partition;
  }

  @Override
  public void addPartition(HCatAddPartitionDesc partInfo)
    throws HCatException {
    Table tbl = null;
    try {
      tbl = hmsClient.getTable(partInfo.getDatabaseName(),
          partInfo.getTableName());
      // TODO: Should be moved out.
      if (tbl.getPartitionKeysSize() == 0) {
        throw new HCatException("The table " + partInfo.getTableName()
          + " is not partitioned.");
      }

      HCatTable hcatTable = new HCatTable(tbl);

      HCatPartition hcatPartition = partInfo.getHCatPartition();

      // TODO: Remove in Hive 0.16.
      // This is only required to support the deprecated methods in HCatAddPartitionDesc.Builder.
      if (hcatPartition == null) {
        hcatPartition = partInfo.getHCatPartition(hcatTable);
      }

      hmsClient.add_partition(hcatPartition.toHivePartition());
    } catch (InvalidObjectException e) {
      throw new HCatException(
        "InvalidObjectException while adding partition.", e);
    } catch (AlreadyExistsException e) {
      throw new HCatException(
        "AlreadyExistsException while adding partition.", e);
    } catch (MetaException e) {
      throw new HCatException("MetaException while adding partition.", e);
    } catch (NoSuchObjectException e) {
      throw new ObjectNotFoundException("The table " + partInfo.getTableName()
        + " is could not be found.", e);
    } catch (TException e) {
      throw new ConnectionFailureException(
          "TException while adding partition.", e);
    }
  }

  /**
   * Helper class to help build ExprDesc tree to represent the partitions to be dropped.
   * Note: At present, the ExpressionBuilder only constructs partition predicates where
   * partition-keys equal specific values, and logical-AND expressions. E.g.
   *  ( dt = '20150310' AND region = 'US' )
   * This only supports the partition-specs specified by the Map argument of:
   * {@link org.apache.hive.hcatalog.api.HCatClient#dropPartitions(String, String, Map, boolean)}
   */
  private static class ExpressionBuilder {

    private Map<String, PrimitiveTypeInfo> partColumnTypesMap = Maps.newHashMap();
    private Map<String, String> partSpecs;

    public ExpressionBuilder(Table table, Map<String, String> partSpecs) {
      this.partSpecs = partSpecs;
      for (FieldSchema partField : table.getPartitionKeys()) {
        partColumnTypesMap.put(partField.getName().toLowerCase(),
            TypeInfoFactory.getPrimitiveTypeInfo(partField.getType()));
      }
    }

    private PrimitiveTypeInfo getTypeFor(String partColumn) {
      return partColumnTypesMap.get(partColumn.toLowerCase());
    }

    private Object getTypeAppropriateValueFor(PrimitiveTypeInfo type, String value) {
      ObjectInspectorConverters.Converter converter = ObjectInspectorConverters.getConverter(
          TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(TypeInfoFactory.stringTypeInfo),
          TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(type));

      return converter.convert(value);
    }

    public ExprNodeGenericFuncDesc equalityPredicate(String partColumn, String value) throws SemanticException {

      PrimitiveTypeInfo partColumnType = getTypeFor(partColumn);
      ExprNodeColumnDesc partColumnExpr = new ExprNodeColumnDesc(partColumnType, partColumn, null, true);
      ExprNodeConstantDesc valueExpr = new ExprNodeConstantDesc(partColumnType,
          getTypeAppropriateValueFor(partColumnType, value));

      return binaryPredicate("=", partColumnExpr, valueExpr);
    }

    public ExprNodeGenericFuncDesc binaryPredicate(String function, ExprNodeDesc lhs, ExprNodeDesc rhs) throws SemanticException {
      return new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo,
          FunctionRegistry.getFunctionInfo(function).getGenericUDF(),
          Lists.newArrayList(lhs, rhs));
    }

    public ExprNodeGenericFuncDesc build() throws SemanticException {
      ExprNodeGenericFuncDesc resultExpr = null;

      for (Map.Entry<String,String> partSpec : partSpecs.entrySet()) {
        String column = partSpec.getKey();
        String value  = partSpec.getValue();
        ExprNodeGenericFuncDesc partExpr = equalityPredicate(column, value);

        resultExpr = (resultExpr == null? partExpr : binaryPredicate("and", resultExpr, partExpr));
      }

      return resultExpr;
    }
  } // class ExpressionBuilder;

  private static boolean isExternal(Table table) {
    return table.getParameters() != null
        && "TRUE".equalsIgnoreCase(table.getParameters().get("EXTERNAL"));
  }

  private void dropPartitionsUsingExpressions(Table table, Map<String, String> partitionSpec,
                                              boolean ifExists, boolean deleteData)
      throws SemanticException, TException {
    LOG.info("HCatClient: Dropping partitions using partition-predicate Expressions.");
    ExprNodeGenericFuncDesc partitionExpression = new ExpressionBuilder(table, partitionSpec).build();
    Pair<Integer, byte[]> serializedPartitionExpression = Pair.of(partitionSpec.size(),
            SerializationUtilities.serializeExpressionToKryo(partitionExpression));
    hmsClient.dropPartitions(table.getDbName(), table.getTableName(), Arrays.asList(serializedPartitionExpression),
        deleteData && !isExternal(table),  // Delete data?
        ifExists,                          // Fail if table doesn't exist?
        false);                            // Need results back?
  }

  private void dropPartitionsIteratively(String dbName, String tableName,
                                         Map<String, String> partitionSpec, boolean ifExists, boolean deleteData)
      throws HCatException, TException {
    LOG.info("HCatClient: Dropping partitions iteratively.");
    List<Partition> partitions = hmsClient.listPartitionsByFilter(dbName, tableName,
        getFilterString(partitionSpec), (short) -1);
    for (Partition partition : partitions) {
      dropPartition(partition, ifExists, deleteData);
    }
  }

  @Override
  public void dropPartitions(String dbName, String tableName,
                 Map<String, String> partitionSpec, boolean ifExists, boolean deleteData)
    throws HCatException {
    LOG.info("HCatClient dropPartitions(db=" + dbName + ",table=" + tableName + ", partitionSpec: ["+ partitionSpec + "]).");
    try {
      dbName = checkDB(dbName);
      Table table = hmsClient.getTable(dbName, tableName);

      if (hiveConfig.getBoolVar(HiveConf.ConfVars.METASTORE_CLIENT_DROP_PARTITIONS_WITH_EXPRESSIONS)) {
        try {
          dropPartitionsUsingExpressions(table, partitionSpec, ifExists, deleteData);
        }
        catch (SemanticException parseFailure) {
          LOG.warn("Could not push down partition-specification to back-end, for dropPartitions(). Resorting to iteration.",
              parseFailure);
          dropPartitionsIteratively(dbName, tableName, partitionSpec, ifExists, deleteData);
        }
      }
      else {
        // Not using expressions.
        dropPartitionsIteratively(dbName, tableName, partitionSpec, ifExists, deleteData);
      }
    } catch (NoSuchObjectException e) {
      throw new ObjectNotFoundException(
          "NoSuchObjectException while dropping partition. " +
              "Either db(" + dbName + ") or table(" + tableName + ") missing.", e);
    } catch (MetaException e) {
      throw new HCatException("MetaException while dropping partition.",
        e);
    } catch (TException e) {
      throw new ConnectionFailureException(
        "TException while dropping partition.", e);
    }
  }

  @Override
  public void dropPartitions(String dbName, String tableName,
                             Map<String, String> partitionSpec, boolean ifExists) throws HCatException {
    dropPartitions(dbName, tableName, partitionSpec, ifExists, true);
  }

  private void dropPartition(Partition partition, boolean ifExists, boolean deleteData)
    throws HCatException, MetaException, TException {
    try {
      hmsClient.dropPartition(partition.getDbName(), partition.getTableName(), partition.getValues(), deleteData);
    } catch (NoSuchObjectException e) {
      if (!ifExists) {
        throw new ObjectNotFoundException(
            "NoSuchObjectException while dropping partition: " + partition.getValues(), e);
      }
    }
  }

  @Override
  public List<HCatPartition> listPartitionsByFilter(String dbName,
                            String tblName, String filter) throws HCatException {
    List<HCatPartition> hcatPtns = new ArrayList<HCatPartition>();
    try {
      HCatTable table = getTable(dbName, tblName);
      List<Partition> hivePtns = hmsClient.listPartitionsByFilter(
          table.getDbName(), table.getTableName(), filter, (short) -1);
      for (Partition ptn : hivePtns) {
        hcatPtns.add(new HCatPartition(table, ptn));
      }
    } catch (MetaException e) {
      throw new HCatException("MetaException while fetching partitions.",
        e);
    } catch (NoSuchObjectException e) {
      throw new ObjectNotFoundException(
        "NoSuchObjectException while fetching partitions.", e);
    } catch (TException e) {
      throw new ConnectionFailureException(
        "TException while fetching partitions.", e);
    }
    return hcatPtns;
  }

  @Override
  @InterfaceAudience.LimitedPrivate({"Hive"})
  @InterfaceStability.Evolving
  public HCatPartitionSpec listPartitionSpecsByFilter(String dbName, String tblName, String filter, int maxPartitions)
      throws HCatException {
    try {
      return new HCatPartitionSpec(getTable(dbName, tblName),
                                   hmsClient.listPartitionSpecsByFilter(dbName, tblName, filter, maxPartitions));
    }
    catch(MetaException e) {
      throw new HCatException("MetaException while fetching partitions.", e);
    }
    catch (NoSuchObjectException e) {
      throw new ObjectNotFoundException(
          "NoSuchObjectException while fetching partitions.", e);
    }
    catch (TException e) {
      throw new ConnectionFailureException(
        "TException while fetching partitions.", e);
    }
  }

  @Override
  public void markPartitionForEvent(String dbName, String tblName,
                    Map<String, String> partKVs, PartitionEventType eventType)
    throws HCatException {
    try {
      hmsClient.markPartitionForEvent(checkDB(dbName), tblName, partKVs,
        eventType);
    } catch (MetaException e) {
      throw new HCatException(
        "MetaException while marking partition for event.", e);
    } catch (NoSuchObjectException e) {
      throw new ObjectNotFoundException(
        "NoSuchObjectException while marking partition for event.",
        e);
    } catch (UnknownTableException e) {
      throw new HCatException(
        "UnknownTableException while marking partition for event.",
        e);
    } catch (UnknownDBException e) {
      throw new HCatException(
        "UnknownDBException while marking partition for event.", e);
    } catch (TException e) {
      throw new ConnectionFailureException(
        "TException while marking partition for event.", e);
    }
  }

  @Override
  public boolean isPartitionMarkedForEvent(String dbName, String tblName,
                       Map<String, String> partKVs, PartitionEventType eventType)
    throws HCatException {
    boolean isMarked = false;
    try {
      isMarked = hmsClient.isPartitionMarkedForEvent(checkDB(dbName),
        tblName, partKVs, eventType);
    } catch (MetaException e) {
      throw new HCatException(
        "MetaException while checking partition for event.", e);
    } catch (NoSuchObjectException e) {
      throw new ObjectNotFoundException(
        "NoSuchObjectException while checking partition for event.",
        e);
    } catch (UnknownTableException e) {
      throw new HCatException(
        "UnknownTableException while checking partition for event.",
        e);
    } catch (UnknownDBException e) {
      throw new HCatException(
        "UnknownDBException while checking partition for event.", e);
    } catch (TException e) {
      throw new ConnectionFailureException(
        "TException while checking partition for event.", e);
    }
    return isMarked;
  }

  @Override
  public String getDelegationToken(String owner,
                   String renewerKerberosPrincipalName) throws HCatException {
    String token = null;
    try {
      token = hmsClient.getDelegationToken(owner,
          renewerKerberosPrincipalName);
    } catch (MetaException e) {
      throw new HCatException(
        "MetaException while getting delegation token.", e);
    } catch (TException e) {
      throw new ConnectionFailureException(
        "TException while getting delegation token.", e);
    }

    return token;
  }

  @Override
  public long renewDelegationToken(String tokenStrForm) throws HCatException {
    long time = 0;
    try {
      time = hmsClient.renewDelegationToken(tokenStrForm);
    } catch (MetaException e) {
      throw new HCatException(
        "MetaException while renewing delegation token.", e);
    } catch (TException e) {
      throw new ConnectionFailureException(
        "TException while renewing delegation token.", e);
    }

    return time;
  }

  @Override
  public void cancelDelegationToken(String tokenStrForm)
    throws HCatException {
    try {
      hmsClient.cancelDelegationToken(tokenStrForm);
    } catch (MetaException e) {
      throw new HCatException(
        "MetaException while canceling delegation token.", e);
    } catch (TException e) {
      throw new ConnectionFailureException(
        "TException while canceling delegation token.", e);
    }
  }

  /*
   * @param conf /* @throws HCatException,ConnectionFailureException
   *
   * @see
   * org.apache.hive.hcatalog.api.HCatClient#initialize(org.apache.hadoop.conf.
   * Configuration)
   */
  @Override
  void initialize(Configuration conf) throws HCatException {
    this.config = conf;
    try {
      hiveConfig = HCatUtil.getHiveConf(config);
      hmsClient = HCatUtil.getHiveMetastoreClient(hiveConfig);
    } catch (MetaException exp) {
      throw new HCatException("MetaException while creating HMS client",
        exp);
    } catch (IOException exp) {
      throw new HCatException("IOException while creating HMS client",
        exp);
    }

  }

  @Override
  public String getConfVal(String key, String defaultVal) {
    return hiveConfig.get(key,defaultVal);
  }

  private Table getHiveTableLike(String dbName, String existingTblName,
                   String newTableName, boolean isExternal, String location)
    throws HCatException {
    Table oldtbl = null;
    Table newTable = null;
    try {
      oldtbl = hmsClient.getTable(checkDB(dbName), existingTblName);
    } catch (MetaException e1) {
      throw new HCatException(
        "MetaException while retrieving existing table.", e1);
    } catch (NoSuchObjectException e1) {
      throw new ObjectNotFoundException(
        "NoSuchObjectException while retrieving existing table.",
        e1);
    } catch (TException e1) {
      throw new ConnectionFailureException(
          "TException while retrieving existing table.", e1);
    }
    if (oldtbl != null) {
      newTable = new Table();
      newTable.setTableName(newTableName);
      newTable.setDbName(dbName);
      StorageDescriptor sd = new StorageDescriptor(oldtbl.getSd());
      newTable.setSd(sd);
      newTable.setParameters(oldtbl.getParameters());
      if (location == null) {
        newTable.getSd().setLocation(oldtbl.getSd().getLocation());
      } else {
        newTable.getSd().setLocation(location);
      }
      if (isExternal) {
        newTable.putToParameters("EXTERNAL", "TRUE");
        newTable.setTableType(TableType.EXTERNAL_TABLE.toString());
      } else {
        newTable.getParameters().remove("EXTERNAL");
      }
      // set create time
      newTable.setCreateTime((int) (System.currentTimeMillis() / 1000));
      newTable.setLastAccessTimeIsSet(false);
    }
    return newTable;
  }

  /*
   * @throws HCatException
   *
   * @see org.apache.hive.hcatalog.api.HCatClient#closeClient()
   */
  @Override
  public void close() throws HCatException {
    hmsClient.close();
  }

  private String checkDB(String name) {
    if (StringUtils.isEmpty(name)) {
      return Warehouse.DEFAULT_DATABASE_NAME;
    } else {
      return name;
    }
  }

  /*
   * @param partInfoList
   *  @return The size of the list of partitions.
   * @throws HCatException,ConnectionFailureException
   * @see org.apache.hive.hcatalog.api.HCatClient#addPartitions(java.util.List)
   */
  @Override
  public int addPartitions(List<HCatAddPartitionDesc> partInfoList)
    throws HCatException {
    int numPartitions = -1;
    if ((partInfoList == null) || (partInfoList.size() == 0)) {
      throw new HCatException("The partition list is null or empty.");
    }

    Table tbl = null;
    try {
      tbl = hmsClient.getTable(partInfoList.get(0).getDatabaseName(),
        partInfoList.get(0).getTableName());
      HCatTable hcatTable = new HCatTable(tbl);
      ArrayList<Partition> ptnList = new ArrayList<Partition>();
      for (HCatAddPartitionDesc desc : partInfoList) {
        HCatPartition hCatPartition = desc.getHCatPartition();

        // TODO: Remove in Hive 0.16.
        // This is required only to support the deprecated HCatAddPartitionDesc.Builder interfaces.
        if (hCatPartition == null) {
          hCatPartition = desc.getHCatPartition(hcatTable);
        }

        ptnList.add(hCatPartition.toHivePartition());
      }
      numPartitions = hmsClient.add_partitions(ptnList);
    } catch (InvalidObjectException e) {
      throw new HCatException(
        "InvalidObjectException while adding partition.", e);
    } catch (AlreadyExistsException e) {
      throw new HCatException(
        "AlreadyExistsException while adding partition.", e);
    } catch (MetaException e) {
      throw new HCatException("MetaException while adding partition.", e);
    } catch (NoSuchObjectException e) {
      throw new ObjectNotFoundException("The table "
        + partInfoList.get(0).getTableName()
        + " is could not be found.", e);
    } catch (TException e) {
      throw new ConnectionFailureException(
          "TException while adding partition.", e);
    }
    return numPartitions;
  }

  @Override
  @InterfaceAudience.LimitedPrivate({"Hive"})
  @InterfaceStability.Evolving
  public int addPartitionSpec(HCatPartitionSpec partitionSpec) throws HCatException {

    try {
      return hmsClient.add_partitions_pspec(partitionSpec.toPartitionSpecProxy());
    } catch (InvalidObjectException e) {
      throw new HCatException(
          "InvalidObjectException while adding partition.", e);
    } catch (AlreadyExistsException e) {
      throw new HCatException(
          "AlreadyExistsException while adding partition.", e);
    } catch (MetaException e) {
      throw new HCatException("MetaException while adding partition.", e);
    } catch (NoSuchObjectException e) {
      throw new ObjectNotFoundException("The table "
          + "could not be found.", e);
    } catch (TException e) {
      throw new ConnectionFailureException(
          "TException while adding partition.", e);
    }
  }

  @Override
  public String getMessageBusTopicName(String dbName, String tableName) throws HCatException {
    try {
      return hmsClient.getTable(dbName, tableName).getParameters().get(
          HCatConstants.HCAT_MSGBUS_TOPIC_NAME);
    }
    catch (MetaException e) {
      throw new HCatException("MetaException while retrieving JMS Topic name.", e);
    } catch (NoSuchObjectException e) {
      throw new HCatException("Could not find DB:" + dbName + " or Table:" + tableName, e);
    } catch (TException e) {
      throw new ConnectionFailureException(
          "TException while retrieving JMS Topic name.", e);
    }
  }

  @Override
  public Iterator<ReplicationTask> getReplicationTasks(
      long lastEventId, int maxEvents, String dbName, String tableName) throws HCatException {
    return new HCatReplicationTaskIterator(this,lastEventId,maxEvents,dbName,tableName);
  }

  @Override
  public List<HCatNotificationEvent> getNextNotification(long lastEventId, int maxEvents,
                                                         IMetaStoreClient.NotificationFilter filter)
      throws HCatException {
    try {
      NotificationEventResponse rsp = hmsClient.getNextNotification(lastEventId, maxEvents, filter);
      if (rsp != null && rsp.getEvents() != null) {
        return Lists.transform(rsp.getEvents(), new Function<NotificationEvent, HCatNotificationEvent>() {
          @Override
          public HCatNotificationEvent apply(@Nullable NotificationEvent notificationEvent) {
            return new HCatNotificationEvent(notificationEvent);
          }
        });
      } else {
        return Collections.emptyList();
      }
    } catch (TException e) {
      throw new ConnectionFailureException("TException while getting notifications", e);
    }
  }

  @Override
  public long getCurrentNotificationEventId() throws HCatException {
    try {
      CurrentNotificationEventId id = hmsClient.getCurrentNotificationEventId();
      return id.getEventId();
    } catch (TException e) {
      throw new ConnectionFailureException("TException while getting current notification event " +
          "id " , e);
    }
  }

  @Override
  public String serializeTable(HCatTable hcatTable) throws HCatException {
    return MetadataSerializer.get().serializeTable(hcatTable);
  }

  @Override
  public HCatTable deserializeTable(String hcatTableStringRep) throws HCatException {
    return MetadataSerializer.get().deserializeTable(hcatTableStringRep);
  }

  @Override
  public String serializePartition(HCatPartition hcatPartition) throws HCatException {
    return MetadataSerializer.get().serializePartition(hcatPartition);
  }

  @Override
  public List<String> serializePartitions(List<HCatPartition> hcatPartitions) throws HCatException {
    List<String> partStrings = new ArrayList<String>(hcatPartitions.size());
    MetadataSerializer serializer = MetadataSerializer.get();

    for (HCatPartition partition : hcatPartitions) {
      partStrings.add(serializer.serializePartition(partition));
    }

    return partStrings;
  }

  @Override
  public HCatPartition deserializePartition(String hcatPartitionStringRep) throws HCatException {
    HCatPartition hcatPartition = MetadataSerializer.get().deserializePartition(hcatPartitionStringRep);
    hcatPartition.hcatTable(getTable(hcatPartition.getDatabaseName(), hcatPartition.getTableName()));
    return hcatPartition;
  }

  @Override
  public List<HCatPartition> deserializePartitions(List<String> hcatPartitionStringReps) throws HCatException {
    List<HCatPartition> partitions = new ArrayList<HCatPartition>(hcatPartitionStringReps.size());
    MetadataSerializer deserializer = MetadataSerializer.get();
    HCatTable table = null;
    for (String partString : hcatPartitionStringReps) {
      HCatPartition partition;
      if (table == null) {
        partition = deserializePartition(partString);
        table = partition.hcatTable();
      }
      else {
        partition = deserializer.deserializePartition(partString);
        if (partition.getDatabaseName().equals(table.getDbName())
            && partition.getTableName().equals(table.getTableName())) {
          partition.hcatTable(table);
        }
        else {
          throw new HCatException("All partitions are not of the same table: "
              + table.getDbName() + "." + table.getTableName());
        }
      }
      partitions.add(partition);
    }
    return partitions;
  }

  @Override
  public List<String> serializePartitionSpec(HCatPartitionSpec partitionSpec) throws HCatException {
    return MetadataSerializer.get().serializePartitionSpec(partitionSpec);
  }

  @Override
  public HCatPartitionSpec deserializePartitionSpec(List<String> hcatPartitionSpecStrings) throws HCatException {
    HCatPartitionSpec hcatPartitionSpec = MetadataSerializer.get()
        .deserializePartitionSpec(hcatPartitionSpecStrings);
    hcatPartitionSpec
        .hcatTable(getTable(hcatPartitionSpec.getDbName(), hcatPartitionSpec.getTableName()));
    return hcatPartitionSpec;
  }
}