PageRenderTime 132ms CodeModel.GetById 20ms app.highlight 98ms RepoModel.GetById 2ms app.codeStats 1ms

/tags/release-0.1-rc2/hive/external/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java

#
Java | 3258 lines | 2454 code | 332 blank | 472 comment | 676 complexity | 834d1c5310bbb8a1f599a85916c3cad4 MD5 | raw file

Large files files are truncated, but you can click here to view the full file

   1/**
   2 * Licensed to the Apache Software Foundation (ASF) under one
   3 * or more contributor license agreements.  See the NOTICE file
   4 * distributed with this work for additional information
   5 * regarding copyright ownership.  The ASF licenses this file
   6 * to you under the Apache License, Version 2.0 (the
   7 * "License"); you may not use this file except in compliance
   8 * with the License.  You may obtain a copy of the License at
   9 *
  10 *     http://www.apache.org/licenses/LICENSE-2.0
  11 *
  12 * Unless required by applicable law or agreed to in writing, software
  13 * distributed under the License is distributed on an "AS IS" BASIS,
  14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15 * See the License for the specific language governing permissions and
  16 * limitations under the License.
  17 */
  18
  19package org.apache.hadoop.hive.ql.exec;
  20
  21import static org.apache.commons.lang.StringUtils.join;
  22import static org.apache.hadoop.util.StringUtils.stringifyException;
  23
  24import java.io.BufferedWriter;
  25import java.io.DataOutput;
  26import java.io.FileNotFoundException;
  27import java.io.IOException;
  28import java.io.OutputStreamWriter;
  29import java.io.Serializable;
  30import java.io.Writer;
  31import java.net.URI;
  32import java.net.URISyntaxException;
  33import java.util.ArrayList;
  34import java.util.Collections;
  35import java.util.Comparator;
  36import java.util.HashSet;
  37import java.util.Iterator;
  38import java.util.List;
  39import java.util.Map;
  40import java.util.Set;
  41import java.util.SortedSet;
  42import java.util.TreeSet;
  43import java.util.Map.Entry;
  44
  45import org.apache.commons.logging.Log;
  46import org.apache.commons.logging.LogFactory;
  47import org.apache.hadoop.fs.FSDataOutputStream;
  48import org.apache.hadoop.fs.FileStatus;
  49import org.apache.hadoop.fs.FileSystem;
  50import org.apache.hadoop.fs.FsShell;
  51import org.apache.hadoop.fs.Path;
  52import org.apache.hadoop.hive.conf.HiveConf;
  53import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
  54import org.apache.hadoop.hive.metastore.MetaStoreUtils;
  55import org.apache.hadoop.hive.metastore.ProtectMode;
  56import org.apache.hadoop.hive.metastore.TableType;
  57import org.apache.hadoop.hive.metastore.Warehouse;
  58import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
  59import org.apache.hadoop.hive.metastore.api.Database;
  60import org.apache.hadoop.hive.metastore.api.FieldSchema;
  61import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege;
  62import org.apache.hadoop.hive.metastore.api.HiveObjectRef;
  63import org.apache.hadoop.hive.metastore.api.HiveObjectType;
  64import org.apache.hadoop.hive.metastore.api.Index;
  65import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
  66import org.apache.hadoop.hive.metastore.api.MetaException;
  67import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
  68import org.apache.hadoop.hive.metastore.api.Order;
  69import org.apache.hadoop.hive.metastore.api.PrincipalType;
  70import org.apache.hadoop.hive.metastore.api.PrivilegeBag;
  71import org.apache.hadoop.hive.metastore.api.PrivilegeGrantInfo;
  72import org.apache.hadoop.hive.metastore.api.Role;
  73import org.apache.hadoop.hive.ql.Context;
  74import org.apache.hadoop.hive.ql.DriverContext;
  75import org.apache.hadoop.hive.ql.QueryPlan;
  76import org.apache.hadoop.hive.ql.hooks.ReadEntity;
  77import org.apache.hadoop.hive.ql.hooks.WriteEntity;
  78import org.apache.hadoop.hive.ql.lockmgr.HiveLock;
  79import org.apache.hadoop.hive.ql.lockmgr.HiveLockManager;
  80import org.apache.hadoop.hive.ql.lockmgr.HiveLockMode;
  81import org.apache.hadoop.hive.ql.lockmgr.HiveLockObject;
  82import org.apache.hadoop.hive.ql.lockmgr.HiveLockObject.HiveLockObjectData;
  83import org.apache.hadoop.hive.ql.metadata.CheckResult;
  84import org.apache.hadoop.hive.ql.metadata.Hive;
  85import org.apache.hadoop.hive.ql.metadata.HiveException;
  86import org.apache.hadoop.hive.ql.metadata.HiveMetaStoreChecker;
  87import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler;
  88import org.apache.hadoop.hive.ql.metadata.InvalidTableException;
  89import org.apache.hadoop.hive.ql.metadata.MetaDataFormatUtils;
  90import org.apache.hadoop.hive.ql.metadata.Partition;
  91import org.apache.hadoop.hive.ql.metadata.Table;
  92import org.apache.hadoop.hive.ql.plan.AddPartitionDesc;
  93import org.apache.hadoop.hive.ql.plan.AlterDatabaseDesc;
  94import org.apache.hadoop.hive.ql.plan.AlterIndexDesc;
  95import org.apache.hadoop.hive.ql.plan.AlterTableDesc;
  96import org.apache.hadoop.hive.ql.plan.AlterTableSimpleDesc;
  97import org.apache.hadoop.hive.ql.plan.CreateDatabaseDesc;
  98import org.apache.hadoop.hive.ql.plan.CreateIndexDesc;
  99import org.apache.hadoop.hive.ql.plan.CreateTableDesc;
 100import org.apache.hadoop.hive.ql.plan.CreateTableLikeDesc;
 101import org.apache.hadoop.hive.ql.plan.CreateViewDesc;
 102import org.apache.hadoop.hive.ql.plan.DDLWork;
 103import org.apache.hadoop.hive.ql.plan.DescDatabaseDesc;
 104import org.apache.hadoop.hive.ql.plan.DescFunctionDesc;
 105import org.apache.hadoop.hive.ql.plan.DescTableDesc;
 106import org.apache.hadoop.hive.ql.plan.DropDatabaseDesc;
 107import org.apache.hadoop.hive.ql.plan.DropIndexDesc;
 108import org.apache.hadoop.hive.ql.plan.DropTableDesc;
 109import org.apache.hadoop.hive.ql.plan.GrantDesc;
 110import org.apache.hadoop.hive.ql.plan.GrantRevokeRoleDDL;
 111import org.apache.hadoop.hive.ql.plan.LockTableDesc;
 112import org.apache.hadoop.hive.ql.plan.MsckDesc;
 113import org.apache.hadoop.hive.ql.plan.PrincipalDesc;
 114import org.apache.hadoop.hive.ql.plan.PrivilegeDesc;
 115import org.apache.hadoop.hive.ql.plan.PrivilegeObjectDesc;
 116import org.apache.hadoop.hive.ql.plan.RevokeDesc;
 117import org.apache.hadoop.hive.ql.plan.RoleDDLDesc;
 118import org.apache.hadoop.hive.ql.plan.ShowDatabasesDesc;
 119import org.apache.hadoop.hive.ql.plan.ShowFunctionsDesc;
 120import org.apache.hadoop.hive.ql.plan.ShowGrantDesc;
 121import org.apache.hadoop.hive.ql.plan.ShowIndexesDesc;
 122import org.apache.hadoop.hive.ql.plan.ShowLocksDesc;
 123import org.apache.hadoop.hive.ql.plan.ShowPartitionsDesc;
 124import org.apache.hadoop.hive.ql.plan.ShowTableStatusDesc;
 125import org.apache.hadoop.hive.ql.plan.ShowTablesDesc;
 126import org.apache.hadoop.hive.ql.plan.SwitchDatabaseDesc;
 127import org.apache.hadoop.hive.ql.plan.UnlockTableDesc;
 128import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes;
 129import org.apache.hadoop.hive.ql.plan.api.StageType;
 130import org.apache.hadoop.hive.ql.security.authorization.Privilege;
 131import org.apache.hadoop.hive.serde.Constants;
 132import org.apache.hadoop.hive.serde2.Deserializer;
 133import org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe;
 134import org.apache.hadoop.hive.serde2.SerDeException;
 135import org.apache.hadoop.hive.serde2.SerDeUtils;
 136import org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe;
 137import org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe;
 138import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
 139import org.apache.hadoop.hive.shims.HadoopShims;
 140import org.apache.hadoop.hive.shims.ShimLoader;
 141import org.apache.hadoop.util.ToolRunner;
 142
 143/**
 144 * DDLTask implementation.
 145 *
 146 **/
 147public class DDLTask extends Task<DDLWork> implements Serializable {
 148  private static final long serialVersionUID = 1L;
 149  private static final Log LOG = LogFactory.getLog("hive.ql.exec.DDLTask");
 150
 151  transient HiveConf conf;
 152  private static final int separator = Utilities.tabCode;
 153  private static final int terminator = Utilities.newLineCode;
 154
 155  // These are suffixes attached to intermediate directory names used in the
 156  // archiving / un-archiving process.
 157  private static String INTERMEDIATE_ARCHIVED_DIR_SUFFIX;
 158  private static String INTERMEDIATE_ORIGINAL_DIR_SUFFIX;
 159  private static String INTERMEDIATE_EXTRACTED_DIR_SUFFIX;
 160
 161  public DDLTask() {
 162    super();
 163  }
 164
 165  @Override
 166  public void initialize(HiveConf conf, QueryPlan queryPlan, DriverContext ctx) {
 167    super.initialize(conf, queryPlan, ctx);
 168    this.conf = conf;
 169
 170    INTERMEDIATE_ARCHIVED_DIR_SUFFIX =
 171      HiveConf.getVar(conf, ConfVars.METASTORE_INT_ARCHIVED);
 172    INTERMEDIATE_ORIGINAL_DIR_SUFFIX =
 173      HiveConf.getVar(conf, ConfVars.METASTORE_INT_ORIGINAL);
 174    INTERMEDIATE_EXTRACTED_DIR_SUFFIX =
 175      HiveConf.getVar(conf, ConfVars.METASTORE_INT_EXTRACTED);
 176  }
 177
 178  @Override
 179  public int execute(DriverContext driverContext) {
 180
 181    // Create the db
 182    Hive db;
 183    try {
 184      db = Hive.get(conf);
 185
 186      CreateDatabaseDesc createDatabaseDesc = work.getCreateDatabaseDesc();
 187      if (null != createDatabaseDesc) {
 188        return createDatabase(db, createDatabaseDesc);
 189      }
 190
 191      DropDatabaseDesc dropDatabaseDesc = work.getDropDatabaseDesc();
 192      if (dropDatabaseDesc != null) {
 193        return dropDatabase(db, dropDatabaseDesc);
 194      }
 195
 196      SwitchDatabaseDesc switchDatabaseDesc = work.getSwitchDatabaseDesc();
 197      if (switchDatabaseDesc != null) {
 198        return switchDatabase(db, switchDatabaseDesc);
 199      }
 200
 201      DescDatabaseDesc descDatabaseDesc = work.getDescDatabaseDesc();
 202      if (descDatabaseDesc != null) {
 203        return descDatabase(descDatabaseDesc);
 204      }
 205
 206      AlterDatabaseDesc alterDatabaseDesc = work.getAlterDatabaseDesc();
 207      if (alterDatabaseDesc != null) {
 208        return alterDatabase(alterDatabaseDesc);
 209      }
 210
 211      CreateTableDesc crtTbl = work.getCreateTblDesc();
 212      if (crtTbl != null) {
 213        return createTable(db, crtTbl);
 214      }
 215
 216      CreateIndexDesc crtIndex = work.getCreateIndexDesc();
 217      if (crtIndex != null) {
 218        return createIndex(db, crtIndex);
 219      }
 220
 221      AlterIndexDesc alterIndex = work.getAlterIndexDesc();
 222      if (alterIndex != null) {
 223        return alterIndex(db, alterIndex);
 224      }
 225
 226      DropIndexDesc dropIdx = work.getDropIdxDesc();
 227      if (dropIdx != null) {
 228        return dropIndex(db, dropIdx);
 229      }
 230
 231      CreateTableLikeDesc crtTblLike = work.getCreateTblLikeDesc();
 232      if (crtTblLike != null) {
 233        return createTableLike(db, crtTblLike);
 234      }
 235
 236      DropTableDesc dropTbl = work.getDropTblDesc();
 237      if (dropTbl != null) {
 238        return dropTable(db, dropTbl);
 239      }
 240
 241      AlterTableDesc alterTbl = work.getAlterTblDesc();
 242      if (alterTbl != null) {
 243        return alterTable(db, alterTbl);
 244      }
 245
 246      CreateViewDesc crtView = work.getCreateViewDesc();
 247      if (crtView != null) {
 248        return createView(db, crtView);
 249      }
 250
 251      AddPartitionDesc addPartitionDesc = work.getAddPartitionDesc();
 252      if (addPartitionDesc != null) {
 253        return addPartition(db, addPartitionDesc);
 254      }
 255
 256      AlterTableSimpleDesc simpleDesc = work.getAlterTblSimpleDesc();
 257      if (simpleDesc != null) {
 258        if (simpleDesc.getType() == AlterTableTypes.TOUCH) {
 259          return touch(db, simpleDesc);
 260        } else if (simpleDesc.getType() == AlterTableTypes.ARCHIVE) {
 261          return archive(db, simpleDesc, driverContext);
 262        } else if (simpleDesc.getType() == AlterTableTypes.UNARCHIVE) {
 263          return unarchive(db, simpleDesc);
 264        }
 265      }
 266
 267      MsckDesc msckDesc = work.getMsckDesc();
 268      if (msckDesc != null) {
 269        return msck(db, msckDesc);
 270      }
 271
 272      DescTableDesc descTbl = work.getDescTblDesc();
 273      if (descTbl != null) {
 274        return describeTable(db, descTbl);
 275      }
 276
 277      DescFunctionDesc descFunc = work.getDescFunctionDesc();
 278      if (descFunc != null) {
 279        return describeFunction(descFunc);
 280      }
 281
 282      ShowDatabasesDesc showDatabases = work.getShowDatabasesDesc();
 283      if (showDatabases != null) {
 284        return showDatabases(db, showDatabases);
 285      }
 286
 287      ShowTablesDesc showTbls = work.getShowTblsDesc();
 288      if (showTbls != null) {
 289        return showTables(db, showTbls);
 290      }
 291
 292      ShowTableStatusDesc showTblStatus = work.getShowTblStatusDesc();
 293      if (showTblStatus != null) {
 294        return showTableStatus(db, showTblStatus);
 295      }
 296
 297      ShowFunctionsDesc showFuncs = work.getShowFuncsDesc();
 298      if (showFuncs != null) {
 299        return showFunctions(showFuncs);
 300      }
 301
 302      ShowLocksDesc showLocks = work.getShowLocksDesc();
 303      if (showLocks != null) {
 304        return showLocks(showLocks);
 305      }
 306
 307      LockTableDesc lockTbl = work.getLockTblDesc();
 308      if (lockTbl != null) {
 309        return lockTable(lockTbl);
 310      }
 311
 312      UnlockTableDesc unlockTbl = work.getUnlockTblDesc();
 313      if (unlockTbl != null) {
 314        return unlockTable(unlockTbl);
 315      }
 316
 317      ShowPartitionsDesc showParts = work.getShowPartsDesc();
 318      if (showParts != null) {
 319        return showPartitions(db, showParts);
 320      }
 321
 322      RoleDDLDesc roleDDLDesc = work.getRoleDDLDesc();
 323      if (roleDDLDesc != null) {
 324        return roleDDL(roleDDLDesc);
 325      }
 326
 327      GrantDesc grantDesc = work.getGrantDesc();
 328      if (grantDesc != null) {
 329        return grantOrRevokePrivileges(grantDesc.getPrincipals(), grantDesc
 330            .getPrivileges(), grantDesc.getPrivilegeSubjectDesc(), grantDesc.getGrantor(), grantDesc.getGrantorType(), grantDesc.isGrantOption(), true);
 331      }
 332
 333      RevokeDesc revokeDesc = work.getRevokeDesc();
 334      if (revokeDesc != null) {
 335        return grantOrRevokePrivileges(revokeDesc.getPrincipals(), revokeDesc
 336            .getPrivileges(), revokeDesc.getPrivilegeSubjectDesc(), null, null, false, false);
 337      }
 338
 339      ShowGrantDesc showGrantDesc = work.getShowGrantDesc();
 340      if (showGrantDesc != null) {
 341        return showGrants(showGrantDesc);
 342      }
 343
 344      GrantRevokeRoleDDL grantOrRevokeRoleDDL = work.getGrantRevokeRoleDDL();
 345      if (grantOrRevokeRoleDDL != null) {
 346        return grantOrRevokeRole(grantOrRevokeRoleDDL);
 347      }
 348
 349      ShowIndexesDesc showIndexes = work.getShowIndexesDesc();
 350      if (showIndexes != null) {
 351        return showIndexes(db, showIndexes);
 352      }
 353
 354    } catch (InvalidTableException e) {
 355      console.printError("Table " + e.getTableName() + " does not exist");
 356      LOG.debug(stringifyException(e));
 357      return 1;
 358    } catch (HiveException e) {
 359      console.printError("FAILED: Error in metadata: " + e.getMessage(), "\n"
 360          + stringifyException(e));
 361      LOG.debug(stringifyException(e));
 362      return 1;
 363    } catch (Exception e) {
 364      console.printError("Failed with exception " + e.getMessage(), "\n"
 365          + stringifyException(e));
 366      return (1);
 367    }
 368    assert false;
 369    return 0;
 370  }
 371
 372  private int grantOrRevokeRole(GrantRevokeRoleDDL grantOrRevokeRoleDDL)
 373      throws HiveException {
 374    try {
 375      boolean grantRole = grantOrRevokeRoleDDL.getGrant();
 376      List<PrincipalDesc> principals = grantOrRevokeRoleDDL.getPrincipalDesc();
 377      List<String> roles = grantOrRevokeRoleDDL.getRoles();
 378      for (PrincipalDesc principal : principals) {
 379        String userName = principal.getName();
 380        for (String roleName : roles) {
 381          if (grantRole) {
 382            db.grantRole(roleName, userName, principal.getType(),
 383                grantOrRevokeRoleDDL.getGrantor(), grantOrRevokeRoleDDL
 384                    .getGrantorType(), grantOrRevokeRoleDDL.isGrantOption());
 385          } else {
 386            db.revokeRole(roleName, userName, principal.getType());
 387          }
 388        }
 389      }
 390    } catch (Exception e) {
 391      throw new HiveException(e);
 392    }
 393    return 0;
 394  }
 395
 396  private int showGrants(ShowGrantDesc showGrantDesc) throws HiveException {
 397    try {
 398      Path resFile = new Path(showGrantDesc.getResFile());
 399      FileSystem fs = resFile.getFileSystem(conf);
 400      DataOutput outStream = fs.create(resFile);
 401      PrincipalDesc principalDesc = showGrantDesc.getPrincipalDesc();
 402      PrivilegeObjectDesc hiveObjectDesc = showGrantDesc.getHiveObj();
 403      String principalName = principalDesc.getName();
 404      if (hiveObjectDesc == null) {
 405        List<HiveObjectPrivilege> users = db.showPrivilegeGrant(
 406            HiveObjectType.GLOBAL, principalName, principalDesc.getType(),
 407            null, null, null, null);
 408        if (users != null && users.size() > 0) {
 409          boolean first = true;
 410          for (HiveObjectPrivilege usr : users) {
 411            if (!first) {
 412              outStream.write(terminator);
 413            } else {
 414              first = false;
 415            }
 416
 417            writeGrantInfo(outStream, principalDesc.getType(), principalName,
 418                null, null, null, null, usr.getGrantInfo());
 419
 420          }
 421        }
 422      } else {
 423        String obj = hiveObjectDesc.getObject();
 424        boolean notFound = true;
 425        String dbName = null;
 426        String tableName = null;
 427        Table tableObj = null;
 428        Database dbObj = null;
 429
 430        if (hiveObjectDesc.getTable()) {
 431          String[] dbTab = obj.split("\\.");
 432          if (dbTab.length == 2) {
 433            dbName = dbTab[0];
 434            tableName = dbTab[1];
 435          } else {
 436            dbName = db.getCurrentDatabase();
 437            tableName = obj;
 438          }
 439          dbObj = db.getDatabase(dbName);
 440          tableObj = db.getTable(dbName, tableName);
 441          notFound = (dbObj == null || tableObj == null);
 442        } else {
 443          dbName = hiveObjectDesc.getObject();
 444          dbObj = db.getDatabase(dbName);
 445          notFound = (dbObj == null);
 446        }
 447        if (notFound) {
 448          throw new HiveException(obj + " can not be found");
 449        }
 450
 451        String partName = null;
 452        List<String> partValues = null;
 453        if (hiveObjectDesc.getPartSpec() != null) {
 454          partName = Warehouse
 455              .makePartName(hiveObjectDesc.getPartSpec(), false);
 456          partValues = Warehouse.getPartValuesFromPartName(partName);
 457        }
 458
 459        if (!hiveObjectDesc.getTable()) {
 460          // show database level privileges
 461          List<HiveObjectPrivilege> dbs = db.showPrivilegeGrant(HiveObjectType.DATABASE, principalName,
 462              principalDesc.getType(), dbName, null, null, null);
 463          if (dbs != null && dbs.size() > 0) {
 464            boolean first = true;
 465            for (HiveObjectPrivilege db : dbs) {
 466              if (!first) {
 467                outStream.write(terminator);
 468              } else {
 469                first = false;
 470              }
 471
 472              writeGrantInfo(outStream, principalDesc.getType(), principalName,
 473                  dbName, null, null, null, db.getGrantInfo());
 474
 475            }
 476          }
 477
 478        } else {
 479          if (showGrantDesc.getColumns() != null) {
 480            // show column level privileges
 481            for (String columnName : showGrantDesc.getColumns()) {
 482              List<HiveObjectPrivilege> columnss = db.showPrivilegeGrant(
 483                  HiveObjectType.COLUMN, principalName,
 484                  principalDesc.getType(), dbName, tableName, partValues,
 485                  columnName);
 486              if (columnss != null && columnss.size() > 0) {
 487                boolean first = true;
 488                for (HiveObjectPrivilege col : columnss) {
 489                  if (!first) {
 490                    outStream.write(terminator);
 491                  } else {
 492                    first = false;
 493                  }
 494
 495                  writeGrantInfo(outStream, principalDesc.getType(),
 496                      principalName, dbName, tableName, partName, columnName,
 497                      col.getGrantInfo());
 498                }
 499              }
 500            }
 501          } else if (hiveObjectDesc.getPartSpec() != null) {
 502            // show partition level privileges
 503            List<HiveObjectPrivilege> parts = db.showPrivilegeGrant(
 504                HiveObjectType.PARTITION, principalName, principalDesc
 505                    .getType(), dbName, tableName, partValues, null);
 506            if (parts != null && parts.size() > 0) {
 507              boolean first = true;
 508              for (HiveObjectPrivilege part : parts) {
 509                if (!first) {
 510                  outStream.write(terminator);
 511                } else {
 512                  first = false;
 513                }
 514
 515                writeGrantInfo(outStream, principalDesc.getType(),
 516                    principalName, dbName, tableName, partName, null, part.getGrantInfo());
 517
 518              }
 519            }
 520          } else {
 521            // show table level privileges
 522            List<HiveObjectPrivilege> tbls = db.showPrivilegeGrant(
 523                HiveObjectType.TABLE, principalName, principalDesc.getType(),
 524                dbName, tableName, null, null);
 525            if (tbls != null && tbls.size() > 0) {
 526              boolean first = true;
 527              for (HiveObjectPrivilege tbl : tbls) {
 528                if (!first) {
 529                  outStream.write(terminator);
 530                } else {
 531                  first = false;
 532                }
 533
 534                writeGrantInfo(outStream, principalDesc.getType(),
 535                    principalName, dbName, tableName, null, null, tbl.getGrantInfo());
 536
 537              }
 538            }
 539          }
 540        }
 541      }
 542      ((FSDataOutputStream) outStream).close();
 543    } catch (FileNotFoundException e) {
 544      LOG.info("show table status: " + stringifyException(e));
 545      return 1;
 546    } catch (IOException e) {
 547      LOG.info("show table status: " + stringifyException(e));
 548      return 1;
 549    } catch (Exception e) {
 550      e.printStackTrace();
 551      throw new HiveException(e);
 552    }
 553    return 0;
 554  }
 555
 556  private int grantOrRevokePrivileges(List<PrincipalDesc> principals,
 557      List<PrivilegeDesc> privileges, PrivilegeObjectDesc privSubjectDesc,
 558      String grantor, PrincipalType grantorType, boolean grantOption, boolean isGrant) {
 559    if (privileges == null || privileges.size() == 0) {
 560      console.printError("No privilege found.");
 561      return 1;
 562    }
 563
 564    String dbName = null;
 565    String tableName = null;
 566    Table tableObj = null;
 567    Database dbObj = null;
 568
 569    try {
 570
 571      if (privSubjectDesc != null) {
 572        if (privSubjectDesc.getPartSpec() != null && isGrant) {
 573          throw new HiveException("Grant does not support partition level.");
 574        }
 575        String obj = privSubjectDesc.getObject();
 576        boolean notFound = true;
 577        if (privSubjectDesc.getTable()) {
 578          String[] dbTab = obj.split("\\.");
 579          if (dbTab.length == 2) {
 580            dbName = dbTab[0];
 581            tableName = dbTab[1];
 582          } else {
 583            dbName = db.getCurrentDatabase();
 584            tableName = obj;
 585          }
 586          dbObj = db.getDatabase(dbName);
 587          tableObj = db.getTable(dbName, tableName);
 588          notFound = (dbObj == null || tableObj == null);
 589        } else {
 590          dbName = privSubjectDesc.getObject();
 591          dbObj = db.getDatabase(dbName);
 592          notFound = (dbObj == null);
 593        }
 594        if (notFound) {
 595          throw new HiveException(obj + " can not be found");
 596        }
 597      }
 598
 599      PrivilegeBag privBag = new PrivilegeBag();
 600      if (privSubjectDesc == null) {
 601        for (int idx = 0; idx < privileges.size(); idx++) {
 602          Privilege priv = privileges.get(idx).getPrivilege();
 603          if (privileges.get(idx).getColumns() != null
 604              && privileges.get(idx).getColumns().size() > 0) {
 605            throw new HiveException(
 606                "For user-level privileges, column sets should be null. columns="
 607                    + privileges.get(idx).getColumns().toString());
 608          }
 609
 610          privBag.addToPrivileges(new HiveObjectPrivilege(new HiveObjectRef(
 611              HiveObjectType.GLOBAL, null, null, null, null), null, null,
 612              new PrivilegeGrantInfo(priv.toString(), 0, grantor, grantorType,
 613                  grantOption)));
 614        }
 615      } else {
 616        org.apache.hadoop.hive.metastore.api.Partition partObj = null;
 617        List<String> partValues = null;
 618        if (tableObj != null) {
 619          if ((!tableObj.isPartitioned())
 620              && privSubjectDesc.getPartSpec() != null) {
 621            throw new HiveException(
 622                "Table is not partitioned, but partition name is present: partSpec="
 623                    + privSubjectDesc.getPartSpec().toString());
 624          }
 625
 626          if (privSubjectDesc.getPartSpec() != null) {
 627            partObj = db.getPartition(tableObj, privSubjectDesc.getPartSpec(),
 628                false).getTPartition();
 629            partValues = partObj.getValues();
 630          }
 631        }
 632
 633        for (PrivilegeDesc privDesc : privileges) {
 634          List<String> columns = privDesc.getColumns();
 635          Privilege priv = privDesc.getPrivilege();
 636          if (columns != null && columns.size() > 0) {
 637            if (!priv.supportColumnLevel()) {
 638              throw new HiveException(priv.toString()
 639                  + " does not support column level.");
 640            }
 641            if (privSubjectDesc == null || tableName == null) {
 642              throw new HiveException(
 643                  "For user-level/database-level privileges, column sets should be null. columns="
 644                      + columns);
 645            }
 646            for (int i = 0; i < columns.size(); i++) {
 647              privBag.addToPrivileges(new HiveObjectPrivilege(
 648                  new HiveObjectRef(HiveObjectType.COLUMN, dbName, tableName,
 649                      partValues, columns.get(i)), null, null,  new PrivilegeGrantInfo(priv.toString(), 0, grantor, grantorType, grantOption)));
 650            }
 651          } else {
 652            if (privSubjectDesc.getTable()) {
 653              if (privSubjectDesc.getPartSpec() != null) {
 654                privBag.addToPrivileges(new HiveObjectPrivilege(
 655                    new HiveObjectRef(HiveObjectType.PARTITION, dbName,
 656                        tableName, partValues, null), null, null,  new PrivilegeGrantInfo(priv.toString(), 0, grantor, grantorType, grantOption)));
 657              } else {
 658                privBag
 659                    .addToPrivileges(new HiveObjectPrivilege(
 660                        new HiveObjectRef(HiveObjectType.TABLE, dbName,
 661                            tableName, null, null), null, null, new PrivilegeGrantInfo(priv.toString(), 0, grantor, grantorType, grantOption)));
 662              }
 663            } else {
 664              privBag.addToPrivileges(new HiveObjectPrivilege(
 665                  new HiveObjectRef(HiveObjectType.DATABASE, dbName, null,
 666                      null, null), null, null, new PrivilegeGrantInfo(priv.toString(), 0, grantor, grantorType, grantOption)));
 667            }
 668          }
 669        }
 670      }
 671
 672      for (PrincipalDesc principal : principals) {
 673        for (int i = 0; i < privBag.getPrivileges().size(); i++) {
 674          HiveObjectPrivilege objPrivs = privBag.getPrivileges().get(i);
 675          objPrivs.setPrincipalName(principal.getName());
 676          objPrivs.setPrincipalType(principal.getType());
 677        }
 678        if (isGrant) {
 679          db.grantPrivileges(privBag);
 680        } else {
 681          db.revokePrivileges(privBag);
 682        }
 683      }
 684    } catch (Exception e) {
 685      console.printError("Error: " + e.getMessage());
 686      return 1;
 687    }
 688
 689    return 0;
 690  }
 691
 692  private int roleDDL(RoleDDLDesc roleDDLDesc) {
 693    RoleDDLDesc.RoleOperation operation = roleDDLDesc.getOperation();
 694    try {
 695      if (operation.equals(RoleDDLDesc.RoleOperation.CREATE_ROLE)) {
 696        db.createRole(roleDDLDesc.getName(), roleDDLDesc.getRoleOwnerName());
 697      } else if (operation.equals(RoleDDLDesc.RoleOperation.DROP_ROLE)) {
 698        db.dropRole(roleDDLDesc.getName());
 699      } else if (operation.equals(RoleDDLDesc.RoleOperation.SHOW_ROLE_GRANT)) {
 700        List<Role> roles = db.showRoleGrant(roleDDLDesc.getName(), roleDDLDesc
 701            .getPrincipalType());
 702        if (roles != null && roles.size() > 0) {
 703          Path resFile = new Path(roleDDLDesc.getResFile());
 704          FileSystem fs = resFile.getFileSystem(conf);
 705          DataOutput outStream = fs.create(resFile);
 706          for (Role role : roles) {
 707            outStream.writeBytes("role name:" + role.getRoleName());
 708            outStream.write(terminator);
 709          }
 710          ((FSDataOutputStream) outStream).close();
 711        }
 712      } else {
 713        throw new HiveException("Unkown role operation "
 714            + operation.getOperationName());
 715      }
 716    } catch (HiveException e) {
 717      console.printError("Error in role operation "
 718          + operation.getOperationName() + " on role name "
 719          + roleDDLDesc.getName() + ", error message " + e.getMessage());
 720      return 1;
 721    } catch (IOException e) {
 722      LOG.info("role ddl exception: " + stringifyException(e));
 723      return 1;
 724    }
 725
 726    return 0;
 727  }
 728
 729  private int alterDatabase(AlterDatabaseDesc alterDbDesc) throws HiveException {
 730
 731    String dbName = alterDbDesc.getDatabaseName();
 732    Database database = db.getDatabase(dbName);
 733    Map<String, String> newParams = alterDbDesc.getDatabaseProperties();
 734
 735    if (database != null) {
 736      Map<String, String> params = database.getParameters();
 737      // if both old and new params are not null, merge them
 738      if (params != null && newParams != null) {
 739        params.putAll(newParams);
 740        database.setParameters(params);
 741      } else { // if one of them is null, replace the old params with the new one
 742        database.setParameters(newParams);
 743      }
 744      db.alterDatabase(database.getName(), database);
 745    } else {
 746      throw new HiveException("ERROR: The database " + dbName + " does not exist.");
 747    }
 748    return 0;
 749  }
 750
 751  private int dropIndex(Hive db, DropIndexDesc dropIdx) throws HiveException {
 752    db.dropIndex(db.getCurrentDatabase(), dropIdx.getTableName(),
 753        dropIdx.getIndexName(), true);
 754    return 0;
 755  }
 756
 757  private int createIndex(Hive db, CreateIndexDesc crtIndex) throws HiveException {
 758
 759    if( crtIndex.getSerde() != null) {
 760      validateSerDe(crtIndex.getSerde());
 761    }
 762
 763    db
 764        .createIndex(
 765        crtIndex.getTableName(), crtIndex.getIndexName(), crtIndex.getIndexTypeHandlerClass(),
 766        crtIndex.getIndexedCols(), crtIndex.getIndexTableName(), crtIndex.getDeferredRebuild(),
 767        crtIndex.getInputFormat(), crtIndex.getOutputFormat(), crtIndex.getSerde(),
 768        crtIndex.getStorageHandler(), crtIndex.getLocation(), crtIndex.getIdxProps(), crtIndex.getTblProps(),
 769        crtIndex.getSerdeProps(), crtIndex.getCollItemDelim(), crtIndex.getFieldDelim(), crtIndex.getFieldEscape(),
 770        crtIndex.getLineDelim(), crtIndex.getMapKeyDelim(), crtIndex.getIndexComment()
 771        );
 772    return 0;
 773  }
 774
 775  private int alterIndex(Hive db, AlterIndexDesc alterIndex) throws HiveException {
 776    String dbName = alterIndex.getDbName();
 777    String baseTableName = alterIndex.getBaseTableName();
 778    String indexName = alterIndex.getIndexName();
 779    Index idx = db.getIndex(dbName, baseTableName, indexName);
 780
 781    if (alterIndex.getOp() == AlterIndexDesc.AlterIndexTypes.ADDPROPS) {
 782      idx.getParameters().putAll(alterIndex.getProps());
 783    } else {
 784      console.printError("Unsupported Alter commnad");
 785      return 1;
 786    }
 787
 788    // set last modified by properties
 789    if (!updateModifiedParameters(idx.getParameters(), conf)) {
 790      return 1;
 791    }
 792
 793    try {
 794      db.alterIndex(dbName, baseTableName, indexName, idx);
 795    } catch (InvalidOperationException e) {
 796      console.printError("Invalid alter operation: " + e.getMessage());
 797      LOG.info("alter index: " + stringifyException(e));
 798      return 1;
 799    } catch (HiveException e) {
 800      console.printError("Invalid alter operation: " + e.getMessage());
 801      return 1;
 802    }
 803    return 0;
 804  }
 805
 806  /**
 807   * Add a partition to a table.
 808   *
 809   * @param db
 810   *          Database to add the partition to.
 811   * @param addPartitionDesc
 812   *          Add this partition.
 813   * @return Returns 0 when execution succeeds and above 0 if it fails.
 814   * @throws HiveException
 815   */
 816  /**
 817   * Add a partition to a table.
 818   *
 819   * @param db
 820   *          Database to add the partition to.
 821   * @param addPartitionDesc
 822   *          Add this partition.
 823   * @return Returns 0 when execution succeeds and above 0 if it fails.
 824   * @throws HiveException
 825   */
 826  private int addPartition(Hive db, AddPartitionDesc addPartitionDesc) throws HiveException {
 827
 828    Table tbl = db.getTable(addPartitionDesc.getDbName(), addPartitionDesc.getTableName());
 829
 830    validateAlterTableType(tbl, AlterTableDesc.AlterTableTypes.ADDPARTITION);
 831
 832    // If the add partition was created with IF NOT EXISTS, then we should
 833    // not throw an error if the specified part does exist.
 834    Partition checkPart = db.getPartition(tbl, addPartitionDesc.getPartSpec(), false);
 835    if (checkPart != null && addPartitionDesc.getIfNotExists()) {
 836      return 0;
 837    }
 838
 839    if (addPartitionDesc.getLocation() == null) {
 840      db.createPartition(tbl, addPartitionDesc.getPartSpec());
 841    } else {
 842      // set partition path relative to table
 843      db.createPartition(tbl, addPartitionDesc.getPartSpec(), new Path(tbl
 844          .getPath(), addPartitionDesc.getLocation()));
 845    }
 846
 847    Partition part = db
 848        .getPartition(tbl, addPartitionDesc.getPartSpec(), false);
 849    work.getOutputs().add(new WriteEntity(part));
 850
 851    return 0;
 852  }
 853
 854  /**
 855   * Rewrite the partition's metadata and force the pre/post execute hooks to
 856   * be fired.
 857   *
 858   * @param db
 859   * @param touchDesc
 860   * @return
 861   * @throws HiveException
 862   */
 863  private int touch(Hive db, AlterTableSimpleDesc touchDesc)
 864      throws HiveException {
 865
 866    String dbName = touchDesc.getDbName();
 867    String tblName = touchDesc.getTableName();
 868
 869    Table tbl = db.getTable(dbName, tblName);
 870
 871    validateAlterTableType(tbl, AlterTableDesc.AlterTableTypes.TOUCH);
 872
 873    if (touchDesc.getPartSpec() == null) {
 874      try {
 875        db.alterTable(tblName, tbl);
 876      } catch (InvalidOperationException e) {
 877        throw new HiveException("Uable to update table");
 878      }
 879      work.getInputs().add(new ReadEntity(tbl));
 880      work.getOutputs().add(new WriteEntity(tbl));
 881    } else {
 882      Partition part = db.getPartition(tbl, touchDesc.getPartSpec(), false);
 883      if (part == null) {
 884        throw new HiveException("Specified partition does not exist");
 885      }
 886      try {
 887        db.alterPartition(tblName, part);
 888      } catch (InvalidOperationException e) {
 889        throw new HiveException(e);
 890      }
 891      work.getInputs().add(new ReadEntity(part));
 892      work.getOutputs().add(new WriteEntity(part));
 893    }
 894    return 0;
 895  }
 896
 897  /**
 898   * Determines whether a partition has been archived
 899   *
 900   * @param p
 901   * @return
 902   */
 903
 904  private boolean isArchived(Partition p) {
 905    Map<String, String> params = p.getParameters();
 906    if ("true".equalsIgnoreCase(params.get(
 907        org.apache.hadoop.hive.metastore.api.Constants.IS_ARCHIVED))) {
 908      return true;
 909    } else {
 910      return false;
 911    }
 912  }
 913
 914  private void setIsArchived(Partition p, boolean state) {
 915    Map<String, String> params = p.getParameters();
 916    if (state) {
 917      params.put(org.apache.hadoop.hive.metastore.api.Constants.IS_ARCHIVED,
 918          "true");
 919    } else {
 920      params.remove(org.apache.hadoop.hive.metastore.api.Constants.IS_ARCHIVED);
 921    }
 922  }
 923
 924  private String getOriginalLocation(Partition p) {
 925    Map<String, String> params = p.getParameters();
 926    return params.get(
 927        org.apache.hadoop.hive.metastore.api.Constants.ORIGINAL_LOCATION);
 928  }
 929
 930  private void setOriginalLocation(Partition p, String loc) {
 931    Map<String, String> params = p.getParameters();
 932    if (loc == null) {
 933      params.remove(org.apache.hadoop.hive.metastore.api.Constants.ORIGINAL_LOCATION);
 934    } else {
 935      params.put(org.apache.hadoop.hive.metastore.api.Constants.ORIGINAL_LOCATION, loc);
 936    }
 937  }
 938
 939  // Returns only the path component of the URI
 940  private String getArchiveDirOnly(Path parentDir, String archiveName) {
 941    URI parentUri = parentDir.toUri();
 942    Path harDir = new Path(parentUri.getPath(), archiveName);
 943    return harDir.toString();
 944  }
 945
 946  /**
 947   * Sets the appropriate attributes in the supplied Partition object to mark
 948   * it as archived. Note that the metastore is not touched - a separate
 949   * call to alter_partition is needed.
 950   *
 951   * @param p - the partition object to modify
 952   * @param parentDir - the parent directory of the archive, which is the
 953   * original directory that the partition's files resided in
 954   * @param dirInArchive - the directory within the archive file that contains
 955   * the partitions files
 956   * @param archiveName - the name of the archive
 957   * @throws URISyntaxException
 958   */
 959  private void setArchived(Partition p, Path parentDir, String dirInArchive, String archiveName)
 960      throws URISyntaxException {
 961    assert(isArchived(p) == false);
 962    Map<String, String> params = p.getParameters();
 963
 964    URI parentUri = parentDir.toUri();
 965    String parentHost = parentUri.getHost();
 966    String harHost = null;
 967    if (parentHost == null) {
 968     harHost = "";
 969    } else {
 970      harHost = parentUri.getScheme() + "-" + parentHost;
 971    }
 972
 973    // harUri is used to access the partition's files, which are in the archive
 974    // The format of the RI is something like:
 975    // har://underlyingfsscheme-host:port/archivepath
 976    URI harUri = null;
 977    if (dirInArchive.length() == 0) {
 978      harUri = new URI("har", parentUri.getUserInfo(), harHost, parentUri.getPort(),
 979          getArchiveDirOnly(parentDir, archiveName),
 980          parentUri.getQuery(), parentUri.getFragment());
 981    } else {
 982      harUri = new URI("har", parentUri.getUserInfo(), harHost, parentUri.getPort(),
 983          new Path(getArchiveDirOnly(parentDir, archiveName), dirInArchive).toUri().getPath(),
 984          parentUri.getQuery(), parentUri.getFragment());
 985    }
 986    setIsArchived(p, true);
 987    setOriginalLocation(p, parentDir.toString());
 988    p.setLocation(harUri.toString());
 989  }
 990
 991  /**
 992   * Sets the appropriate attributes in the supplied Partition object to mark
 993   * it as not archived. Note that the metastore is not touched - a separate
 994   * call to alter_partition is needed.
 995   *
 996   * @param p - the partition to modify
 997   */
 998  private void setUnArchived(Partition p) {
 999    assert(isArchived(p) == true);
1000    String parentDir = getOriginalLocation(p);
1001    setIsArchived(p, false);
1002    setOriginalLocation(p, null);
1003    assert(parentDir != null);
1004    p.setLocation(parentDir);
1005  }
1006
1007  private boolean pathExists(Path p) throws HiveException {
1008    try {
1009      FileSystem fs = p.getFileSystem(conf);
1010      return fs.exists(p);
1011    } catch (IOException e) {
1012      throw new HiveException(e);
1013    }
1014  }
1015
1016  private void moveDir(FileSystem fs, Path from, Path to) throws HiveException {
1017    try {
1018      if (!fs.rename(from, to)) {
1019        throw new HiveException("Moving " + from + " to " + to + " failed!");
1020      }
1021    } catch (IOException e) {
1022      throw new HiveException(e);
1023    }
1024  }
1025
1026  private void deleteDir(Path dir) throws HiveException {
1027    try {
1028      Warehouse wh = new Warehouse(conf);
1029      wh.deleteDir(dir, true);
1030    } catch (MetaException e) {
1031      throw new HiveException(e);
1032    }
1033  }
1034
1035  private int archive(Hive db, AlterTableSimpleDesc simpleDesc, DriverContext driverContext)
1036      throws HiveException {
1037    String dbName = simpleDesc.getDbName();
1038    String tblName = simpleDesc.getTableName();
1039
1040    Table tbl = db.getTable(dbName, tblName);
1041    validateAlterTableType(tbl, AlterTableDesc.AlterTableTypes.ARCHIVE);
1042
1043    Map<String, String> partSpec = simpleDesc.getPartSpec();
1044    Partition p = db.getPartition(tbl, partSpec, false);
1045
1046    if (tbl.getTableType() != TableType.MANAGED_TABLE) {
1047      throw new HiveException("ARCHIVE can only be performed on managed tables");
1048    }
1049
1050    if (p == null) {
1051      throw new HiveException("Specified partition does not exist");
1052    }
1053
1054    if (isArchived(p)) {
1055      // If there were a failure right after the metadata was updated in an
1056      // archiving operation, it's possible that the original, unarchived files
1057      // weren't deleted.
1058      Path originalDir = new Path(getOriginalLocation(p));
1059      Path leftOverIntermediateOriginal = new Path(originalDir.getParent(),
1060          originalDir.getName() + INTERMEDIATE_ORIGINAL_DIR_SUFFIX);
1061
1062      if (pathExists(leftOverIntermediateOriginal)) {
1063        console.printInfo("Deleting " + leftOverIntermediateOriginal +
1064        " left over from a previous archiving operation");
1065        deleteDir(leftOverIntermediateOriginal);
1066      }
1067
1068      throw new HiveException("Specified partition is already archived");
1069    }
1070
1071    Path originalDir = p.getPartitionPath();
1072    Path intermediateArchivedDir = new Path(originalDir.getParent(),
1073        originalDir.getName() + INTERMEDIATE_ARCHIVED_DIR_SUFFIX);
1074    Path intermediateOriginalDir = new Path(originalDir.getParent(),
1075        originalDir.getName() + INTERMEDIATE_ORIGINAL_DIR_SUFFIX);
1076    String archiveName = "data.har";
1077    FileSystem fs = null;
1078    try {
1079      fs = originalDir.getFileSystem(conf);
1080    } catch (IOException e) {
1081      throw new HiveException(e);
1082    }
1083
1084    // The following steps seem roundabout, but they are meant to aid in
1085    // recovery if a failure occurs and to keep a consistent state in the FS
1086
1087    // Steps:
1088    // 1. Create the archive in a temporary folder
1089    // 2. Move the archive dir to an intermediate dir that is in at the same
1090    //    dir as the original partition dir. Call the new dir
1091    //    intermediate-archive.
1092    // 3. Rename the original partition dir to an intermediate dir. Call the
1093    //    renamed dir intermediate-original
1094    // 4. Rename intermediate-archive to the original partition dir
1095    // 5. Change the metadata
1096    // 6. Delete the original partition files in intermediate-original
1097
1098    // The original partition files are deleted after the metadata change
1099    // because the presence of those files are used to indicate whether
1100    // the original partition directory contains archived or unarchived files.
1101
1102    // Create an archived version of the partition in a directory ending in
1103    // ARCHIVE_INTERMEDIATE_DIR_SUFFIX that's the same level as the partition,
1104    // if it does not already exist. If it does exist, we assume the dir is good
1105    // to use as the move operation that created it is atomic.
1106    if (!pathExists(intermediateArchivedDir) &&
1107        !pathExists(intermediateOriginalDir)) {
1108
1109      // First create the archive in a tmp dir so that if the job fails, the
1110      // bad files don't pollute the filesystem
1111      Path tmpDir = new Path(driverContext.getCtx().getExternalTmpFileURI(originalDir.toUri()), "partlevel");
1112
1113      console.printInfo("Creating " + archiveName + " for " + originalDir.toString());
1114      console.printInfo("in " + tmpDir);
1115      console.printInfo("Please wait... (this may take a while)");
1116
1117      // Create the Hadoop archive
1118      HadoopShims shim = ShimLoader.getHadoopShims();
1119      int ret=0;
1120      try {
1121        ret = shim.createHadoopArchive(conf, originalDir, tmpDir, archiveName);
1122      } catch (Exception e) {
1123        throw new HiveException(e);
1124      }
1125      if (ret != 0) {
1126        throw new HiveException("Error while creating HAR");
1127      }
1128      // Move from the tmp dir to an intermediate directory, in the same level as
1129      // the partition directory. e.g. .../hr=12-intermediate-archived
1130      try {
1131        console.printInfo("Moving " + tmpDir + " to " + intermediateArchivedDir);
1132        if (pathExists(intermediateArchivedDir)) {
1133          throw new HiveException("The intermediate archive directory already exists.");
1134        }
1135        fs.rename(tmpDir, intermediateArchivedDir);
1136      } catch (IOException e) {
1137        throw new HiveException("Error while moving tmp directory");
1138      }
1139    } else {
1140      if (pathExists(intermediateArchivedDir)) {
1141        console.printInfo("Intermediate archive directory " + intermediateArchivedDir +
1142        " already exists. Assuming it contains an archived version of the partition");
1143      }
1144    }
1145
1146    // If we get to here, we know that we've archived the partition files, but
1147    // they may be in the original partition location, or in the intermediate
1148    // original dir.
1149
1150    // Move the original parent directory to the intermediate original directory
1151    // if the move hasn't been made already
1152    if (!pathExists(intermediateOriginalDir)) {
1153      console.printInfo("Moving " + originalDir + " to " +
1154          intermediateOriginalDir);
1155      moveDir(fs, originalDir, intermediateOriginalDir);
1156    } else {
1157      console.printInfo(intermediateOriginalDir + " already exists. " +
1158          "Assuming it contains the original files in the partition");
1159    }
1160
1161    // If there's a failure from here to when the metadata is updated,
1162    // there will be no data in the partition, or an error while trying to read
1163    // the partition (if the archive files have been moved to the original
1164    // partition directory.) But re-running the archive command will allow
1165    // recovery
1166
1167    // Move the intermediate archived directory to the original parent directory
1168    if (!pathExists(originalDir)) {
1169      console.printInfo("Moving " + intermediateArchivedDir + " to " +
1170          originalDir);
1171      moveDir(fs, intermediateArchivedDir, originalDir);
1172    } else {
1173      console.printInfo(originalDir + " already exists. " +
1174          "Assuming it contains the archived version of the partition");
1175    }
1176
1177    // Record this change in the metastore
1178    try {
1179      boolean parentSettable =
1180        conf.getBoolVar(HiveConf.ConfVars.HIVEHARPARENTDIRSETTABLE);
1181
1182      // dirInArchive is the directory within the archive that has all the files
1183      // for this partition. With older versions of Hadoop, archiving a
1184      // a directory would produce the same directory structure
1185      // in the archive. So if you created myArchive.har of /tmp/myDir, the
1186      // files in /tmp/myDir would be located under myArchive.har/tmp/myDir/*
1187      // In this case, dirInArchive should be tmp/myDir
1188
1189      // With newer versions of Hadoop, the parent directory could be specified.
1190      // Assuming the parent directory was set to /tmp/myDir when creating the
1191      // archive, the files can be found under myArchive.har/*
1192      // In this case, dirInArchive should be empty
1193
1194      String dirInArchive = "";
1195      if (!parentSettable) {
1196        dirInArchive = originalDir.toUri().getPath();
1197        if(dirInArchive.length() > 1 && dirInArchive.charAt(0)=='/') {
1198          dirInArchive = dirInArchive.substring(1);
1199        }
1200      }
1201      setArchived(p, originalDir, dirInArchive, archiveName);
1202      db.alterPartition(tblName, p);
1203    } catch (Exception e) {
1204      throw new HiveException("Unable to change the partition info for HAR", e);
1205    }
1206
1207    // If a failure occurs here, the directory containing the original files
1208    // will not be deleted. The user will run ARCHIVE again to clear this up
1209    deleteDir(intermediateOriginalDir);
1210
1211
1212    return 0;
1213  }
1214
1215  private int unarchive(Hive db, AlterTableSimpleDesc simpleDesc)
1216      throws HiveException {
1217    String dbName = simpleDesc.getDbName();
1218    String tblName = simpleDesc.getTableName();
1219
1220    Table tbl = db.getTable(dbName, tblName);
1221    validateAlterTableType(tbl, AlterTableDesc.AlterTableTypes.UNARCHIVE);
1222
1223    // Means user specified a table, not a partition
1224    if (simpleDesc.getPartSpec() == null) {
1225      throw new HiveException("ARCHIVE is for partitions only");
1226    }
1227
1228    Map<String, String> partSpec = simpleDesc.getPartSpec();
1229    Partition p = db.getPartition(tbl, partSpec, false);
1230
1231    if (tbl.getTableType() != TableType.MANAGED_TABLE) {
1232      throw new HiveException("UNARCHIVE can only be performed on managed tables");
1233    }
1234
1235    if (p == null) {
1236      throw new HiveException("Specified partition does not exist");
1237    }
1238
1239    if (!isArchived(p)) {
1240      Path location = new Path(p.getLocation());
1241      Path leftOverArchiveDir = new Path(location.getParent(),
1242          location.getName() + INTERMEDIATE_ARCHIVED_DIR_SUFFIX);
1243
1244      if (pathExists(leftOverArchiveDir)) {
1245        console.printInfo("Deleting " + leftOverArchiveDir + " left over " +
1246        "from a previous unarchiving operation");
1247        deleteDir(leftOverArchiveDir);
1248      }
1249
1250      throw new HiveException("Specified partition is not archived");
1251    }
1252
1253    Path originalLocation = new Path(getOriginalLocation(p));
1254    Path sourceDir = new Path(p.getLocation());
1255    Path intermediateArchiveDir = new Path(originalLocation.getParent(),
1256        originalLocation.getName() + INTERMEDIATE_ARCHIVED_DIR_SUFFIX);
1257    Path intermediateExtractedDir = new Path(originalLocation.getParent(),
1258        originalLocation.getName() + INTERMEDIATE_EXTRACTED_DIR_SUFFIX);
1259
1260    Path tmpDir = new Path(driverContext
1261          .getCtx()
1262          .getExternalTmpFileURI(originalLocation.toUri()));
1263
1264    FileSystem fs = null;
1265    try {
1266      fs = tmpDir.getFileSystem(conf);
1267      // Verify that there are no files in the tmp dir, because if there are, it
1268      // would be copied to the partition
1269      FileStatus [] filesInTmpDir = fs.listStatus(tmpDir);
1270      if (filesInTmpDir != null && filesInTmpDir.length != 0) {
1271        for (FileStatus file : filesInTmpDir) {
1272          console.printInfo(file.getPath().toString());
1273        }
1274        throw new HiveException("Temporary directory " + tmpDir + " is not empty");
1275      }
1276
1277    } catch (IOException e) {
1278      throw new HiveException(e);
1279    }
1280
1281    // Some sanity checks
1282    if (originalLocation == null) {
1283      throw new HiveException("Missing archive data in the partition");
1284    }
1285    if (!"har".equals(sourceDir.toUri().getScheme())) {
1286      throw new HiveException("Location should refer to a HAR");
1287    }
1288
1289    // Clarification of terms:
1290    // - The originalLocation directory represents the original directory of the
1291    //   partition's files. They now contain an archived version of those files
1292    //   eg. hdfs:/warehouse/myTable/ds=1/
1293    // - The source directory is the directory containing all the files that
1294    //   should be in the partition. e.g. har:/warehouse/myTable/ds=1/myTable.har/
1295    //   Note the har:/ scheme
1296
1297    // Steps:
1298    // 1. Extract the archive in a temporary folder
1299    // 2. Move the archive dir to an intermediate dir that is in at the same
1300    //    dir as originalLocation. Call the new dir intermediate-extracted.
1301    // 3. Rename the original partition dir to an intermediate dir. Call the
1302    //    renamed dir intermediate-archive
1303    // 4. Rename intermediate-extracted to the original partition dir
1304    // 5. Change the metadata
1305    // 6. Delete the archived partition files in intermediate-archive
1306
1307    if (!pathExists(intermediateExtractedDir) &&
1308        !pathExists(intermediateArchiveDir)) {
1309      try {
1310
1311        // Copy the files out of 

Large files files are truncated, but you can click here to view the full file