PageRenderTime 41ms CodeModel.GetById 14ms RepoModel.GetById 0ms app.codeStats 0ms

/tags/release-0.2.0-rc0/hive/external/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreChecker.java

#
Java | 258 lines | 179 code | 39 blank | 40 comment | 1 complexity | 884aedc15482638d713ec0a9523cd7b8 MD5 | raw file
Possible License(s): Apache-2.0, BSD-3-Clause, JSON, CPL-1.0
  1. /**
  2. * Licensed to the Apache Software Foundation (ASF) under one
  3. * or more contributor license agreements. See the NOTICE file
  4. * distributed with this work for additional information
  5. * regarding copyright ownership. The ASF licenses this file
  6. * to you under the Apache License, Version 2.0 (the
  7. * "License"); you may not use this file except in compliance
  8. * with the License. You may obtain a copy of the License at
  9. *
  10. * http://www.apache.org/licenses/LICENSE-2.0
  11. *
  12. * Unless required by applicable law or agreed to in writing, software
  13. * distributed under the License is distributed on an "AS IS" BASIS,
  14. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. * See the License for the specific language governing permissions and
  16. * limitations under the License.
  17. */
  18. package org.apache.hadoop.hive.ql.metadata;
  19. import java.io.IOException;
  20. import java.util.ArrayList;
  21. import java.util.HashMap;
  22. import java.util.List;
  23. import java.util.Map;
  24. import junit.framework.TestCase;
  25. import org.apache.hadoop.fs.FileSystem;
  26. import org.apache.hadoop.fs.Path;
  27. import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
  28. import org.apache.hadoop.hive.metastore.api.Database;
  29. import org.apache.hadoop.hive.metastore.api.FieldSchema;
  30. import org.apache.hadoop.hive.metastore.api.MetaException;
  31. import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
  32. import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat;
  33. import org.apache.hadoop.hive.serde.Constants;
  34. import org.apache.hadoop.mapred.TextInputFormat;
  35. import org.apache.thrift.TException;
  36. /**
  37. * TestHiveMetaStoreChecker.
  38. *
  39. */
  40. public class TestHiveMetaStoreChecker extends TestCase {
  41. private Hive hive;
  42. private FileSystem fs;
  43. private HiveMetaStoreChecker checker = null;
  44. private final String dbName = "dbname";
  45. private final String tableName = "tablename";
  46. private final String partDateName = "partdate";
  47. private final String partCityName = "partcity";
  48. private List<FieldSchema> partCols;
  49. private List<Map<String, String>> parts;
  50. @Override
  51. protected void setUp() throws Exception {
  52. super.setUp();
  53. hive = Hive.get();
  54. checker = new HiveMetaStoreChecker(hive);
  55. partCols = new ArrayList<FieldSchema>();
  56. partCols.add(new FieldSchema(partDateName, Constants.STRING_TYPE_NAME, ""));
  57. partCols.add(new FieldSchema(partCityName, Constants.STRING_TYPE_NAME, ""));
  58. parts = new ArrayList<Map<String, String>>();
  59. Map<String, String> part1 = new HashMap<String, String>();
  60. part1.put(partDateName, "2008-01-01");
  61. part1.put(partCityName, "london");
  62. parts.add(part1);
  63. Map<String, String> part2 = new HashMap<String, String>();
  64. part2.put(partDateName, "2008-01-02");
  65. part2.put(partCityName, "stockholm");
  66. parts.add(part2);
  67. // cleanup
  68. hive.dropTable(dbName, tableName, true, true);
  69. try {
  70. hive.dropDatabase(dbName);
  71. } catch (NoSuchObjectException e) {
  72. // ignore
  73. }
  74. }
  75. @Override
  76. protected void tearDown() throws Exception {
  77. super.tearDown();
  78. Hive.closeCurrent();
  79. }
  80. public void testTableCheck() throws HiveException, MetaException,
  81. IOException, TException, AlreadyExistsException {
  82. CheckResult result = new CheckResult();
  83. checker.checkMetastore(dbName, null, null, result);
  84. // we haven't added anything so should return an all ok
  85. assertTrue(result.getTablesNotInMs().isEmpty());
  86. assertTrue(result.getTablesNotOnFs().isEmpty());
  87. assertTrue(result.getPartitionsNotOnFs().isEmpty());
  88. assertTrue(result.getPartitionsNotInMs().isEmpty());
  89. // check table only, should not exist in ms
  90. result = new CheckResult();
  91. checker.checkMetastore(dbName, tableName, null, result);
  92. assertEquals(1, result.getTablesNotInMs().size());
  93. assertEquals(tableName, result.getTablesNotInMs().get(0));
  94. assertTrue(result.getTablesNotOnFs().isEmpty());
  95. assertTrue(result.getPartitionsNotOnFs().isEmpty());
  96. assertTrue(result.getPartitionsNotInMs().isEmpty());
  97. Database db = new Database();
  98. db.setName(dbName);
  99. hive.createDatabase(db);
  100. Table table = new Table(dbName, tableName);
  101. table.setDbName(dbName);
  102. table.setInputFormatClass(TextInputFormat.class);
  103. table.setOutputFormatClass(HiveIgnoreKeyTextOutputFormat.class);
  104. hive.createTable(table);
  105. // now we've got a table, check that it works
  106. // first check all (1) tables
  107. result = new CheckResult();
  108. checker.checkMetastore(dbName, null, null, result);
  109. assertTrue(result.getTablesNotInMs().isEmpty());
  110. assertTrue(result.getTablesNotOnFs().isEmpty());
  111. assertTrue(result.getPartitionsNotOnFs().isEmpty());
  112. assertTrue(result.getPartitionsNotInMs().isEmpty());
  113. // then let's check the one we know about
  114. result = new CheckResult();
  115. checker.checkMetastore(dbName, tableName, null, result);
  116. assertTrue(result.getTablesNotInMs().isEmpty());
  117. assertTrue(result.getTablesNotOnFs().isEmpty());
  118. assertTrue(result.getPartitionsNotOnFs().isEmpty());
  119. assertTrue(result.getPartitionsNotInMs().isEmpty());
  120. // remove the table folder
  121. fs = table.getPath().getFileSystem(hive.getConf());
  122. fs.delete(table.getPath(), true);
  123. // now this shouldn't find the path on the fs
  124. result = new CheckResult();
  125. checker.checkMetastore(dbName, tableName, null, result);
  126. assertTrue(result.getTablesNotInMs().isEmpty());
  127. assertEquals(1, result.getTablesNotOnFs().size());
  128. assertEquals(tableName, result.getTablesNotOnFs().get(0));
  129. assertTrue(result.getPartitionsNotOnFs().isEmpty());
  130. assertTrue(result.getPartitionsNotInMs().isEmpty());
  131. // put it back and one additional table
  132. fs.mkdirs(table.getPath());
  133. Path fakeTable = table.getPath().getParent().suffix(
  134. Path.SEPARATOR + "faketable");
  135. fs.mkdirs(fakeTable);
  136. // find the extra table
  137. result = new CheckResult();
  138. checker.checkMetastore(dbName, null, null, result);
  139. assertEquals(1, result.getTablesNotInMs().size());
  140. assertEquals(fakeTable.getName(), result.getTablesNotInMs().get(0));
  141. assertTrue(result.getTablesNotOnFs().isEmpty());
  142. assertTrue(result.getPartitionsNotOnFs().isEmpty());
  143. assertTrue(result.getPartitionsNotInMs().isEmpty());
  144. // create a new external table
  145. hive.dropTable(dbName, tableName);
  146. table.setProperty("EXTERNAL", "TRUE");
  147. hive.createTable(table);
  148. // should return all ok
  149. result = new CheckResult();
  150. checker.checkMetastore(dbName, null, null, result);
  151. assertTrue(result.getTablesNotInMs().isEmpty());
  152. assertTrue(result.getTablesNotOnFs().isEmpty());
  153. assertTrue(result.getPartitionsNotOnFs().isEmpty());
  154. assertTrue(result.getPartitionsNotInMs().isEmpty());
  155. }
  156. public void testPartitionsCheck() throws HiveException, MetaException,
  157. IOException, TException, AlreadyExistsException {
  158. Database db = new Database();
  159. db.setName(dbName);
  160. hive.createDatabase(db);
  161. Table table = new Table(dbName, tableName);
  162. table.setDbName(dbName);
  163. table.setInputFormatClass(TextInputFormat.class);
  164. table.setOutputFormatClass(HiveIgnoreKeyTextOutputFormat.class);
  165. table.setPartCols(partCols);
  166. hive.createTable(table);
  167. table = hive.getTable(dbName, tableName);
  168. for (Map<String, String> partSpec : parts) {
  169. hive.createPartition(table, partSpec);
  170. }
  171. CheckResult result = new CheckResult();
  172. checker.checkMetastore(dbName, tableName, null, result);
  173. // all is well
  174. assertTrue(result.getTablesNotInMs().isEmpty());
  175. assertTrue(result.getTablesNotOnFs().isEmpty());
  176. assertTrue(result.getPartitionsNotOnFs().isEmpty());
  177. assertTrue(result.getPartitionsNotInMs().isEmpty());
  178. List<Partition> partitions = hive.getPartitions(table);
  179. assertEquals(2, partitions.size());
  180. Partition partToRemove = partitions.get(0);
  181. Path partToRemovePath = new Path(partToRemove.getDataLocation().toString());
  182. fs = partToRemovePath.getFileSystem(hive.getConf());
  183. fs.delete(partToRemovePath, true);
  184. result = new CheckResult();
  185. checker.checkMetastore(dbName, tableName, null, result);
  186. // missing one partition on fs
  187. assertTrue(result.getTablesNotInMs().isEmpty());
  188. assertTrue(result.getTablesNotOnFs().isEmpty());
  189. assertEquals(1, result.getPartitionsNotOnFs().size());
  190. assertEquals(partToRemove.getName(), result.getPartitionsNotOnFs().get(0)
  191. .getPartitionName());
  192. assertEquals(partToRemove.getTable().getTableName(), result
  193. .getPartitionsNotOnFs().get(0).getTableName());
  194. assertTrue(result.getPartitionsNotInMs().isEmpty());
  195. List<Map<String, String>> partsCopy = new ArrayList<Map<String, String>>();
  196. partsCopy.add(partitions.get(1).getSpec());
  197. // check only the partition that exists, all should be well
  198. result = new CheckResult();
  199. checker.checkMetastore(dbName, tableName, partsCopy, result);
  200. assertTrue(result.getTablesNotInMs().isEmpty());
  201. assertTrue(result.getTablesNotOnFs().isEmpty());
  202. assertTrue(result.getPartitionsNotOnFs().isEmpty());
  203. assertTrue(result.getPartitionsNotInMs().isEmpty());
  204. // put the other one back
  205. fs.mkdirs(partToRemovePath);
  206. // add a partition dir on fs
  207. Path fakePart = new Path(table.getDataLocation().toString(),
  208. "fakepartition=fakevalue");
  209. fs.mkdirs(fakePart);
  210. checker.checkMetastore(dbName, tableName, null, result);
  211. // one extra partition
  212. assertTrue(result.getTablesNotInMs().isEmpty());
  213. assertTrue(result.getTablesNotOnFs().isEmpty());
  214. assertTrue(result.getPartitionsNotOnFs().isEmpty());
  215. assertEquals(1, result.getPartitionsNotInMs().size());
  216. assertEquals(fakePart.getName(), result.getPartitionsNotInMs().get(0)
  217. .getPartitionName());
  218. }
  219. }