PageRenderTime 33ms CodeModel.GetById 11ms app.highlight 19ms RepoModel.GetById 1ms app.codeStats 0ms

/tags/release-0.2.0-rc0/hive/external/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreChecker.java

#
Java | 258 lines | 179 code | 39 blank | 40 comment | 1 complexity | 884aedc15482638d713ec0a9523cd7b8 MD5 | raw file
  1/**
  2 * Licensed to the Apache Software Foundation (ASF) under one
  3 * or more contributor license agreements.  See the NOTICE file
  4 * distributed with this work for additional information
  5 * regarding copyright ownership.  The ASF licenses this file
  6 * to you under the Apache License, Version 2.0 (the
  7 * "License"); you may not use this file except in compliance
  8 * with the License.  You may obtain a copy of the License at
  9 *
 10 *     http://www.apache.org/licenses/LICENSE-2.0
 11 *
 12 * Unless required by applicable law or agreed to in writing, software
 13 * distributed under the License is distributed on an "AS IS" BASIS,
 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 15 * See the License for the specific language governing permissions and
 16 * limitations under the License.
 17 */
 18package org.apache.hadoop.hive.ql.metadata;
 19
 20import java.io.IOException;
 21import java.util.ArrayList;
 22import java.util.HashMap;
 23import java.util.List;
 24import java.util.Map;
 25
 26import junit.framework.TestCase;
 27
 28import org.apache.hadoop.fs.FileSystem;
 29import org.apache.hadoop.fs.Path;
 30import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
 31import org.apache.hadoop.hive.metastore.api.Database;
 32import org.apache.hadoop.hive.metastore.api.FieldSchema;
 33import org.apache.hadoop.hive.metastore.api.MetaException;
 34import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
 35import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat;
 36import org.apache.hadoop.hive.serde.Constants;
 37import org.apache.hadoop.mapred.TextInputFormat;
 38import org.apache.thrift.TException;
 39
 40/**
 41 * TestHiveMetaStoreChecker.
 42 *
 43 */
 44public class TestHiveMetaStoreChecker extends TestCase {
 45
 46  private Hive hive;
 47  private FileSystem fs;
 48  private HiveMetaStoreChecker checker = null;
 49
 50  private final String dbName = "dbname";
 51  private final String tableName = "tablename";
 52
 53  private final String partDateName = "partdate";
 54  private final String partCityName = "partcity";
 55
 56  private List<FieldSchema> partCols;
 57  private List<Map<String, String>> parts;
 58
 59  @Override
 60  protected void setUp() throws Exception {
 61    super.setUp();
 62    hive = Hive.get();
 63    checker = new HiveMetaStoreChecker(hive);
 64
 65    partCols = new ArrayList<FieldSchema>();
 66    partCols.add(new FieldSchema(partDateName, Constants.STRING_TYPE_NAME, ""));
 67    partCols.add(new FieldSchema(partCityName, Constants.STRING_TYPE_NAME, ""));
 68
 69    parts = new ArrayList<Map<String, String>>();
 70    Map<String, String> part1 = new HashMap<String, String>();
 71    part1.put(partDateName, "2008-01-01");
 72    part1.put(partCityName, "london");
 73    parts.add(part1);
 74    Map<String, String> part2 = new HashMap<String, String>();
 75    part2.put(partDateName, "2008-01-02");
 76    part2.put(partCityName, "stockholm");
 77    parts.add(part2);
 78
 79    // cleanup
 80    hive.dropTable(dbName, tableName, true, true);
 81    try {
 82      hive.dropDatabase(dbName);
 83    } catch (NoSuchObjectException e) {
 84      // ignore
 85    }
 86  }
 87
 88  @Override
 89  protected void tearDown() throws Exception {
 90    super.tearDown();
 91    Hive.closeCurrent();
 92  }
 93
 94  public void testTableCheck() throws HiveException, MetaException,
 95      IOException, TException, AlreadyExistsException {
 96
 97    CheckResult result = new CheckResult();
 98    checker.checkMetastore(dbName, null, null, result);
 99    // we haven't added anything so should return an all ok
100    assertTrue(result.getTablesNotInMs().isEmpty());
101    assertTrue(result.getTablesNotOnFs().isEmpty());
102    assertTrue(result.getPartitionsNotOnFs().isEmpty());
103    assertTrue(result.getPartitionsNotInMs().isEmpty());
104
105    // check table only, should not exist in ms
106    result = new CheckResult();
107    checker.checkMetastore(dbName, tableName, null, result);
108    assertEquals(1, result.getTablesNotInMs().size());
109    assertEquals(tableName, result.getTablesNotInMs().get(0));
110    assertTrue(result.getTablesNotOnFs().isEmpty());
111    assertTrue(result.getPartitionsNotOnFs().isEmpty());
112    assertTrue(result.getPartitionsNotInMs().isEmpty());
113
114    Database db = new Database();
115    db.setName(dbName);
116    hive.createDatabase(db);
117
118    Table table = new Table(dbName, tableName);
119    table.setDbName(dbName);
120    table.setInputFormatClass(TextInputFormat.class);
121    table.setOutputFormatClass(HiveIgnoreKeyTextOutputFormat.class);
122
123    hive.createTable(table);
124    // now we've got a table, check that it works
125    // first check all (1) tables
126    result = new CheckResult();
127    checker.checkMetastore(dbName, null, null, result);
128    assertTrue(result.getTablesNotInMs().isEmpty());
129    assertTrue(result.getTablesNotOnFs().isEmpty());
130    assertTrue(result.getPartitionsNotOnFs().isEmpty());
131    assertTrue(result.getPartitionsNotInMs().isEmpty());
132
133    // then let's check the one we know about
134    result = new CheckResult();
135    checker.checkMetastore(dbName, tableName, null, result);
136    assertTrue(result.getTablesNotInMs().isEmpty());
137    assertTrue(result.getTablesNotOnFs().isEmpty());
138    assertTrue(result.getPartitionsNotOnFs().isEmpty());
139    assertTrue(result.getPartitionsNotInMs().isEmpty());
140
141    // remove the table folder
142    fs = table.getPath().getFileSystem(hive.getConf());
143    fs.delete(table.getPath(), true);
144
145    // now this shouldn't find the path on the fs
146    result = new CheckResult();
147    checker.checkMetastore(dbName, tableName, null, result);
148    assertTrue(result.getTablesNotInMs().isEmpty());
149    assertEquals(1, result.getTablesNotOnFs().size());
150    assertEquals(tableName, result.getTablesNotOnFs().get(0));
151    assertTrue(result.getPartitionsNotOnFs().isEmpty());
152    assertTrue(result.getPartitionsNotInMs().isEmpty());
153
154    // put it back and one additional table
155    fs.mkdirs(table.getPath());
156    Path fakeTable = table.getPath().getParent().suffix(
157        Path.SEPARATOR + "faketable");
158    fs.mkdirs(fakeTable);
159
160    // find the extra table
161    result = new CheckResult();
162    checker.checkMetastore(dbName, null, null, result);
163    assertEquals(1, result.getTablesNotInMs().size());
164    assertEquals(fakeTable.getName(), result.getTablesNotInMs().get(0));
165    assertTrue(result.getTablesNotOnFs().isEmpty());
166    assertTrue(result.getPartitionsNotOnFs().isEmpty());
167    assertTrue(result.getPartitionsNotInMs().isEmpty());
168
169    // create a new external table
170    hive.dropTable(dbName, tableName);
171    table.setProperty("EXTERNAL", "TRUE");
172    hive.createTable(table);
173
174    // should return all ok
175    result = new CheckResult();
176    checker.checkMetastore(dbName, null, null, result);
177    assertTrue(result.getTablesNotInMs().isEmpty());
178    assertTrue(result.getTablesNotOnFs().isEmpty());
179    assertTrue(result.getPartitionsNotOnFs().isEmpty());
180    assertTrue(result.getPartitionsNotInMs().isEmpty());
181  }
182
183  public void testPartitionsCheck() throws HiveException, MetaException,
184      IOException, TException, AlreadyExistsException {
185
186    Database db = new Database();
187    db.setName(dbName);
188    hive.createDatabase(db);
189
190    Table table = new Table(dbName, tableName);
191    table.setDbName(dbName);
192    table.setInputFormatClass(TextInputFormat.class);
193    table.setOutputFormatClass(HiveIgnoreKeyTextOutputFormat.class);
194    table.setPartCols(partCols);
195
196    hive.createTable(table);
197    table = hive.getTable(dbName, tableName);
198
199    for (Map<String, String> partSpec : parts) {
200      hive.createPartition(table, partSpec);
201    }
202
203    CheckResult result = new CheckResult();
204    checker.checkMetastore(dbName, tableName, null, result);
205    // all is well
206    assertTrue(result.getTablesNotInMs().isEmpty());
207    assertTrue(result.getTablesNotOnFs().isEmpty());
208    assertTrue(result.getPartitionsNotOnFs().isEmpty());
209    assertTrue(result.getPartitionsNotInMs().isEmpty());
210
211    List<Partition> partitions = hive.getPartitions(table);
212    assertEquals(2, partitions.size());
213    Partition partToRemove = partitions.get(0);
214    Path partToRemovePath = new Path(partToRemove.getDataLocation().toString());
215    fs = partToRemovePath.getFileSystem(hive.getConf());
216    fs.delete(partToRemovePath, true);
217
218    result = new CheckResult();
219    checker.checkMetastore(dbName, tableName, null, result);
220    // missing one partition on fs
221    assertTrue(result.getTablesNotInMs().isEmpty());
222    assertTrue(result.getTablesNotOnFs().isEmpty());
223    assertEquals(1, result.getPartitionsNotOnFs().size());
224    assertEquals(partToRemove.getName(), result.getPartitionsNotOnFs().get(0)
225        .getPartitionName());
226    assertEquals(partToRemove.getTable().getTableName(), result
227        .getPartitionsNotOnFs().get(0).getTableName());
228    assertTrue(result.getPartitionsNotInMs().isEmpty());
229
230    List<Map<String, String>> partsCopy = new ArrayList<Map<String, String>>();
231    partsCopy.add(partitions.get(1).getSpec());
232    // check only the partition that exists, all should be well
233    result = new CheckResult();
234    checker.checkMetastore(dbName, tableName, partsCopy, result);
235    assertTrue(result.getTablesNotInMs().isEmpty());
236    assertTrue(result.getTablesNotOnFs().isEmpty());
237    assertTrue(result.getPartitionsNotOnFs().isEmpty());
238    assertTrue(result.getPartitionsNotInMs().isEmpty());
239
240    // put the other one back
241    fs.mkdirs(partToRemovePath);
242
243    // add a partition dir on fs
244    Path fakePart = new Path(table.getDataLocation().toString(),
245        "fakepartition=fakevalue");
246    fs.mkdirs(fakePart);
247
248    checker.checkMetastore(dbName, tableName, null, result);
249    // one extra partition
250    assertTrue(result.getTablesNotInMs().isEmpty());
251    assertTrue(result.getTablesNotOnFs().isEmpty());
252    assertTrue(result.getPartitionsNotOnFs().isEmpty());
253    assertEquals(1, result.getPartitionsNotInMs().size());
254    assertEquals(fakePart.getName(), result.getPartitionsNotInMs().get(0)
255        .getPartitionName());
256  }
257
258}