PageRenderTime 78ms CodeModel.GetById 23ms app.highlight 50ms RepoModel.GetById 2ms app.codeStats 0ms

/tags/release-0.0.0-rc0/hive/external/service/src/test/org/apache/hadoop/hive/service/TestHiveServer.java

#
Java | 335 lines | 245 code | 46 blank | 44 comment | 17 complexity | 6e20eccfa9a8a138149eb8e02071cc07 MD5 | raw file
  1/**
  2 * Licensed to the Apache Software Foundation (ASF) under one
  3 * or more contributor license agreements.  See the NOTICE file
  4 * distributed with this work for additional information
  5 * regarding copyright ownership.  The ASF licenses this file
  6 * to you under the Apache License, Version 2.0 (the
  7 * "License"); you may not use this file except in compliance
  8 * with the License.  You may obtain a copy of the License at
  9 *
 10 *     http://www.apache.org/licenses/LICENSE-2.0
 11 *
 12 * Unless required by applicable law or agreed to in writing, software
 13 * distributed under the License is distributed on an "AS IS" BASIS,
 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 15 * See the License for the specific language governing permissions and
 16 * limitations under the License.
 17 */
 18package org.apache.hadoop.hive.service;
 19
 20import java.util.List;
 21import java.util.Properties;
 22
 23import junit.framework.TestCase;
 24
 25import org.apache.hadoop.conf.Configuration;
 26import org.apache.hadoop.fs.Path;
 27import org.apache.hadoop.hive.conf.HiveConf;
 28import org.apache.hadoop.hive.metastore.api.FieldSchema;
 29import org.apache.hadoop.hive.metastore.api.Schema;
 30import org.apache.hadoop.hive.serde.Constants;
 31import org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe;
 32import org.apache.hadoop.io.BytesWritable;
 33import org.apache.thrift.protocol.TBinaryProtocol;
 34import org.apache.thrift.protocol.TProtocol;
 35import org.apache.thrift.transport.TSocket;
 36import org.apache.thrift.transport.TTransport;
 37
 38/**
 39 * TestHiveServer.
 40 *
 41 */
 42public class TestHiveServer extends TestCase {
 43
 44  private HiveInterface client;
 45  private static final String host = "localhost";
 46  private static final int port = 10000;
 47  private final Path dataFilePath;
 48
 49  private static String tableName = "testhivedrivertable";
 50  private final HiveConf conf;
 51  private boolean standAloneServer = false;
 52  private TTransport transport;
 53
 54  public TestHiveServer(String name) {
 55    super(name);
 56    conf = new HiveConf(TestHiveServer.class);
 57    String dataFileDir = conf.get("test.data.files").replace('\\', '/')
 58        .replace("c:", "");
 59    dataFilePath = new Path(dataFileDir, "kv1.txt");
 60    // See data/conf/hive-site.xml
 61    String paramStr = System.getProperty("test.service.standalone.server");
 62    if (paramStr != null && paramStr.equals("true")) {
 63      standAloneServer = true;
 64    }
 65  }
 66
 67  @Override
 68  protected void setUp() throws Exception {
 69    super.setUp();
 70
 71    if (standAloneServer) {
 72      try {
 73        transport = new TSocket(host, port);
 74        TProtocol protocol = new TBinaryProtocol(transport);
 75        client = new HiveClient(protocol);
 76        transport.open();
 77      } catch (Throwable e) {
 78        e.printStackTrace();
 79      }
 80    } else {
 81      client = new HiveServer.HiveServerHandler();
 82    }
 83  }
 84
 85  @Override
 86  protected void tearDown() throws Exception {
 87    super.tearDown();
 88    if (standAloneServer) {
 89      transport.close();
 90    }
 91  }
 92
 93  public void testExecute() throws Exception {
 94    try {
 95      client.execute("set hive.support.concurrency = false");
 96      client.execute("drop table " + tableName);
 97    } catch (Exception ex) {
 98    }
 99
100    try {
101      client.execute("create table " + tableName + " (num int)");
102      client.execute("load data local inpath '" + dataFilePath.toString()
103          + "' into table " + tableName);
104      client.execute("select count(1) as cnt from " + tableName);
105      String row = client.fetchOne();
106      assertEquals(row, "500");
107
108      Schema hiveSchema = client.getSchema();
109      List<FieldSchema> listFields = hiveSchema.getFieldSchemas();
110      assertEquals(listFields.size(), 1);
111      assertEquals(listFields.get(0).getName(), "cnt");
112      assertEquals(listFields.get(0).getType(), "bigint");
113
114      Schema thriftSchema = client.getThriftSchema();
115      List<FieldSchema> listThriftFields = thriftSchema.getFieldSchemas();
116      assertEquals(listThriftFields.size(), 1);
117      assertEquals(listThriftFields.get(0).getName(), "cnt");
118      assertEquals(listThriftFields.get(0).getType(), "i64");
119
120      client.execute("drop table " + tableName);
121    } catch (Throwable t) {
122      t.printStackTrace();
123    }
124  }
125
126  public void notestExecute() throws Exception {
127    try {
128      client.execute("set hive.support.concurrency = false");
129      client.execute("drop table " + tableName);
130    } catch (Exception ex) {
131    }
132
133    client.execute("create table " + tableName + " (num int)");
134    client.execute("load data local inpath '" + dataFilePath.toString()
135        + "' into table " + tableName);
136    client.execute("select count(1) from " + tableName);
137    String row = client.fetchOne();
138    assertEquals(row, "500");
139    client.execute("drop table " + tableName);
140    transport.close();
141  }
142
143  public void testNonHiveCommand() throws Exception {
144    try {
145      client.execute("set hive.support.concurrency = false");
146      client.execute("drop table " + tableName);
147    } catch (Exception ex) {
148    }
149
150    client.execute("create table " + tableName + " (num int)");
151    client.execute("load data local inpath '" + dataFilePath.toString()
152        + "' into table " + tableName);
153
154    // Command not part of HiveQL - verify no results
155    client.execute("SET hive.mapred.mode = nonstrict");
156
157    Schema schema = client.getSchema();
158    assertEquals(schema.getFieldSchemasSize(), 0);
159    assertEquals(schema.getPropertiesSize(), 0);
160
161    Schema thriftschema = client.getThriftSchema();
162    assertEquals(thriftschema.getFieldSchemasSize(), 0);
163    assertEquals(thriftschema.getPropertiesSize(), 0);
164
165    assertEquals(client.fetchOne(), "");
166    assertEquals(client.fetchN(10).size(), 0);
167    assertEquals(client.fetchAll().size(), 0);
168
169    // Execute Hive query and fetch
170    client.execute("select * from " + tableName + " limit 10");
171    client.fetchOne();
172
173    // Re-execute command not part of HiveQL - verify still no results
174    client.execute("SET hive.mapred.mode = nonstrict");
175
176    schema = client.getSchema();
177    assertEquals(schema.getFieldSchemasSize(), 0);
178    assertEquals(schema.getPropertiesSize(), 0);
179
180    thriftschema = client.getThriftSchema();
181    assertEquals(thriftschema.getFieldSchemasSize(), 0);
182    assertEquals(thriftschema.getPropertiesSize(), 0);
183
184    assertEquals(client.fetchOne(), "");
185    assertEquals(client.fetchN(10).size(), 0);
186    assertEquals(client.fetchAll().size(), 0);
187
188    // Cleanup
189    client.execute("drop table " + tableName);
190  }
191
192  /**
193   * Test metastore call.
194   */
195  public void testMetastore() throws Exception {
196    try {
197      client.execute("set hive.support.concurrency = false");
198      client.execute("drop table " + tableName);
199    } catch (Exception ex) {
200    }
201
202    client.execute("create table " + tableName + " (num int)");
203    List<String> tabs = client.get_tables("default", tableName);
204    assertEquals(tabs.get(0), tableName);
205    client.execute("drop table " + tableName);
206  }
207
208  /**
209   * Test cluster status retrieval.
210   */
211  public void testGetClusterStatus() throws Exception {
212    HiveClusterStatus clusterStatus = client.getClusterStatus();
213    assertNotNull(clusterStatus);
214    assertTrue(clusterStatus.getTaskTrackers() >= 0);
215    assertTrue(clusterStatus.getMapTasks() >= 0);
216    assertTrue(clusterStatus.getReduceTasks() >= 0);
217    assertTrue(clusterStatus.getMaxMapTasks() >= 0);
218    assertTrue(clusterStatus.getMaxReduceTasks() >= 0);
219    assertTrue(clusterStatus.getState() == JobTrackerState.INITIALIZING
220        || clusterStatus.getState() == JobTrackerState.RUNNING);
221  }
222
223  /**
224   *
225   */
226  public void testFetch() throws Exception {
227    // create and populate a table with 500 rows.
228    try {
229      client.execute("set hive.support.concurrency = false");
230      client.execute("drop table " + tableName);
231    } catch (Exception ex) {
232    }
233    client.execute("create table " + tableName + " (key int, value string)");
234    client.execute("load data local inpath '" + dataFilePath.toString()
235        + "' into table " + tableName);
236
237    try {
238      // fetchAll test
239      client.execute("select key, value from " + tableName);
240      assertEquals(client.fetchAll().size(), 500);
241      assertEquals(client.fetchAll().size(), 0);
242
243      // fetchOne test
244      client.execute("select key, value from " + tableName);
245      for (int i = 0; i < 500; i++) {
246        String str = client.fetchOne();
247        if (str.equals("")) {
248          assertTrue(false);
249        }
250      }
251      assertEquals(client.fetchOne(), "");
252
253      // fetchN test
254      client.execute("select key, value from " + tableName);
255      assertEquals(client.fetchN(499).size(), 499);
256      assertEquals(client.fetchN(499).size(), 1);
257      assertEquals(client.fetchN(499).size(), 0);
258    } catch (Throwable e) {
259      e.printStackTrace();
260    }
261  }
262
263  public void testDynamicSerde() throws Exception {
264    try {
265      client.execute("set hive.support.concurrency = false");
266      client.execute("drop table " + tableName);
267    } catch (Exception ex) {
268    }
269
270    client.execute("create table " + tableName + " (key int, value string)");
271    client.execute("load data local inpath '" + dataFilePath.toString()
272        + "' into table " + tableName);
273    // client.execute("select key, count(1) from " + tableName +
274    // " where key > 10 group by key");
275    String sql = "select key, value from " + tableName + " where key > 10";
276    client.execute(sql);
277
278    // Instantiate DynamicSerDe
279    DynamicSerDe ds = new DynamicSerDe();
280    Properties dsp = new Properties();
281    dsp.setProperty(Constants.SERIALIZATION_FORMAT,
282        org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol.class
283            .getName());
284    dsp.setProperty(
285        org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_NAME,
286        "result");
287    String serDDL = new String("struct result { ");
288    List<FieldSchema> schema = client.getThriftSchema().getFieldSchemas();
289    for (int pos = 0; pos < schema.size(); pos++) {
290      if (pos != 0) {
291        serDDL = serDDL.concat(",");
292      }
293      serDDL = serDDL.concat(schema.get(pos).getType());
294      serDDL = serDDL.concat(" ");
295      serDDL = serDDL.concat(schema.get(pos).getName());
296    }
297    serDDL = serDDL.concat("}");
298
299    dsp.setProperty(Constants.SERIALIZATION_DDL, serDDL);
300    dsp.setProperty(Constants.SERIALIZATION_LIB, ds.getClass().toString());
301    dsp.setProperty(Constants.FIELD_DELIM, "9");
302    ds.initialize(new Configuration(), dsp);
303
304    String row = client.fetchOne();
305    Object o = ds.deserialize(new BytesWritable(row.getBytes()));
306
307    assertEquals(o.getClass().toString(), "class java.util.ArrayList");
308    List<?> lst = (List<?>) o;
309    assertEquals(lst.get(0), 238);
310
311    // TODO: serde doesn't like underscore -- struct result { string _c0}
312    sql = "select count(1) as c from " + tableName;
313    client.execute(sql);
314    row = client.fetchOne();
315
316    serDDL = new String("struct result { ");
317    schema = client.getThriftSchema().getFieldSchemas();
318    for (int pos = 0; pos < schema.size(); pos++) {
319      if (pos != 0) {
320        serDDL = serDDL.concat(",");
321      }
322      serDDL = serDDL.concat(schema.get(pos).getType());
323      serDDL = serDDL.concat(" ");
324      serDDL = serDDL.concat(schema.get(pos).getName());
325    }
326    serDDL = serDDL.concat("}");
327
328    dsp.setProperty(Constants.SERIALIZATION_DDL, serDDL);
329    // Need a new DynamicSerDe instance - re-initialization is not supported.
330    ds = new DynamicSerDe();
331    ds.initialize(new Configuration(), dsp);
332    o = ds.deserialize(new BytesWritable(row.getBytes()));
333  }
334
335}