PageRenderTime 64ms CodeModel.GetById 27ms RepoModel.GetById 1ms app.codeStats 0ms

/hbase2/src/test/java/site/ycsb/db/hbase2/HBaseClient2Test.java

https://github.com/brianfrankcooper/YCSB
Java | 322 lines | 240 code | 43 blank | 39 comment | 13 complexity | eab3c852d0e9bc0d0c7b015a20f100c0 MD5 | raw file
  1. /**
  2. * Licensed under the Apache License, Version 2.0 (the "License"); you
  3. * may not use this file except in compliance with the License. You
  4. * may obtain a copy of the License at
  5. *
  6. * http://www.apache.org/licenses/LICENSE-2.0
  7. *
  8. * Unless required by applicable law or agreed to in writing, software
  9. * distributed under the License is distributed on an "AS IS" BASIS,
  10. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  11. * implied. See the License for the specific language governing
  12. * permissions and limitations under the License. See accompanying
  13. * LICENSE file.
  14. */
  15. package site.ycsb.db.hbase2;
  16. import static org.junit.Assert.assertArrayEquals;
  17. import static site.ycsb.workloads.CoreWorkload.TABLENAME_PROPERTY;
  18. import static site.ycsb.workloads.CoreWorkload.TABLENAME_PROPERTY_DEFAULT;
  19. import static org.junit.Assert.assertEquals;
  20. import static org.junit.Assert.assertFalse;
  21. import static org.junit.Assert.assertTrue;
  22. import static org.junit.Assert.fail;
  23. import static org.junit.Assume.assumeTrue;
  24. import site.ycsb.ByteIterator;
  25. import site.ycsb.Status;
  26. import site.ycsb.StringByteIterator;
  27. import site.ycsb.measurements.Measurements;
  28. import site.ycsb.workloads.CoreWorkload;
  29. import org.apache.hadoop.conf.Configuration;
  30. import org.apache.hadoop.hbase.HBaseTestingUtility;
  31. import org.apache.hadoop.hbase.TableName;
  32. import org.apache.hadoop.hbase.client.Get;
  33. import org.apache.hadoop.hbase.client.Put;
  34. import org.apache.hadoop.hbase.client.Result;
  35. import org.apache.hadoop.hbase.client.Table;
  36. import org.apache.hadoop.hbase.util.Bytes;
  37. import org.junit.After;
  38. import org.junit.AfterClass;
  39. import org.junit.BeforeClass;
  40. import org.junit.Ignore;
  41. import org.junit.Test;
  42. import java.nio.ByteBuffer;
  43. import java.util.ArrayList;
  44. import java.util.Collections;
  45. import java.util.HashMap;
  46. import java.util.List;
  47. import java.util.Properties;
  48. import java.util.Vector;
  49. /**
  50. * Integration tests for the YCSB HBase 2 client using an HBase minicluster.
  51. */
  52. public class HBaseClient2Test {
  53. private final static String COLUMN_FAMILY = "cf";
  54. private static HBaseTestingUtility testingUtil;
  55. private HBaseClient2 client;
  56. private Table table = null;
  57. private String tableName;
  58. private static boolean isWindows() {
  59. final String os = System.getProperty("os.name");
  60. return os.startsWith("Windows");
  61. }
  62. /**
  63. * Creates a mini-cluster for use in these tests.
  64. * This is a heavy-weight operation, so invoked only once for the test class.
  65. */
  66. @BeforeClass
  67. public static void setUpClass() throws Exception {
  68. // Minicluster setup fails on Windows with an UnsatisfiedLinkError.
  69. // Skip if windows.
  70. assumeTrue(!isWindows());
  71. testingUtil = HBaseTestingUtility.createLocalHTU();
  72. testingUtil.startMiniCluster();
  73. }
  74. /**
  75. * Tears down mini-cluster.
  76. */
  77. @AfterClass
  78. public static void tearDownClass() throws Exception {
  79. if (testingUtil != null) {
  80. testingUtil.shutdownMiniCluster();
  81. }
  82. }
  83. /**
  84. * Re-create the table for each test. Using default properties.
  85. */
  86. public void setUp() throws Exception {
  87. setUp(new Properties());
  88. }
  89. /**
  90. * Re-create the table for each test. Using custom properties.
  91. */
  92. public void setUp(Properties p) throws Exception {
  93. client = new HBaseClient2();
  94. client.setConfiguration(new Configuration(testingUtil.getConfiguration()));
  95. p.setProperty("columnfamily", COLUMN_FAMILY);
  96. Measurements.setProperties(p);
  97. final CoreWorkload workload = new CoreWorkload();
  98. workload.init(p);
  99. tableName = p.getProperty(TABLENAME_PROPERTY, TABLENAME_PROPERTY_DEFAULT);
  100. table = testingUtil.createTable(TableName.valueOf(tableName), Bytes.toBytes(COLUMN_FAMILY));
  101. client.setProperties(p);
  102. client.init();
  103. }
  104. @After
  105. public void tearDown() throws Exception {
  106. table.close();
  107. testingUtil.deleteTable(TableName.valueOf(tableName));
  108. }
  109. @Test
  110. public void testRead() throws Exception {
  111. setUp();
  112. final String rowKey = "row1";
  113. final Put p = new Put(Bytes.toBytes(rowKey));
  114. p.addColumn(Bytes.toBytes(COLUMN_FAMILY),
  115. Bytes.toBytes("column1"), Bytes.toBytes("value1"));
  116. p.addColumn(Bytes.toBytes(COLUMN_FAMILY),
  117. Bytes.toBytes("column2"), Bytes.toBytes("value2"));
  118. table.put(p);
  119. final HashMap<String, ByteIterator> result = new HashMap<String, ByteIterator>();
  120. final Status status = client.read(tableName, rowKey, null, result);
  121. assertEquals(Status.OK, status);
  122. assertEquals(2, result.size());
  123. assertEquals("value1", result.get("column1").toString());
  124. assertEquals("value2", result.get("column2").toString());
  125. }
  126. @Test
  127. public void testReadMissingRow() throws Exception {
  128. setUp();
  129. final HashMap<String, ByteIterator> result = new HashMap<String, ByteIterator>();
  130. final Status status = client.read(tableName, "Missing row", null, result);
  131. assertEquals(Status.NOT_FOUND, status);
  132. assertEquals(0, result.size());
  133. }
  134. @Test
  135. public void testScan() throws Exception {
  136. setUp();
  137. // Fill with data
  138. final String colStr = "row_number";
  139. final byte[] col = Bytes.toBytes(colStr);
  140. final int n = 10;
  141. final List<Put> puts = new ArrayList<Put>(n);
  142. for(int i = 0; i < n; i++) {
  143. final byte[] key = Bytes.toBytes(String.format("%05d", i));
  144. final byte[] value = java.nio.ByteBuffer.allocate(4).putInt(i).array();
  145. final Put p = new Put(key);
  146. p.addColumn(Bytes.toBytes(COLUMN_FAMILY), col, value);
  147. puts.add(p);
  148. }
  149. table.put(puts);
  150. // Test
  151. final Vector<HashMap<String, ByteIterator>> result =
  152. new Vector<HashMap<String, ByteIterator>>();
  153. // Scan 5 records, skipping the first
  154. client.scan(tableName, "00001", 5, null, result);
  155. assertEquals(5, result.size());
  156. for(int i = 0; i < 5; i++) {
  157. final HashMap<String, ByteIterator> row = result.get(i);
  158. assertEquals(1, row.size());
  159. assertTrue(row.containsKey(colStr));
  160. final byte[] bytes = row.get(colStr).toArray();
  161. final ByteBuffer buf = ByteBuffer.wrap(bytes);
  162. final int rowNum = buf.getInt();
  163. assertEquals(i + 1, rowNum);
  164. }
  165. }
  166. @Test
  167. public void testScanWithValueFilteringUsingDefaultProperties() throws Exception {
  168. testScanWithValueFiltering(null, null, 100, new byte[][] {
  169. Bytes.fromHex("0000"), Bytes.fromHex("1111"), Bytes.fromHex("2222"), Bytes.fromHex("3333"),
  170. Bytes.fromHex("4444"), Bytes.fromHex("5555"), Bytes.fromHex("6666"), Bytes.fromHex("7777"),
  171. });
  172. }
  173. @Test
  174. public void testScanWithValueFilteringOperationLessOrEqual() throws Exception {
  175. testScanWithValueFiltering("less_or_equal", "3333", 100, new byte[][] {
  176. Bytes.fromHex("0000"), Bytes.fromHex("1111"), Bytes.fromHex("2222"), Bytes.fromHex("3333"),
  177. });
  178. }
  179. @Test
  180. public void testScanWithValueFilteringOperationEqual() throws Exception {
  181. testScanWithValueFiltering("equal", "AAAA", 100, new byte[][]{
  182. Bytes.fromHex("AAAA")
  183. });
  184. }
  185. @Test
  186. public void testScanWithValueFilteringOperationNotEqual() throws Exception {
  187. testScanWithValueFiltering("not_equal", "AAAA", 100 , new byte[][]{
  188. Bytes.fromHex("0000"), Bytes.fromHex("1111"), Bytes.fromHex("2222"), Bytes.fromHex("3333"),
  189. Bytes.fromHex("4444"), Bytes.fromHex("5555"), Bytes.fromHex("6666"), Bytes.fromHex("7777"),
  190. Bytes.fromHex("8888"), Bytes.fromHex("9999"), Bytes.fromHex("BBBB"),
  191. Bytes.fromHex("CCCC"), Bytes.fromHex("DDDD"), Bytes.fromHex("EEEE"), Bytes.fromHex("FFFF")
  192. });
  193. }
  194. @Test
  195. public void testScanWithValueFilteringAndRowLimit() throws Exception {
  196. testScanWithValueFiltering("greater", "8887", 3, new byte[][] {
  197. Bytes.fromHex("8888"), Bytes.fromHex("9999"), Bytes.fromHex("AAAA")
  198. });
  199. }
  200. @Test
  201. public void testUpdate() throws Exception{
  202. setUp();
  203. final String key = "key";
  204. final HashMap<String, String> input = new HashMap<String, String>();
  205. input.put("column1", "value1");
  206. input.put("column2", "value2");
  207. final Status status = client.insert(tableName, key, StringByteIterator.getByteIteratorMap(input));
  208. assertEquals(Status.OK, status);
  209. // Verify result
  210. final Get get = new Get(Bytes.toBytes(key));
  211. final Result result = this.table.get(get);
  212. assertFalse(result.isEmpty());
  213. assertEquals(2, result.size());
  214. for(final java.util.Map.Entry<String, String> entry : input.entrySet()) {
  215. assertEquals(entry.getValue(),
  216. new String(result.getValue(Bytes.toBytes(COLUMN_FAMILY),
  217. Bytes.toBytes(entry.getKey()))));
  218. }
  219. }
  220. @Test
  221. @Ignore("Not yet implemented")
  222. public void testDelete() {
  223. fail("Not yet implemented");
  224. }
  225. private void testScanWithValueFiltering(String operation, String filterValue, int scanRowLimit,
  226. byte[][] expectedValuesReturned) throws Exception {
  227. Properties properties = new Properties();
  228. properties.setProperty("hbase.usescanvaluefiltering", String.valueOf(true));
  229. if(operation != null) {
  230. properties.setProperty("hbase.scanfilteroperator", operation);
  231. }
  232. if(filterValue != null) {
  233. properties.setProperty("hbase.scanfiltervalue", filterValue);
  234. }
  235. // setup the client and fill two columns with data
  236. setUp(properties);
  237. setupTableColumnWithHexValues("col_1");
  238. setupTableColumnWithHexValues("col_2");
  239. Vector<HashMap<String, ByteIterator>> result = new Vector<>();
  240. // first scan the whole table (both columns)
  241. client.scan(tableName, "00000", scanRowLimit, null, result);
  242. assertEquals(expectedValuesReturned.length, result.size());
  243. for(int i = 0; i < expectedValuesReturned.length; i++) {
  244. final HashMap<String, ByteIterator> row = result.get(i);
  245. assertEquals(2, row.size());
  246. assertTrue(row.containsKey("col_1") && row.containsKey("col_2"));
  247. assertArrayEquals(expectedValuesReturned[i], row.get("col_1").toArray());
  248. assertArrayEquals(expectedValuesReturned[i], row.get("col_2").toArray());
  249. }
  250. // now scan only a single column (the filter should work here too)
  251. result = new Vector<>();
  252. client.scan(tableName, "00000", scanRowLimit, Collections.singleton("col_1"), result);
  253. assertEquals(expectedValuesReturned.length, result.size());
  254. for(int i = 0; i < expectedValuesReturned.length; i++) {
  255. final HashMap<String, ByteIterator> row = result.get(i);
  256. assertEquals(1, row.size());
  257. assertTrue(row.containsKey("col_1"));
  258. assertArrayEquals(expectedValuesReturned[i], row.get("col_1").toArray());
  259. }
  260. }
  261. private void setupTableColumnWithHexValues(String colStr) throws Exception {
  262. final byte[] col = Bytes.toBytes(colStr);
  263. final byte[][] values = {
  264. Bytes.fromHex("0000"), Bytes.fromHex("1111"), Bytes.fromHex("2222"), Bytes.fromHex("3333"),
  265. Bytes.fromHex("4444"), Bytes.fromHex("5555"), Bytes.fromHex("6666"), Bytes.fromHex("7777"),
  266. Bytes.fromHex("8888"), Bytes.fromHex("9999"), Bytes.fromHex("AAAA"), Bytes.fromHex("BBBB"),
  267. Bytes.fromHex("CCCC"), Bytes.fromHex("DDDD"), Bytes.fromHex("EEEE"), Bytes.fromHex("FFFF")
  268. };
  269. final List<Put> puts = new ArrayList<>(16);
  270. for(int i = 0; i < 16; i++) {
  271. final byte[] key = Bytes.toBytes(String.format("%05d", i));
  272. final byte[] value = values[i];
  273. final Put p = new Put(key);
  274. p.addColumn(Bytes.toBytes(COLUMN_FAMILY), col, value);
  275. puts.add(p);
  276. }
  277. table.put(puts);
  278. }
  279. }