PageRenderTime 95ms CodeModel.GetById 21ms RepoModel.GetById 0ms app.codeStats 0ms

/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestSerialReplication.java

http://github.com/apache/hbase
Java | 253 lines | 222 code | 11 blank | 20 comment | 18 complexity | 350e90abb22572cc98f9cc7e0198d56d MD5 | raw file
Possible License(s): Apache-2.0, MIT
  1. /**
  2. * Licensed to the Apache Software Foundation (ASF) under one
  3. * or more contributor license agreements. See the NOTICE file
  4. * distributed with this work for additional information
  5. * regarding copyright ownership. The ASF licenses this file
  6. * to you under the Apache License, Version 2.0 (the
  7. * "License"); you may not use this file except in compliance
  8. * with the License. You may obtain a copy of the License at
  9. *
  10. * http://www.apache.org/licenses/LICENSE-2.0
  11. *
  12. * Unless required by applicable law or agreed to in writing, software
  13. * distributed under the License is distributed on an "AS IS" BASIS,
  14. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. * See the License for the specific language governing permissions and
  16. * limitations under the License.
  17. */
  18. package org.apache.hadoop.hbase.replication;
  19. import static org.junit.Assert.assertEquals;
  20. import static org.junit.Assert.assertNotEquals;
  21. import static org.junit.Assert.assertNotNull;
  22. import static org.junit.Assert.assertTrue;
  23. import java.io.IOException;
  24. import java.util.HashMap;
  25. import java.util.List;
  26. import java.util.Map;
  27. import java.util.concurrent.TimeUnit;
  28. import java.util.stream.Collectors;
  29. import org.apache.hadoop.hbase.HBaseClassTestRule;
  30. import org.apache.hadoop.hbase.HConstants;
  31. import org.apache.hadoop.hbase.TableName;
  32. import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
  33. import org.apache.hadoop.hbase.client.Put;
  34. import org.apache.hadoop.hbase.client.RegionInfo;
  35. import org.apache.hadoop.hbase.client.Table;
  36. import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
  37. import org.apache.hadoop.hbase.regionserver.HRegionServer;
  38. import org.apache.hadoop.hbase.testclassification.MediumTests;
  39. import org.apache.hadoop.hbase.testclassification.ReplicationTests;
  40. import org.apache.hadoop.hbase.util.Bytes;
  41. import org.apache.hadoop.hbase.util.CommonFSUtils.StreamLacksCapabilityException;
  42. import org.apache.hadoop.hbase.wal.WAL;
  43. import org.apache.hadoop.hbase.wal.WAL.Entry;
  44. import org.apache.hadoop.hbase.wal.WALFactory;
  45. import org.junit.Before;
  46. import org.junit.ClassRule;
  47. import org.junit.Test;
  48. import org.junit.experimental.categories.Category;
  49. @Category({ ReplicationTests.class, MediumTests.class })
  50. public class TestSerialReplication extends SerialReplicationTestBase {
  51. @ClassRule
  52. public static final HBaseClassTestRule CLASS_RULE =
  53. HBaseClassTestRule.forClass(TestSerialReplication.class);
  54. @Before
  55. public void setUp() throws IOException, StreamLacksCapabilityException {
  56. setupWALWriter();
  57. // add in disable state, so later when enabling it all sources will start push together.
  58. addPeer(false);
  59. }
  60. @Test
  61. public void testRegionMove() throws Exception {
  62. TableName tableName = createTable();
  63. try (Table table = UTIL.getConnection().getTable(tableName)) {
  64. for (int i = 0; i < 100; i++) {
  65. table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i)));
  66. }
  67. }
  68. RegionInfo region = UTIL.getAdmin().getRegions(tableName).get(0);
  69. HRegionServer rs = UTIL.getOtherRegionServer(UTIL.getRSForFirstRegionInTable(tableName));
  70. moveRegion(region, rs);
  71. try (Table table = UTIL.getConnection().getTable(tableName)) {
  72. for (int i = 100; i < 200; i++) {
  73. table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i)));
  74. }
  75. }
  76. enablePeerAndWaitUntilReplicationDone(200);
  77. checkOrder(200);
  78. }
  79. @Test
  80. public void testRegionSplit() throws Exception {
  81. TableName tableName = createTable();
  82. try (Table table = UTIL.getConnection().getTable(tableName)) {
  83. for (int i = 0; i < 100; i++) {
  84. table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i)));
  85. }
  86. }
  87. UTIL.flush(tableName);
  88. RegionInfo region = UTIL.getAdmin().getRegions(tableName).get(0);
  89. UTIL.getAdmin().splitRegionAsync(region.getEncodedNameAsBytes(), Bytes.toBytes(50)).get(30,
  90. TimeUnit.SECONDS);
  91. UTIL.waitUntilNoRegionsInTransition(30000);
  92. List<RegionInfo> regions = UTIL.getAdmin().getRegions(tableName);
  93. assertEquals(2, regions.size());
  94. try (Table table = UTIL.getConnection().getTable(tableName)) {
  95. for (int i = 0; i < 100; i++) {
  96. table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i)));
  97. }
  98. }
  99. enablePeerAndWaitUntilReplicationDone(200);
  100. Map<String, Long> regionsToSeqId = new HashMap<>();
  101. regionsToSeqId.put(region.getEncodedName(), -1L);
  102. regions.stream().map(RegionInfo::getEncodedName).forEach(n -> regionsToSeqId.put(n, -1L));
  103. try (WAL.Reader reader =
  104. WALFactory.createReader(UTIL.getTestFileSystem(), logPath, UTIL.getConfiguration())) {
  105. int count = 0;
  106. for (Entry entry;;) {
  107. entry = reader.next();
  108. if (entry == null) {
  109. break;
  110. }
  111. String encodedName = Bytes.toString(entry.getKey().getEncodedRegionName());
  112. Long seqId = regionsToSeqId.get(encodedName);
  113. assertNotNull(
  114. "Unexcepted entry " + entry + ", expected regions " + region + ", or " + regions, seqId);
  115. assertTrue("Sequence id go backwards from " + seqId + " to " +
  116. entry.getKey().getSequenceId() + " for " + encodedName,
  117. entry.getKey().getSequenceId() >= seqId.longValue());
  118. if (count < 100) {
  119. assertEquals(encodedName + " is pushed before parent " + region.getEncodedName(),
  120. region.getEncodedName(), encodedName);
  121. } else {
  122. assertNotEquals(region.getEncodedName(), encodedName);
  123. }
  124. count++;
  125. }
  126. assertEquals(200, count);
  127. }
  128. }
  129. @Test
  130. public void testRegionMerge() throws Exception {
  131. byte[] splitKey = Bytes.toBytes(50);
  132. TableName tableName = TableName.valueOf(name.getMethodName());
  133. UTIL.getAdmin().createTable(
  134. TableDescriptorBuilder.newBuilder(tableName)
  135. .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(CF)
  136. .setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build())
  137. .build(),
  138. new byte[][] { splitKey });
  139. UTIL.waitTableAvailable(tableName);
  140. try (Table table = UTIL.getConnection().getTable(tableName)) {
  141. for (int i = 0; i < 100; i++) {
  142. table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i)));
  143. }
  144. }
  145. List<RegionInfo> regions = UTIL.getAdmin().getRegions(tableName);
  146. UTIL.getAdmin()
  147. .mergeRegionsAsync(
  148. regions.stream().map(RegionInfo::getEncodedNameAsBytes).toArray(byte[][]::new), false)
  149. .get(30, TimeUnit.SECONDS);
  150. UTIL.waitUntilNoRegionsInTransition(30000);
  151. List<RegionInfo> regionsAfterMerge = UTIL.getAdmin().getRegions(tableName);
  152. assertEquals(1, regionsAfterMerge.size());
  153. try (Table table = UTIL.getConnection().getTable(tableName)) {
  154. for (int i = 0; i < 100; i++) {
  155. table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i)));
  156. }
  157. }
  158. enablePeerAndWaitUntilReplicationDone(200);
  159. Map<String, Long> regionsToSeqId = new HashMap<>();
  160. RegionInfo region = regionsAfterMerge.get(0);
  161. regionsToSeqId.put(region.getEncodedName(), -1L);
  162. regions.stream().map(RegionInfo::getEncodedName).forEach(n -> regionsToSeqId.put(n, -1L));
  163. try (WAL.Reader reader =
  164. WALFactory.createReader(UTIL.getTestFileSystem(), logPath, UTIL.getConfiguration())) {
  165. int count = 0;
  166. for (Entry entry;;) {
  167. entry = reader.next();
  168. if (entry == null) {
  169. break;
  170. }
  171. String encodedName = Bytes.toString(entry.getKey().getEncodedRegionName());
  172. Long seqId = regionsToSeqId.get(encodedName);
  173. assertNotNull(
  174. "Unexcepted entry " + entry + ", expected regions " + region + ", or " + regions, seqId);
  175. assertTrue("Sequence id go backwards from " + seqId + " to " +
  176. entry.getKey().getSequenceId() + " for " + encodedName,
  177. entry.getKey().getSequenceId() >= seqId.longValue());
  178. if (count < 100) {
  179. assertNotEquals(
  180. encodedName + " is pushed before parents " +
  181. regions.stream().map(RegionInfo::getEncodedName).collect(Collectors.joining(" and ")),
  182. region.getEncodedName(), encodedName);
  183. } else {
  184. assertEquals(region.getEncodedName(), encodedName);
  185. }
  186. count++;
  187. }
  188. assertEquals(200, count);
  189. }
  190. }
  191. @Test
  192. public void testRemovePeerNothingReplicated() throws Exception {
  193. TableName tableName = createTable();
  194. String encodedRegionName =
  195. UTIL.getMiniHBaseCluster().getRegions(tableName).get(0).getRegionInfo().getEncodedName();
  196. ReplicationQueueStorage queueStorage =
  197. UTIL.getMiniHBaseCluster().getMaster().getReplicationPeerManager().getQueueStorage();
  198. assertEquals(HConstants.NO_SEQNUM, queueStorage.getLastSequenceId(encodedRegionName, PEER_ID));
  199. UTIL.getAdmin().removeReplicationPeer(PEER_ID);
  200. assertEquals(HConstants.NO_SEQNUM, queueStorage.getLastSequenceId(encodedRegionName, PEER_ID));
  201. }
  202. @Test
  203. public void testRemovePeer() throws Exception {
  204. TableName tableName = createTable();
  205. try (Table table = UTIL.getConnection().getTable(tableName)) {
  206. for (int i = 0; i < 100; i++) {
  207. table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i)));
  208. }
  209. }
  210. enablePeerAndWaitUntilReplicationDone(100);
  211. checkOrder(100);
  212. String encodedRegionName =
  213. UTIL.getMiniHBaseCluster().getRegions(tableName).get(0).getRegionInfo().getEncodedName();
  214. ReplicationQueueStorage queueStorage =
  215. UTIL.getMiniHBaseCluster().getMaster().getReplicationPeerManager().getQueueStorage();
  216. assertTrue(queueStorage.getLastSequenceId(encodedRegionName, PEER_ID) > 0);
  217. UTIL.getAdmin().removeReplicationPeer(PEER_ID);
  218. // confirm that we delete the last pushed sequence id
  219. assertEquals(HConstants.NO_SEQNUM, queueStorage.getLastSequenceId(encodedRegionName, PEER_ID));
  220. }
  221. @Test
  222. public void testRemoveSerialFlag() throws Exception {
  223. TableName tableName = createTable();
  224. try (Table table = UTIL.getConnection().getTable(tableName)) {
  225. for (int i = 0; i < 100; i++) {
  226. table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i)));
  227. }
  228. }
  229. enablePeerAndWaitUntilReplicationDone(100);
  230. checkOrder(100);
  231. String encodedRegionName =
  232. UTIL.getMiniHBaseCluster().getRegions(tableName).get(0).getRegionInfo().getEncodedName();
  233. ReplicationQueueStorage queueStorage =
  234. UTIL.getMiniHBaseCluster().getMaster().getReplicationPeerManager().getQueueStorage();
  235. assertTrue(queueStorage.getLastSequenceId(encodedRegionName, PEER_ID) > 0);
  236. ReplicationPeerConfig peerConfig = UTIL.getAdmin().getReplicationPeerConfig(PEER_ID);
  237. UTIL.getAdmin().updateReplicationPeerConfig(PEER_ID,
  238. ReplicationPeerConfig.newBuilder(peerConfig).setSerial(false).build());
  239. // confirm that we delete the last pushed sequence id
  240. assertEquals(HConstants.NO_SEQNUM, queueStorage.getLastSequenceId(encodedRegionName, PEER_ID));
  241. }
  242. }