Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.util;
019
020import java.io.IOException;
021import java.util.ArrayList;
022import java.util.List;
023import java.util.concurrent.TimeUnit;
024import java.util.stream.Collectors;
025import org.apache.hadoop.hbase.HBaseClassTestRule;
026import org.apache.hadoop.hbase.HBaseTestingUtil;
027import org.apache.hadoop.hbase.HRegionLocation;
028import org.apache.hadoop.hbase.ServerName;
029import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
030import org.apache.hadoop.hbase.TableName;
031import org.apache.hadoop.hbase.client.Admin;
032import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
033import org.apache.hadoop.hbase.client.Put;
034import org.apache.hadoop.hbase.client.RegionInfoBuilder;
035import org.apache.hadoop.hbase.client.Table;
036import org.apache.hadoop.hbase.client.TableDescriptor;
037import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
038import org.apache.hadoop.hbase.master.RegionState;
039import org.apache.hadoop.hbase.regionserver.HRegion;
040import org.apache.hadoop.hbase.regionserver.HRegionServer;
041import org.apache.hadoop.hbase.testclassification.LargeTests;
042import org.apache.hadoop.hbase.testclassification.MiscTests;
043import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
044import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
045import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
046import org.junit.After;
047import org.junit.AfterClass;
048import org.junit.Assert;
049import org.junit.Before;
050import org.junit.BeforeClass;
051import org.junit.ClassRule;
052import org.junit.Rule;
053import org.junit.Test;
054import org.junit.experimental.categories.Category;
055import org.junit.rules.TestName;
056import org.slf4j.Logger;
057import org.slf4j.LoggerFactory;
058
059/**
060 * Tests for Region Mover Load/Unload functionality with and without ack mode and also to test
061 * exclude functionality useful for rack decommissioning
062 */
063@Category({ MiscTests.class, LargeTests.class })
064public class TestRegionMover2 {
065
066  @ClassRule
067  public static final HBaseClassTestRule CLASS_RULE =
068    HBaseClassTestRule.forClass(TestRegionMover2.class);
069  private static final String CF = "fam1";
070
071  @Rule
072  public TestName name = new TestName();
073
074  private static final Logger LOG = LoggerFactory.getLogger(TestRegionMover2.class);
075  private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
076
077  @BeforeClass
078  public static void setUpBeforeClass() throws Exception {
079    TEST_UTIL.startMiniCluster(3);
080    TEST_UTIL.getAdmin().balancerSwitch(false, true);
081  }
082
083  @AfterClass
084  public static void tearDownAfterClass() throws Exception {
085    TEST_UTIL.shutdownMiniCluster();
086  }
087
088  @Before
089  public void setUp() throws Exception {
090    createTable(name.getMethodName());
091  }
092
093  @After
094  public void tearDown() throws Exception {
095    final TableName tableName = TableName.valueOf(name.getMethodName());
096    TEST_UTIL.getAdmin().disableTable(tableName);
097    TEST_UTIL.getAdmin().deleteTable(tableName);
098  }
099
100  private TableName createTable(String name) throws IOException {
101    final TableName tableName = TableName.valueOf(name);
102    TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(tableName)
103      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(CF)).build();
104    int startKey = 0;
105    int endKey = 80000;
106    TEST_UTIL.getAdmin().createTable(tableDesc, Bytes.toBytes(startKey), Bytes.toBytes(endKey), 9);
107    return tableName;
108  }
109
110  @Test
111  public void testWithMergedRegions() throws Exception {
112    final TableName tableName = TableName.valueOf(name.getMethodName());
113    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
114    Admin admin = TEST_UTIL.getAdmin();
115    Table table = TEST_UTIL.getConnection().getTable(tableName);
116    List<Put> puts = createPuts(10000);
117    table.put(puts);
118    admin.flush(tableName);
119    HRegionServer regionServer = cluster.getRegionServer(0);
120    String rsName = regionServer.getServerName().getAddress().toString();
121    int numRegions = regionServer.getNumberOfOnlineRegions();
122    List<HRegion> hRegions = regionServer.getRegions().stream()
123      .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(tableName))
124      .collect(Collectors.toList());
125    RegionMover.RegionMoverBuilder rmBuilder =
126      new RegionMover.RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true)
127        .maxthreads(8);
128    try (RegionMover rm = rmBuilder.build()) {
129      LOG.debug("Unloading {}", regionServer.getServerName());
130      rm.unload();
131      Assert.assertEquals(0, regionServer.getNumberOfOnlineRegions());
132      LOG.debug("Successfully Unloaded, now Loading");
133      admin.mergeRegionsAsync(new byte[][] { hRegions.get(0).getRegionInfo().getRegionName(),
134        hRegions.get(1).getRegionInfo().getRegionName() }, true).get(5, TimeUnit.SECONDS);
135      Assert.assertTrue(rm.load());
136      Assert.assertEquals(numRegions - 2, regionServer.getNumberOfOnlineRegions());
137    }
138  }
139
140  @Test
141  public void testWithSplitRegions() throws Exception {
142    final TableName tableName = TableName.valueOf(name.getMethodName());
143    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
144    Admin admin = TEST_UTIL.getAdmin();
145    Table table = TEST_UTIL.getConnection().getTable(tableName);
146    List<Put> puts = createPuts(50000);
147    table.put(puts);
148    admin.flush(tableName);
149    admin.compact(tableName);
150    HRegionServer regionServer = cluster.getRegionServer(0);
151    String rsName = regionServer.getServerName().getAddress().toString();
152    int numRegions = regionServer.getNumberOfOnlineRegions();
153    List<HRegion> hRegions = regionServer.getRegions().stream()
154      .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(tableName))
155      .collect(Collectors.toList());
156
157    RegionMover.RegionMoverBuilder rmBuilder =
158      new RegionMover.RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true)
159        .maxthreads(8);
160    try (RegionMover rm = rmBuilder.build()) {
161      LOG.debug("Unloading {}", regionServer.getServerName());
162      rm.unload();
163      Assert.assertEquals(0, regionServer.getNumberOfOnlineRegions());
164      LOG.debug("Successfully Unloaded, now Loading");
165      HRegion hRegion = hRegions.get(1);
166      if (hRegion.getRegionInfo().getStartKey().length == 0) {
167        hRegion = hRegions.get(0);
168      }
169      int startKey = 0;
170      int endKey = Integer.MAX_VALUE;
171      if (hRegion.getRegionInfo().getStartKey().length > 0) {
172        startKey = Bytes.toInt(hRegion.getRegionInfo().getStartKey());
173      }
174      if (hRegion.getRegionInfo().getEndKey().length > 0) {
175        endKey = Bytes.toInt(hRegion.getRegionInfo().getEndKey());
176      }
177      int midKey = startKey + (endKey - startKey) / 2;
178      admin.splitRegionAsync(hRegion.getRegionInfo().getRegionName(), Bytes.toBytes(midKey)).get(5,
179        TimeUnit.SECONDS);
180      Assert.assertTrue(rm.load());
181      Assert.assertEquals(numRegions - 1, regionServer.getNumberOfOnlineRegions());
182    }
183  }
184
185  @Test
186  public void testFailedRegionMove() throws Exception {
187    final TableName tableName = TableName.valueOf(name.getMethodName());
188    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
189    Admin admin = TEST_UTIL.getAdmin();
190    Table table = TEST_UTIL.getConnection().getTable(tableName);
191    List<Put> puts = createPuts(1000);
192    table.put(puts);
193    admin.flush(tableName);
194    HRegionServer regionServer = cluster.getRegionServer(0);
195    String rsName = regionServer.getServerName().getAddress().toString();
196    List<HRegion> hRegions = regionServer.getRegions().stream()
197      .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(tableName))
198      .collect(Collectors.toList());
199    RegionMover.RegionMoverBuilder rmBuilder =
200      new RegionMover.RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true)
201        .maxthreads(8);
202    try (RegionMover rm = rmBuilder.build()) {
203      LOG.debug("Unloading {}", regionServer.getServerName());
204      rm.unload();
205      Assert.assertEquals(0, regionServer.getNumberOfOnlineRegions());
206      LOG.debug("Successfully Unloaded, now Loading");
207      admin.offline(hRegions.get(0).getRegionInfo().getRegionName());
208      // loading regions will fail because of offline region
209      Assert.assertFalse(rm.load());
210    }
211  }
212
213  @Test
214  public void testDeletedTable() throws Exception {
215    TableName tableNameToDelete = createTable(name.getMethodName() + "ToDelete");
216    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
217    HRegionServer regionServer = cluster.getRegionServer(0);
218    String rsName = regionServer.getServerName().getAddress().toString();
219    RegionMover.RegionMoverBuilder rmBuilder =
220      new RegionMover.RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true)
221        .maxthreads(8);
222    try (Admin admin = TEST_UTIL.getAdmin(); RegionMover rm = rmBuilder.build()) {
223      LOG.debug("Unloading {}", regionServer.getServerName());
224      rm.unload();
225      Assert.assertEquals(0, regionServer.getNumberOfOnlineRegions());
226      LOG.debug("Successfully Unloaded, now delete table");
227      admin.disableTable(tableNameToDelete);
228      admin.deleteTable(tableNameToDelete);
229      Assert.assertTrue(rm.load());
230    }
231  }
232
233  public void loadDummyDataInTable(TableName tableName) throws Exception {
234    Admin admin = TEST_UTIL.getAdmin();
235    Table table = TEST_UTIL.getConnection().getTable(tableName);
236    List<Put> puts = createPuts(1000);
237    table.put(puts);
238    admin.flush(tableName);
239  }
240
241  @Test
242  public void testIsolateSingleRegionOnTheSameServer() throws Exception {
243    final TableName tableName = TableName.valueOf(name.getMethodName());
244    loadDummyDataInTable(tableName);
245    ServerName sourceServerName = findSourceServerName(tableName);
246    // Isolating 1 region on the same region server.
247    regionIsolationOperation(sourceServerName, sourceServerName, 1, false);
248  }
249
250  @Test
251  public void testIsolateSingleRegionOnTheDifferentServer() throws Exception {
252    final TableName tableName = TableName.valueOf(name.getMethodName());
253    loadDummyDataInTable(tableName);
254    ServerName sourceServerName = findSourceServerName(tableName);
255    ServerName destinationServerName = findDestinationServerName(sourceServerName);
256    // Isolating 1 region on the different region server.
257    regionIsolationOperation(sourceServerName, destinationServerName, 1, false);
258  }
259
260  @Test
261  public void testIsolateMultipleRegionsOnTheSameServer() throws Exception {
262    final TableName tableName = TableName.valueOf(name.getMethodName());
263    loadDummyDataInTable(tableName);
264    ServerName sourceServerName = findSourceServerName(tableName);
265    // Isolating 2 regions on the same region server.
266    regionIsolationOperation(sourceServerName, sourceServerName, 2, false);
267  }
268
269  @Test
270  public void testIsolateMultipleRegionsOnTheDifferentServer() throws Exception {
271    final TableName tableName = TableName.valueOf(name.getMethodName());
272    loadDummyDataInTable(tableName);
273    // Isolating 2 regions on the different region server.
274    ServerName sourceServerName = findSourceServerName(tableName);
275    ServerName destinationServerName = findDestinationServerName(sourceServerName);
276    regionIsolationOperation(sourceServerName, destinationServerName, 2, false);
277  }
278
279  @Test
280  public void testIsolateMetaOnTheSameSever() throws Exception {
281    ServerName metaServerSource = findMetaRSLocation();
282    regionIsolationOperation(metaServerSource, metaServerSource, 1, true);
283  }
284
285  @Test
286  public void testIsolateMetaOnTheDifferentServer() throws Exception {
287    ServerName metaServerSource = findMetaRSLocation();
288    ServerName metaServerDestination = findDestinationServerName(metaServerSource);
289    regionIsolationOperation(metaServerSource, metaServerDestination, 1, true);
290  }
291
292  @Test
293  public void testIsolateMetaAndRandomRegionOnTheMetaServer() throws Exception {
294    final TableName tableName = TableName.valueOf(name.getMethodName());
295    loadDummyDataInTable(tableName);
296    ServerName metaServerSource = findMetaRSLocation();
297    ServerName randomSeverRegion = findSourceServerName(tableName);
298    regionIsolationOperation(randomSeverRegion, metaServerSource, 2, true);
299  }
300
301  @Test
302  public void testIsolateMetaAndRandomRegionOnTheRandomServer() throws Exception {
303    final TableName tableName = TableName.valueOf(name.getMethodName());
304    loadDummyDataInTable(tableName);
305    ServerName randomSeverRegion = findSourceServerName(tableName);
306    regionIsolationOperation(randomSeverRegion, randomSeverRegion, 2, true);
307  }
308
309  private List<Put> createPuts(int count) {
310    List<Put> puts = new ArrayList<>();
311    for (int i = 0; i < count; i++) {
312      puts.add(new Put(Bytes.toBytes("rowkey_" + i)).addColumn(Bytes.toBytes(CF),
313        Bytes.toBytes("q1"), Bytes.toBytes("val_" + i)));
314    }
315    return puts;
316  }
317
318  public ServerName findMetaRSLocation() throws Exception {
319    ZKWatcher zkWatcher = new ZKWatcher(TEST_UTIL.getConfiguration(), null, null);
320    List<HRegionLocation> result = new ArrayList<>();
321    for (String znode : zkWatcher.getMetaReplicaNodes()) {
322      String path = ZNodePaths.joinZNode(zkWatcher.getZNodePaths().baseZNode, znode);
323      int replicaId = zkWatcher.getZNodePaths().getMetaReplicaIdFromPath(path);
324      RegionState state = MetaTableLocator.getMetaRegionState(zkWatcher, replicaId);
325      result.add(new HRegionLocation(state.getRegion(), state.getServerName()));
326    }
327    return result.get(0).getServerName();
328  }
329
330  public ServerName findSourceServerName(TableName tableName) throws Exception {
331    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
332    int numOfRS = cluster.getNumLiveRegionServers();
333    ServerName sourceServer = null;
334    for (int i = 0; i < numOfRS; i++) {
335      HRegionServer regionServer = cluster.getRegionServer(i);
336      List<HRegion> hRegions = regionServer.getRegions().stream()
337        .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(tableName))
338        .collect(Collectors.toList());
339      if (hRegions.size() >= 2) {
340        sourceServer = regionServer.getServerName();
341        break;
342      }
343    }
344    if (sourceServer == null) {
345      throw new Exception(
346        "This shouldn't happen, No RS found with more than 2 regions of table : " + tableName);
347    }
348    return sourceServer;
349  }
350
351  public ServerName findDestinationServerName(ServerName sourceServerName) throws Exception {
352    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
353    ServerName destinationServerName = null;
354    int numOfRS = cluster.getNumLiveRegionServers();
355    for (int i = 0; i < numOfRS; i++) {
356      destinationServerName = cluster.getRegionServer(i).getServerName();
357      if (!destinationServerName.equals(sourceServerName)) {
358        break;
359      }
360    }
361    if (destinationServerName == null) {
362      throw new Exception("This shouldn't happen, No RS found which is different than source RS");
363    }
364    return destinationServerName;
365  }
366
367  public void regionIsolationOperation(ServerName sourceServerName,
368    ServerName destinationServerName, int numRegionsToIsolate, boolean isolateMetaAlso)
369    throws Exception {
370    final TableName tableName = TableName.valueOf(name.getMethodName());
371    SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
372    Admin admin = TEST_UTIL.getAdmin();
373    HRegionServer sourceRS = cluster.getRegionServer(sourceServerName);
374    List<HRegion> hRegions = sourceRS.getRegions().stream()
375      .filter(hRegion -> hRegion.getRegionInfo().getTable().equals(tableName))
376      .collect(Collectors.toList());
377    List<String> listOfRegionIDsToIsolate = new ArrayList<>();
378    for (int i = 0; i < numRegionsToIsolate; i++) {
379      listOfRegionIDsToIsolate.add(hRegions.get(i).getRegionInfo().getEncodedName());
380    }
381
382    if (isolateMetaAlso) {
383      listOfRegionIDsToIsolate.remove(0);
384      listOfRegionIDsToIsolate.add(RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName());
385    }
386
387    HRegionServer destinationRS = cluster.getRegionServer(destinationServerName);
388    String destinationRSName = destinationRS.getServerName().getAddress().toString();
389    RegionMover.RegionMoverBuilder rmBuilder =
390      new RegionMover.RegionMoverBuilder(destinationRSName, TEST_UTIL.getConfiguration()).ack(true)
391        .maxthreads(8).isolateRegionIdArray(listOfRegionIDsToIsolate);
392    try (RegionMover rm = rmBuilder.build()) {
393      LOG.debug("Unloading {} except regions: {}", destinationRS.getServerName(),
394        listOfRegionIDsToIsolate);
395      rm.isolateRegions();
396      Assert.assertEquals(numRegionsToIsolate, destinationRS.getNumberOfOnlineRegions());
397      List<HRegion> onlineRegions = destinationRS.getRegions();
398      for (int i = 0; i < numRegionsToIsolate; i++) {
399        Assert.assertTrue(
400          listOfRegionIDsToIsolate.contains(onlineRegions.get(i).getRegionInfo().getEncodedName()));
401      }
402      LOG.debug("Successfully Isolated {} regions: {} on {}", listOfRegionIDsToIsolate.size(),
403        listOfRegionIDsToIsolate, destinationRS.getServerName());
404    } finally {
405      admin.recommissionRegionServer(destinationRS.getServerName(), null);
406    }
407  }
408}