Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.util;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022
023import java.io.File;
024import java.io.FileWriter;
025import java.io.IOException;
026import java.util.Collections;
027import java.util.List;
028import org.apache.hadoop.conf.Configuration;
029import org.apache.hadoop.fs.Path;
030import org.apache.hadoop.hbase.HBaseClassTestRule;
031import org.apache.hadoop.hbase.HBaseTestingUtility;
032import org.apache.hadoop.hbase.HConstants;
033import org.apache.hadoop.hbase.MiniHBaseCluster;
034import org.apache.hadoop.hbase.ServerName;
035import org.apache.hadoop.hbase.TableName;
036import org.apache.hadoop.hbase.Waiter.Predicate;
037import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
038import org.apache.hadoop.hbase.client.TableDescriptor;
039import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
040import org.apache.hadoop.hbase.regionserver.HRegion;
041import org.apache.hadoop.hbase.regionserver.HRegionServer;
042import org.apache.hadoop.hbase.testclassification.LargeTests;
043import org.apache.hadoop.hbase.testclassification.MiscTests;
044import org.apache.hadoop.hbase.util.RegionMover.RegionMoverBuilder;
045import org.junit.After;
046import org.junit.AfterClass;
047import org.junit.Assert;
048import org.junit.Before;
049import org.junit.BeforeClass;
050import org.junit.ClassRule;
051import org.junit.Rule;
052import org.junit.Test;
053import org.junit.experimental.categories.Category;
054import org.junit.rules.TestName;
055import org.slf4j.Logger;
056import org.slf4j.LoggerFactory;
057
058/**
059 * Tests for Region Mover Load/Unload functionality with and without ack mode and also to test
060 * exclude functionality useful for rack decommissioning
061 */
062@Category({ MiscTests.class, LargeTests.class })
063public class TestRegionMover1 {
064
065  @ClassRule
066  public static final HBaseClassTestRule CLASS_RULE =
067    HBaseClassTestRule.forClass(TestRegionMover1.class);
068
069  @Rule
070  public TestName name = new TestName();
071
072  private static final Logger LOG = LoggerFactory.getLogger(TestRegionMover1.class);
073
074  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
075
076  @BeforeClass
077  public static void setUpBeforeClass() throws Exception {
078    TEST_UTIL.startMiniCluster(3);
079    TEST_UTIL.getAdmin().balancerSwitch(false, true);
080  }
081
082  @AfterClass
083  public static void tearDownAfterClass() throws Exception {
084    TEST_UTIL.shutdownMiniCluster();
085  }
086
087  @Before
088  public void setUp() throws Exception {
089    final TableName tableName = TableName.valueOf(name.getMethodName());
090    TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(tableName)
091      .setColumnFamily(ColumnFamilyDescriptorBuilder.of("fam1")).build();
092    String startKey = "a";
093    String endKey = "z";
094    TEST_UTIL.getAdmin().createTable(tableDesc, Bytes.toBytes(startKey), Bytes.toBytes(endKey), 9);
095  }
096
097  @After
098  public void tearDown() throws Exception {
099    final TableName tableName = TableName.valueOf(name.getMethodName());
100    TEST_UTIL.getAdmin().disableTable(tableName);
101    TEST_UTIL.getAdmin().deleteTable(tableName);
102  }
103
104  @Test
105  public void testWithAck() throws Exception {
106    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
107    HRegionServer regionServer = cluster.getRegionServer(0);
108    String rsName = regionServer.getServerName().getAddress().toString();
109    int numRegions = regionServer.getNumberOfOnlineRegions();
110    RegionMoverBuilder rmBuilder =
111      new RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true).maxthreads(8);
112    try (RegionMover rm = rmBuilder.build()) {
113      LOG.info("Unloading " + regionServer.getServerName());
114      rm.unload();
115      assertEquals(0, regionServer.getNumberOfOnlineRegions());
116      LOG.info("Successfully Unloaded\nNow Loading");
117      rm.load();
118      assertEquals(numRegions, regionServer.getNumberOfOnlineRegions());
119      // Repeat the same load. It should be very fast because all regions are already moved.
120      rm.load();
121    }
122  }
123
124  /**
125   * Test to unload a regionserver first and then load it using no Ack mode.
126   */
127  @Test
128  public void testWithoutAck() throws Exception {
129    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
130    HRegionServer regionServer = cluster.getRegionServer(0);
131    String rsName = regionServer.getServerName().getAddress().toString();
132    int numRegions = regionServer.getNumberOfOnlineRegions();
133    RegionMoverBuilder rmBuilder =
134      new RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(false);
135    try (RegionMover rm = rmBuilder.build()) {
136      LOG.info("Unloading " + regionServer.getServerName());
137      rm.unload();
138      TEST_UTIL.waitFor(30000, 1000, new Predicate<Exception>() {
139        @Override
140        public boolean evaluate() throws Exception {
141          return regionServer.getNumberOfOnlineRegions() == 0;
142        }
143      });
144      LOG.info("Successfully Unloaded\nNow Loading");
145      rm.load();
146      // In UT we only have 10 regions so it is not likely to fail, so here we check for all
147      // regions, in the real production this may not be true.
148      TEST_UTIL.waitFor(30000, 1000, new Predicate<Exception>() {
149        @Override
150        public boolean evaluate() throws Exception {
151          return regionServer.getNumberOfOnlineRegions() == numRegions;
152        }
153      });
154    }
155  }
156
157  /**
158   * To test that we successfully exclude a server from the unloading process We test for the number
159   * of regions on Excluded server and also test that regions are unloaded successfully
160   */
161  @Test
162  public void testExclude() throws Exception {
163    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
164    File excludeFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(), "exclude_file");
165    FileWriter fos = new FileWriter(excludeFile);
166    HRegionServer excludeServer = cluster.getRegionServer(1);
167    String excludeHostname = excludeServer.getServerName().getHostname();
168    int excludeServerPort = excludeServer.getServerName().getPort();
169    int regionsExcludeServer = excludeServer.getNumberOfOnlineRegions();
170    String excludeServerName = excludeHostname + ":" + Integer.toString(excludeServerPort);
171    fos.write(excludeServerName);
172    fos.close();
173    HRegionServer regionServer = cluster.getRegionServer(0);
174    String rsName = regionServer.getServerName().getHostname();
175    int port = regionServer.getServerName().getPort();
176    String rs = rsName + ":" + Integer.toString(port);
177    RegionMoverBuilder rmBuilder = new RegionMoverBuilder(rs, TEST_UTIL.getConfiguration())
178      .ack(true).excludeFile(excludeFile.getCanonicalPath());
179    try (RegionMover rm = rmBuilder.build()) {
180      rm.unload();
181      LOG.info("Unloading " + rs);
182      assertEquals(0, regionServer.getNumberOfOnlineRegions());
183      assertEquals(regionsExcludeServer, cluster.getRegionServer(1).getNumberOfOnlineRegions());
184      LOG.info("Before:" + regionsExcludeServer + " After:"
185        + cluster.getRegionServer(1).getNumberOfOnlineRegions());
186    }
187  }
188
189  @Test
190  public void testDesignatedFile() throws Exception {
191    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
192    File designatedFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(), "designated_file");
193    HRegionServer designatedServer = cluster.getRegionServer(0);
194    try (FileWriter fos = new FileWriter(designatedFile)) {
195      String designatedHostname = designatedServer.getServerName().getHostname();
196      int designatedServerPort = designatedServer.getServerName().getPort();
197      String excludeServerName = designatedHostname + ":" + designatedServerPort;
198      fos.write(excludeServerName);
199    }
200    int regionsInDesignatedServer = designatedServer.getNumberOfOnlineRegions();
201    HRegionServer regionServer = cluster.getRegionServer(1);
202    String rsName = regionServer.getServerName().getHostname();
203    int port = regionServer.getServerName().getPort();
204    String rs = rsName + ":" + port;
205    int regionsInRegionServer = regionServer.getNumberOfOnlineRegions();
206    RegionMoverBuilder rmBuilder = new RegionMoverBuilder(rs, TEST_UTIL.getConfiguration())
207      .designatedFile(designatedFile.getCanonicalPath());
208    try (RegionMover rm = rmBuilder.build()) {
209      LOG.debug("Unloading {} regions", rs);
210      rm.unload();
211      assertEquals(0, regionServer.getNumberOfOnlineRegions());
212      assertEquals(regionsInDesignatedServer + regionsInRegionServer,
213        designatedServer.getNumberOfOnlineRegions());
214      LOG.debug("Before:{} After:{}", regionsInDesignatedServer,
215        designatedServer.getNumberOfOnlineRegions());
216    }
217  }
218
219  @Test
220  public void testExcludeAndDesignated() throws Exception {
221    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
222    // create designated file
223    File designatedFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(), "designated_file");
224    HRegionServer designatedServer = cluster.getRegionServer(0);
225    try (FileWriter fos = new FileWriter(designatedFile)) {
226      String designatedHostname = designatedServer.getServerName().getHostname();
227      int designatedServerPort = designatedServer.getServerName().getPort();
228      String excludeServerName = designatedHostname + ":" + designatedServerPort;
229      fos.write(excludeServerName);
230    }
231    int regionsInDesignatedServer = designatedServer.getNumberOfOnlineRegions();
232    // create exclude file
233    File excludeFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(), "exclude_file");
234    HRegionServer excludeServer = cluster.getRegionServer(1);
235    try (FileWriter fos = new FileWriter(excludeFile)) {
236      String excludeHostname = excludeServer.getServerName().getHostname();
237      int excludeServerPort = excludeServer.getServerName().getPort();
238      String excludeServerName = excludeHostname + ":" + excludeServerPort;
239      fos.write(excludeServerName);
240    }
241    int regionsInExcludeServer = excludeServer.getNumberOfOnlineRegions();
242
243    HRegionServer targetRegionServer = cluster.getRegionServer(2);
244    String rsName = targetRegionServer.getServerName().getHostname();
245    int port = targetRegionServer.getServerName().getPort();
246    String rs = rsName + ":" + port;
247    int regionsInTargetRegionServer = targetRegionServer.getNumberOfOnlineRegions();
248
249    RegionMoverBuilder rmBuilder = new RegionMoverBuilder(rs, TEST_UTIL.getConfiguration())
250      .designatedFile(designatedFile.getCanonicalPath())
251      .excludeFile(excludeFile.getCanonicalPath());
252    try (RegionMover rm = rmBuilder.build()) {
253      LOG.debug("Unloading {}", rs);
254      rm.unload();
255      assertEquals(0, targetRegionServer.getNumberOfOnlineRegions());
256      assertEquals(regionsInDesignatedServer + regionsInTargetRegionServer,
257        designatedServer.getNumberOfOnlineRegions());
258      LOG.debug("DesignatedServer Before:{} After:{}", regionsInDesignatedServer,
259        designatedServer.getNumberOfOnlineRegions());
260      assertEquals(regionsInExcludeServer, excludeServer.getNumberOfOnlineRegions());
261      LOG.debug("ExcludeServer Before:{} After:{}", regionsInExcludeServer,
262        excludeServer.getNumberOfOnlineRegions());
263    }
264  }
265
266  @Test
267  public void testRegionServerPort() throws Exception {
268    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
269    HRegionServer regionServer = cluster.getRegionServer(0);
270    String rsName = regionServer.getServerName().getHostname();
271
272    final int PORT = 16021;
273    Configuration conf = TEST_UTIL.getConfiguration();
274    String originalPort = conf.get(HConstants.REGIONSERVER_PORT);
275    conf.set(HConstants.REGIONSERVER_PORT, Integer.toString(PORT));
276    RegionMoverBuilder rmBuilder = new RegionMoverBuilder(rsName, conf);
277    assertEquals(PORT, rmBuilder.port);
278    if (originalPort != null) {
279      conf.set(HConstants.REGIONSERVER_PORT, originalPort);
280    }
281  }
282
283  /**
284   * UT for HBASE-21746
285   */
286  @Test
287  public void testLoadMetaRegion() throws Exception {
288    HRegionServer rsWithMeta = TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads().stream()
289      .map(t -> t.getRegionServer())
290      .filter(rs -> rs.getRegions(TableName.META_TABLE_NAME).size() > 0).findFirst().get();
291    int onlineRegions = rsWithMeta.getNumberOfOnlineRegions();
292    String rsName = rsWithMeta.getServerName().getAddress().toString();
293    try (RegionMover rm =
294      new RegionMoverBuilder(rsName, TEST_UTIL.getConfiguration()).ack(true).build()) {
295      LOG.info("Unloading " + rsWithMeta.getServerName());
296      rm.unload();
297      assertEquals(0, rsWithMeta.getNumberOfOnlineRegions());
298      LOG.info("Loading " + rsWithMeta.getServerName());
299      rm.load();
300      assertEquals(onlineRegions, rsWithMeta.getNumberOfOnlineRegions());
301    }
302  }
303
304  /**
305   * UT for HBASE-21746
306   */
307  @Test
308  public void testTargetServerDeadWhenLoading() throws Exception {
309    HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
310    String rsName = rs.getServerName().getAddress().toString();
311    Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
312    // wait 5 seconds at most
313    conf.setInt(RegionMover.SERVERSTART_WAIT_MAX_KEY, 5);
314    String filename =
315      new Path(TEST_UTIL.getDataTestDir(), "testTargetServerDeadWhenLoading").toString();
316    // unload the region server
317    try (
318      RegionMover rm = new RegionMoverBuilder(rsName, conf).filename(filename).ack(true).build()) {
319      LOG.info("Unloading " + rs.getServerName());
320      rm.unload();
321      assertEquals(0, rs.getNumberOfOnlineRegions());
322    }
323    String inexistRsName = "whatever:123";
324    try (RegionMover rm =
325      new RegionMoverBuilder(inexistRsName, conf).filename(filename).ack(true).build()) {
326      // load the regions to an inexist region server, which should fail and return false
327      LOG.info("Loading to an inexist region server {}", inexistRsName);
328      assertFalse(rm.load());
329    }
330  }
331
332  @Test
333  public void testDecomServerExclusionWithAck() throws Exception {
334    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
335    HRegionServer excludeServer = cluster.getRegionServer(1);
336    List<HRegion> regions = excludeServer.getRegions();
337    int regionsExcludeServer = excludeServer.getNumberOfOnlineRegions();
338    TEST_UTIL.getAdmin()
339      .decommissionRegionServers(Collections.singletonList(excludeServer.getServerName()), false);
340
341    waitForServerDecom(excludeServer);
342
343    HRegionServer regionServer = cluster.getRegionServer(0);
344    String rsName = regionServer.getServerName().getHostname();
345    int port = regionServer.getServerName().getPort();
346    String hostname = rsName + ":" + Integer.toString(port);
347    RegionMoverBuilder rmBuilder =
348      new RegionMoverBuilder(hostname, TEST_UTIL.getConfiguration()).ack(true);
349
350    int targetServerRegions = cluster.getRegionServer(2).getRegions().size();
351    int sourceServerRegions = regionServer.getRegions().size();
352
353    try (RegionMover regionMover = rmBuilder.build()) {
354      Assert.assertTrue(regionMover.unload());
355      LOG.info("Unloading {}", hostname);
356      assertEquals(0, regionServer.getNumberOfOnlineRegions());
357      assertEquals(regionsExcludeServer, cluster.getRegionServer(1).getNumberOfOnlineRegions());
358      LOG.info("Before:" + regionsExcludeServer + " After:"
359        + cluster.getRegionServer(1).getNumberOfOnlineRegions());
360      List<HRegion> regionList = cluster.getRegionServer(1).getRegions();
361      int index = 0;
362      for (HRegion hRegion : regionList) {
363        Assert.assertEquals(hRegion, regions.get(index++));
364      }
365      Assert.assertEquals(targetServerRegions + sourceServerRegions,
366        cluster.getRegionServer(2).getNumberOfOnlineRegions());
367      Assert.assertTrue(regionMover.load());
368    }
369
370    TEST_UTIL.getAdmin().recommissionRegionServer(excludeServer.getServerName(),
371      Collections.emptyList());
372  }
373
374  private void waitForServerDecom(HRegionServer excludeServer) {
375
376    TEST_UTIL.waitFor(3000, () -> {
377      try {
378        List<ServerName> decomServers = TEST_UTIL.getAdmin().listDecommissionedRegionServers();
379        return decomServers.size() == 1
380          && decomServers.get(0).equals(excludeServer.getServerName());
381      } catch (IOException e) {
382        throw new RuntimeException(e);
383      }
384    });
385  }
386
387  @Test
388  public void testDecomServerExclusion() throws Exception {
389    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
390    HRegionServer excludeServer = cluster.getRegionServer(0);
391    List<HRegion> regions = excludeServer.getRegions();
392    int regionsExcludeServer = excludeServer.getNumberOfOnlineRegions();
393    TEST_UTIL.getAdmin()
394      .decommissionRegionServers(Collections.singletonList(excludeServer.getServerName()), false);
395
396    waitForServerDecom(excludeServer);
397
398    HRegionServer sourceRegionServer = cluster.getRegionServer(1);
399    String rsName = sourceRegionServer.getServerName().getHostname();
400    int port = sourceRegionServer.getServerName().getPort();
401    String hostname = rsName + ":" + Integer.toString(port);
402    RegionMoverBuilder rmBuilder =
403      new RegionMoverBuilder(hostname, TEST_UTIL.getConfiguration()).ack(false);
404
405    int targetServerRegions = cluster.getRegionServer(2).getRegions().size();
406    int sourceServerRegions = sourceRegionServer.getRegions().size();
407
408    try (RegionMover regionMover = rmBuilder.build()) {
409      Assert.assertTrue(regionMover.unload());
410      LOG.info("Unloading {}", hostname);
411      assertEquals(0, sourceRegionServer.getNumberOfOnlineRegions());
412      assertEquals(regionsExcludeServer, cluster.getRegionServer(0).getNumberOfOnlineRegions());
413      LOG.info("Before:" + regionsExcludeServer + " After:"
414        + cluster.getRegionServer(1).getNumberOfOnlineRegions());
415      List<HRegion> regionList = cluster.getRegionServer(0).getRegions();
416      int index = 0;
417      for (HRegion hRegion : regionList) {
418        Assert.assertEquals(hRegion, regions.get(index++));
419      }
420      Assert.assertEquals(targetServerRegions + sourceServerRegions,
421        cluster.getRegionServer(2).getNumberOfOnlineRegions());
422      Assert.assertTrue(regionMover.load());
423    }
424
425    TEST_UTIL.getAdmin().recommissionRegionServer(excludeServer.getServerName(),
426      Collections.emptyList());
427  }
428
429  @Test
430  public void testExcludeAndDecomServers() throws Exception {
431    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
432    File excludeFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(), "exclude_file");
433    FileWriter fos = new FileWriter(excludeFile);
434    HRegionServer excludeServer = cluster.getRegionServer(1);
435    String excludeHostname = excludeServer.getServerName().getHostname();
436    int excludeServerPort = excludeServer.getServerName().getPort();
437    String excludeServerName = excludeHostname + ":" + Integer.toString(excludeServerPort);
438    fos.write(excludeServerName);
439    fos.close();
440
441    HRegionServer decomServer = cluster.getRegionServer(2);
442    TEST_UTIL.getAdmin()
443      .decommissionRegionServers(Collections.singletonList(decomServer.getServerName()), false);
444
445    waitForServerDecom(decomServer);
446
447    HRegionServer regionServer = cluster.getRegionServer(0);
448    String rsName = regionServer.getServerName().getHostname();
449    int port = regionServer.getServerName().getPort();
450    String sourceServer = rsName + ":" + Integer.toString(port);
451    RegionMoverBuilder rmBuilder =
452      new RegionMoverBuilder(sourceServer, TEST_UTIL.getConfiguration()).ack(true)
453        .excludeFile(excludeFile.getCanonicalPath());
454    try (RegionMover regionMover = rmBuilder.build()) {
455      Assert.assertFalse(regionMover.unload());
456    }
457
458    TEST_UTIL.getAdmin().recommissionRegionServer(decomServer.getServerName(),
459      Collections.emptyList());
460  }
461
462}