Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.balancer;
019
020import static org.apache.hadoop.hbase.ServerName.NON_STARTCODE;
021import static org.junit.Assert.assertEquals;
022import static org.junit.Assert.assertFalse;
023import static org.junit.Assert.assertNotNull;
024import static org.junit.Assert.assertNull;
025import static org.junit.Assert.assertTrue;
026
027import java.io.IOException;
028import java.util.EnumSet;
029import java.util.List;
030import java.util.Map;
031import java.util.Map.Entry;
032import java.util.Set;
033import java.util.stream.Collectors;
034import org.apache.hadoop.conf.Configuration;
035import org.apache.hadoop.hbase.ClusterMetrics.Option;
036import org.apache.hadoop.hbase.HBaseClassTestRule;
037import org.apache.hadoop.hbase.HBaseTestingUtility;
038import org.apache.hadoop.hbase.HColumnDescriptor;
039import org.apache.hadoop.hbase.HConstants;
040import org.apache.hadoop.hbase.HTableDescriptor;
041import org.apache.hadoop.hbase.MiniHBaseCluster;
042import org.apache.hadoop.hbase.ServerName;
043import org.apache.hadoop.hbase.TableName;
044import org.apache.hadoop.hbase.Waiter;
045import org.apache.hadoop.hbase.client.Admin;
046import org.apache.hadoop.hbase.client.RegionInfo;
047import org.apache.hadoop.hbase.favored.FavoredNodeAssignmentHelper;
048import org.apache.hadoop.hbase.favored.FavoredNodesManager;
049import org.apache.hadoop.hbase.favored.FavoredNodesPlan;
050import org.apache.hadoop.hbase.master.HMaster;
051import org.apache.hadoop.hbase.master.LoadBalancer;
052import org.apache.hadoop.hbase.master.ServerManager;
053import org.apache.hadoop.hbase.master.assignment.RegionStateNode;
054import org.apache.hadoop.hbase.master.assignment.RegionStates;
055import org.apache.hadoop.hbase.regionserver.HRegion;
056import org.apache.hadoop.hbase.testclassification.MediumTests;
057import org.apache.hadoop.hbase.util.Bytes;
058import org.apache.hadoop.hbase.util.JVMClusterUtil;
059import org.junit.After;
060import org.junit.Before;
061import org.junit.BeforeClass;
062import org.junit.ClassRule;
063import org.junit.Ignore;
064import org.junit.Test;
065import org.junit.experimental.categories.Category;
066import org.slf4j.Logger;
067import org.slf4j.LoggerFactory;
068
069import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
070import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
071import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
072
073@Ignore // Disabled
074@Category(MediumTests.class)
075public class TestFavoredStochasticLoadBalancer extends BalancerTestBase {
076
077  @ClassRule
078  public static final HBaseClassTestRule CLASS_RULE =
079    HBaseClassTestRule.forClass(TestFavoredStochasticLoadBalancer.class);
080
081  private static final Logger LOG =
082    LoggerFactory.getLogger(TestFavoredStochasticLoadBalancer.class);
083
084  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
085  private static final int SLAVES = 8;
086  private static final int REGION_NUM = SLAVES * 3;
087
088  private Admin admin;
089  private HMaster master;
090  private MiniHBaseCluster cluster;
091
092  @BeforeClass
093  public static void setupBeforeClass() throws Exception {
094    Configuration conf = TEST_UTIL.getConfiguration();
095    // Enable the favored nodes based load balancer
096    conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
097      LoadOnlyFavoredStochasticBalancer.class, LoadBalancer.class);
098  }
099
100  @Before
101  public void startCluster() throws Exception {
102    TEST_UTIL.startMiniCluster(SLAVES);
103    TEST_UTIL.getDFSCluster().waitClusterUp();
104    cluster = TEST_UTIL.getMiniHBaseCluster();
105    master = TEST_UTIL.getMiniHBaseCluster().getMaster();
106    admin = TEST_UTIL.getAdmin();
107    admin.setBalancerRunning(false, true);
108  }
109
110  @After
111  public void stopCluster() throws Exception {
112    TEST_UTIL.cleanupTestDir();
113    TEST_UTIL.shutdownMiniCluster();
114  }
115
116  @Test
117  public void testBasicBalance() throws Exception {
118
119    TableName tableName = TableName.valueOf("testBasicBalance");
120    HTableDescriptor desc = new HTableDescriptor(tableName);
121    desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
122    admin.createTable(desc, Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), REGION_NUM);
123    TEST_UTIL.waitTableAvailable(tableName);
124    TEST_UTIL.loadTable(admin.getConnection().getTable(tableName), HConstants.CATALOG_FAMILY);
125    admin.flush(tableName);
126    compactTable(tableName);
127
128    JVMClusterUtil.RegionServerThread rs1 = cluster.startRegionServerAndWait(10000);
129    JVMClusterUtil.RegionServerThread rs2 = cluster.startRegionServerAndWait(10000);
130
131    // Now try to run balance, and verify no regions are moved to the 2 region servers recently
132    // started.
133    admin.setBalancerRunning(true, true);
134    assertTrue("Balancer did not run", admin.balancer());
135    TEST_UTIL.waitUntilNoRegionsInTransition(120000);
136
137    List<RegionInfo> hris = admin.getRegions(rs1.getRegionServer().getServerName());
138    for (RegionInfo hri : hris) {
139      assertFalse("New RS contains regions belonging to table: " + tableName,
140        hri.getTable().equals(tableName));
141    }
142    hris = admin.getRegions(rs2.getRegionServer().getServerName());
143    for (RegionInfo hri : hris) {
144      assertFalse("New RS contains regions belonging to table: " + tableName,
145        hri.getTable().equals(tableName));
146    }
147  }
148
149  @Test
150  public void testRoundRobinAssignment() throws Exception {
151
152    TableName tableName = TableName.valueOf("testRoundRobinAssignment");
153    HTableDescriptor desc = new HTableDescriptor(tableName);
154    desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
155    admin.createTable(desc, Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), REGION_NUM);
156    TEST_UTIL.waitTableAvailable(tableName);
157    TEST_UTIL.loadTable(admin.getConnection().getTable(tableName), HConstants.CATALOG_FAMILY);
158    admin.flush(tableName);
159
160    LoadBalancer balancer = master.getLoadBalancer();
161    List<RegionInfo> regions = admin.getRegions(tableName);
162    regions.addAll(admin.getTableRegions(TableName.META_TABLE_NAME));
163    regions.addAll(admin.getTableRegions(TableName.NAMESPACE_TABLE_NAME));
164    List<ServerName> servers = Lists.newArrayList(
165      admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics().keySet());
166    Map<ServerName, List<RegionInfo>> map = balancer.roundRobinAssignment(regions, servers);
167    for (List<RegionInfo> regionInfos : map.values()) {
168      regions.removeAll(regionInfos);
169    }
170    assertEquals("No region should be missed by balancer", 0, regions.size());
171  }
172
173  @Test
174  public void testBasicRegionPlacementAndReplicaLoad() throws Exception {
175
176    String tableName = "testBasicRegionPlacement";
177    HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
178    desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
179    admin.createTable(desc, Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), REGION_NUM);
180    TEST_UTIL.waitTableAvailable(desc.getTableName());
181
182    FavoredNodesManager fnm = master.getFavoredNodesManager();
183    List<RegionInfo> regionsOfTable = admin.getRegions(TableName.valueOf(tableName));
184    for (RegionInfo rInfo : regionsOfTable) {
185      Set<ServerName> favNodes = Sets.newHashSet(fnm.getFavoredNodes(rInfo));
186      assertNotNull(favNodes);
187      assertEquals(FavoredNodeAssignmentHelper.FAVORED_NODES_NUM, favNodes.size());
188    }
189
190    Map<ServerName, List<Integer>> replicaLoadMap = fnm.getReplicaLoad(Lists.newArrayList(
191      admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics().keySet()));
192    assertTrue("Not all replica load collected.",
193      admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics().size()
194          == replicaLoadMap.size());
195    for (Entry<ServerName, List<Integer>> entry : replicaLoadMap.entrySet()) {
196      assertTrue(entry.getValue().size() == FavoredNodeAssignmentHelper.FAVORED_NODES_NUM);
197      assertTrue(entry.getValue().get(0) >= 0);
198      assertTrue(entry.getValue().get(1) >= 0);
199      assertTrue(entry.getValue().get(2) >= 0);
200    }
201
202    admin.disableTable(TableName.valueOf(tableName));
203    admin.deleteTable(TableName.valueOf(tableName));
204    replicaLoadMap = fnm.getReplicaLoad(Lists.newArrayList(
205      admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics().keySet()));
206    assertTrue("replica load found " + replicaLoadMap.size() + " instead of 0.",
207      replicaLoadMap.size() == admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS))
208        .getLiveServerMetrics().size());
209  }
210
211  @Test
212  public void testRandomAssignmentWithNoFavNodes() throws Exception {
213
214    final String tableName = "testRandomAssignmentWithNoFavNodes";
215    HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
216    desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
217    admin.createTable(desc);
218    TEST_UTIL.waitTableAvailable(desc.getTableName());
219
220    RegionInfo hri = admin.getTableRegions(TableName.valueOf(tableName)).get(0);
221
222    FavoredNodesManager fnm = master.getFavoredNodesManager();
223    fnm.deleteFavoredNodesForRegions(Lists.newArrayList(hri));
224    assertNull("Favored nodes not found null after delete", fnm.getFavoredNodes(hri));
225
226    LoadBalancer balancer = master.getLoadBalancer();
227    ServerName destination = balancer.randomAssignment(hri,
228      Lists.newArrayList(admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS))
229        .getLiveServerMetrics().keySet().stream().collect(Collectors.toList())));
230    assertNotNull(destination);
231    List<ServerName> favoredNodes = fnm.getFavoredNodes(hri);
232    assertNotNull(favoredNodes);
233    boolean containsFN = false;
234    for (ServerName sn : favoredNodes) {
235      if (ServerName.isSameAddress(destination, sn)) {
236        containsFN = true;
237      }
238    }
239    assertTrue("Destination server does not belong to favored nodes.", containsFN);
240  }
241
242  @Test
243  public void testBalancerWithoutFavoredNodes() throws Exception {
244
245    TableName tableName = TableName.valueOf("testBalancerWithoutFavoredNodes");
246    HTableDescriptor desc = new HTableDescriptor(tableName);
247    desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
248    admin.createTable(desc, Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), REGION_NUM);
249    TEST_UTIL.waitTableAvailable(tableName);
250
251    final RegionInfo region = admin.getTableRegions(tableName).get(0);
252    LOG.info("Region thats supposed to be in transition: " + region);
253    FavoredNodesManager fnm = master.getFavoredNodesManager();
254    List<ServerName> currentFN = fnm.getFavoredNodes(region);
255    assertNotNull(currentFN);
256
257    fnm.deleteFavoredNodesForRegions(Lists.newArrayList(region));
258
259    RegionStates regionStates = master.getAssignmentManager().getRegionStates();
260    admin.setBalancerRunning(true, true);
261
262    // Balancer should unassign the region
263    assertTrue("Balancer did not run", admin.balancer());
264    TEST_UTIL.waitUntilNoRegionsInTransition();
265
266    admin.assign(region.getEncodedNameAsBytes());
267    TEST_UTIL.waitUntilNoRegionsInTransition(60000);
268
269    currentFN = fnm.getFavoredNodes(region);
270    assertNotNull(currentFN);
271    assertEquals("Expected number of FN not present", FavoredNodeAssignmentHelper.FAVORED_NODES_NUM,
272      currentFN.size());
273
274    assertTrue("Balancer did not run", admin.balancer());
275    TEST_UTIL.waitUntilNoRegionsInTransition(60000);
276
277    checkFavoredNodeAssignments(tableName, fnm, regionStates);
278  }
279
280  @Ignore
281  @Test
282  public void testMisplacedRegions() throws Exception {
283
284    TableName tableName = TableName.valueOf("testMisplacedRegions");
285    HTableDescriptor desc = new HTableDescriptor(tableName);
286    desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
287    admin.createTable(desc, Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), REGION_NUM);
288    TEST_UTIL.waitTableAvailable(tableName);
289
290    final RegionInfo misplacedRegion = admin.getTableRegions(tableName).get(0);
291    FavoredNodesManager fnm = master.getFavoredNodesManager();
292    List<ServerName> currentFN = fnm.getFavoredNodes(misplacedRegion);
293    assertNotNull(currentFN);
294
295    List<ServerName> serversForNewFN = Lists.newArrayList();
296    for (ServerName sn : admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS))
297      .getLiveServerMetrics().keySet()) {
298      serversForNewFN.add(ServerName.valueOf(sn.getHostname(), sn.getPort(), NON_STARTCODE));
299    }
300    for (ServerName sn : currentFN) {
301      serversForNewFN.remove(sn);
302    }
303    FavoredNodeAssignmentHelper helper = new FavoredNodeAssignmentHelper(serversForNewFN, conf);
304    helper.initialize();
305    List<ServerName> newFavoredNodes = helper.generateFavoredNodes(misplacedRegion);
306    assertNotNull(newFavoredNodes);
307    assertEquals(FavoredNodeAssignmentHelper.FAVORED_NODES_NUM, newFavoredNodes.size());
308    Map<RegionInfo, List<ServerName>> regionFNMap = Maps.newHashMap();
309    regionFNMap.put(misplacedRegion, newFavoredNodes);
310    fnm.updateFavoredNodes(regionFNMap);
311
312    final RegionStates regionStates = master.getAssignmentManager().getRegionStates();
313    final ServerName current = regionStates.getRegionServerOfRegion(misplacedRegion);
314    assertNull("Misplaced region is still hosted on favored node, not expected.",
315      FavoredNodesPlan.getFavoredServerPosition(fnm.getFavoredNodes(misplacedRegion), current));
316    admin.setBalancerRunning(true, true);
317    assertTrue("Balancer did not run", admin.balancer());
318    TEST_UTIL.waitFor(120000, 30000, new Waiter.Predicate<Exception>() {
319      @Override
320      public boolean evaluate() throws Exception {
321        ServerName host = regionStates.getRegionServerOfRegion(misplacedRegion);
322        return !ServerName.isSameAddress(host, current);
323      }
324    });
325    checkFavoredNodeAssignments(tableName, fnm, regionStates);
326  }
327
328  @Test
329  public void test2FavoredNodesDead() throws Exception {
330
331    TableName tableName = TableName.valueOf("testAllFavoredNodesDead");
332    HTableDescriptor desc = new HTableDescriptor(tableName);
333    desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
334    admin.createTable(desc, Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), REGION_NUM);
335    TEST_UTIL.waitTableAvailable(tableName);
336
337    final RegionInfo region = admin.getTableRegions(tableName).get(0);
338    LOG.info("Region that's supposed to be in transition: " + region);
339    FavoredNodesManager fnm = master.getFavoredNodesManager();
340    List<ServerName> currentFN = fnm.getFavoredNodes(region);
341    assertNotNull(currentFN);
342
343    List<ServerName> serversToStop = Lists.newArrayList(currentFN);
344    serversToStop.remove(currentFN.get(0));
345
346    // Lets kill 2 FN for the region. All regions should still be assigned
347    stopServersAndWaitUntilProcessed(serversToStop);
348
349    TEST_UTIL.waitUntilNoRegionsInTransition();
350    final RegionStates regionStates = master.getAssignmentManager().getRegionStates();
351    TEST_UTIL.waitFor(10000, new Waiter.Predicate<Exception>() {
352      @Override
353      public boolean evaluate() throws Exception {
354        return regionStates.getRegionState(region).isOpened();
355      }
356    });
357
358    assertEquals("Not all regions are online", REGION_NUM, admin.getTableRegions(tableName).size());
359    admin.setBalancerRunning(true, true);
360    assertTrue("Balancer did not run", admin.balancer());
361    TEST_UTIL.waitUntilNoRegionsInTransition(60000);
362
363    checkFavoredNodeAssignments(tableName, fnm, regionStates);
364  }
365
366  @Ignore
367  @Test
368  public void testAllFavoredNodesDead() throws Exception {
369
370    TableName tableName = TableName.valueOf("testAllFavoredNodesDead");
371    HTableDescriptor desc = new HTableDescriptor(tableName);
372    desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
373    admin.createTable(desc, Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), REGION_NUM);
374    TEST_UTIL.waitTableAvailable(tableName);
375
376    final RegionInfo region = admin.getTableRegions(tableName).get(0);
377    LOG.info("Region that's supposed to be in transition: " + region);
378    FavoredNodesManager fnm = master.getFavoredNodesManager();
379    List<ServerName> currentFN = fnm.getFavoredNodes(region);
380    assertNotNull(currentFN);
381
382    // Lets kill all the RS that are favored nodes for this region.
383    stopServersAndWaitUntilProcessed(currentFN);
384
385    final RegionStates regionStates = master.getAssignmentManager().getRegionStates();
386    TEST_UTIL.waitFor(10000, new Waiter.Predicate<Exception>() {
387      @Override
388      public boolean evaluate() throws Exception {
389        return regionStates.getRegionState(region).isFailedOpen();
390      }
391    });
392
393    assertTrue("Region: " + region + " should be RIT",
394      regionStates.getRegionState(region).isFailedOpen());
395
396    // Regenerate FN and assign, everything else should be fine
397    List<ServerName> serversForNewFN = Lists.newArrayList();
398    for (ServerName sn : admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS))
399      .getLiveServerMetrics().keySet()) {
400      serversForNewFN.add(ServerName.valueOf(sn.getHostname(), sn.getPort(), NON_STARTCODE));
401    }
402
403    FavoredNodeAssignmentHelper helper = new FavoredNodeAssignmentHelper(serversForNewFN, conf);
404    helper.initialize();
405
406    for (RegionStateNode regionState : regionStates.getRegionsInTransition()) {
407      RegionInfo regionInfo = regionState.getRegionInfo();
408      List<ServerName> newFavoredNodes = helper.generateFavoredNodes(regionInfo);
409      assertNotNull(newFavoredNodes);
410      assertEquals(FavoredNodeAssignmentHelper.FAVORED_NODES_NUM, newFavoredNodes.size());
411      LOG.info("Region: " + regionInfo.getEncodedName() + " FN: " + newFavoredNodes);
412
413      Map<RegionInfo, List<ServerName>> regionFNMap = Maps.newHashMap();
414      regionFNMap.put(regionInfo, newFavoredNodes);
415      fnm.updateFavoredNodes(regionFNMap);
416      LOG.info("Assigning region: " + regionInfo.getEncodedName());
417      admin.assign(regionInfo.getEncodedNameAsBytes());
418    }
419    TEST_UTIL.waitUntilNoRegionsInTransition(60000);
420    assertEquals("Not all regions are online", REGION_NUM, admin.getTableRegions(tableName).size());
421
422    admin.setBalancerRunning(true, true);
423    assertTrue("Balancer did not run", admin.balancer());
424    TEST_UTIL.waitUntilNoRegionsInTransition(60000);
425
426    checkFavoredNodeAssignments(tableName, fnm, regionStates);
427  }
428
429  @Ignore
430  @Test
431  public void testAllFavoredNodesDeadMasterRestarted() throws Exception {
432
433    TableName tableName = TableName.valueOf("testAllFavoredNodesDeadMasterRestarted");
434    HTableDescriptor desc = new HTableDescriptor(tableName);
435    desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
436    admin.createTable(desc, Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), REGION_NUM);
437    TEST_UTIL.waitTableAvailable(tableName);
438
439    final RegionInfo region = admin.getTableRegions(tableName).get(0);
440    LOG.info("Region that's supposed to be in transition: " + region);
441    FavoredNodesManager fnm = master.getFavoredNodesManager();
442    List<ServerName> currentFN = fnm.getFavoredNodes(region);
443    assertNotNull(currentFN);
444
445    // Lets kill all the RS that are favored nodes for this region.
446    stopServersAndWaitUntilProcessed(currentFN);
447
448    final RegionStates regionStatesBeforeMaster = master.getAssignmentManager().getRegionStates();
449    TEST_UTIL.waitFor(10000, new Waiter.Predicate<Exception>() {
450      @Override
451      public boolean evaluate() throws Exception {
452        return regionStatesBeforeMaster.getRegionState(region).isFailedOpen();
453      }
454    });
455
456    assertTrue("Region: " + region + " should be RIT",
457      regionStatesBeforeMaster.getRegionState(region).isFailedOpen());
458
459    List<RegionInfo> rit = Lists.newArrayList();
460    for (RegionStateNode regionState : regionStatesBeforeMaster.getRegionsInTransition()) {
461      RegionInfo regionInfo = regionState.getRegionInfo();
462      LOG.debug("Region in transition after stopping FN's: " + regionInfo);
463      rit.add(regionInfo);
464      assertTrue("Region: " + regionInfo + " should be RIT",
465        regionStatesBeforeMaster.getRegionState(regionInfo).isFailedOpen());
466      assertEquals("Region: " + regionInfo + " does not belong to table: " + tableName, tableName,
467        regionInfo.getTable());
468    }
469
470    Configuration conf = cluster.getConf();
471    conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART,
472      SLAVES - FavoredNodeAssignmentHelper.FAVORED_NODES_NUM);
473
474    cluster.stopMaster(master.getServerName());
475    cluster.waitForMasterToStop(master.getServerName(), 60000);
476
477    cluster.startMaster();
478    cluster.waitForActiveAndReadyMaster();
479    master = cluster.getMaster();
480    fnm = master.getFavoredNodesManager();
481
482    RegionStates regionStates = master.getAssignmentManager().getRegionStates();
483    assertTrue("Region: " + region + " should be RIT",
484      regionStates.getRegionState(region).isFailedOpen());
485
486    for (RegionInfo regionInfo : rit) {
487      assertTrue("Region: " + regionInfo + " should be RIT",
488        regionStates.getRegionState(regionInfo).isFailedOpen());
489    }
490
491    // Regenerate FN and assign, everything else should be fine
492    List<ServerName> serversForNewFN = Lists.newArrayList();
493    for (ServerName sn : admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS))
494      .getLiveServerMetrics().keySet()) {
495      serversForNewFN.add(ServerName.valueOf(sn.getHostname(), sn.getPort(), NON_STARTCODE));
496    }
497
498    FavoredNodeAssignmentHelper helper = new FavoredNodeAssignmentHelper(serversForNewFN, conf);
499    helper.initialize();
500
501    for (RegionInfo regionInfo : rit) {
502      List<ServerName> newFavoredNodes = helper.generateFavoredNodes(regionInfo);
503      assertNotNull(newFavoredNodes);
504      assertEquals(FavoredNodeAssignmentHelper.FAVORED_NODES_NUM, newFavoredNodes.size());
505      LOG.info("Region: " + regionInfo.getEncodedName() + " FN: " + newFavoredNodes);
506
507      Map<RegionInfo, List<ServerName>> regionFNMap = Maps.newHashMap();
508      regionFNMap.put(regionInfo, newFavoredNodes);
509      fnm.updateFavoredNodes(regionFNMap);
510      LOG.info("Assigning region: " + regionInfo.getEncodedName());
511      admin.assign(regionInfo.getEncodedNameAsBytes());
512    }
513    TEST_UTIL.waitUntilNoRegionsInTransition(60000);
514    assertEquals("Not all regions are online", REGION_NUM, admin.getTableRegions(tableName).size());
515
516    admin.setBalancerRunning(true, true);
517    assertTrue("Balancer did not run", admin.balancer());
518    TEST_UTIL.waitUntilNoRegionsInTransition(60000);
519
520    checkFavoredNodeAssignments(tableName, fnm, regionStates);
521  }
522
523  private void checkFavoredNodeAssignments(TableName tableName, FavoredNodesManager fnm,
524    RegionStates regionStates) throws IOException {
525    for (RegionInfo hri : admin.getTableRegions(tableName)) {
526      ServerName host = regionStates.getRegionServerOfRegion(hri);
527      assertNotNull(
528        "Region: " + hri.getEncodedName() + " not on FN, current: " + host + " FN list: "
529          + fnm.getFavoredNodes(hri),
530        FavoredNodesPlan.getFavoredServerPosition(fnm.getFavoredNodes(hri), host));
531    }
532  }
533
534  private void stopServersAndWaitUntilProcessed(List<ServerName> currentFN) throws Exception {
535    for (ServerName sn : currentFN) {
536      for (JVMClusterUtil.RegionServerThread rst : cluster.getLiveRegionServerThreads()) {
537        if (ServerName.isSameAddress(sn, rst.getRegionServer().getServerName())) {
538          LOG.info("Shutting down server: " + sn);
539          cluster.stopRegionServer(rst.getRegionServer().getServerName());
540          cluster.waitForRegionServerToStop(rst.getRegionServer().getServerName(), 60000);
541        }
542      }
543    }
544
545    // Wait until dead servers are processed.
546    TEST_UTIL.waitFor(60000, new Waiter.Predicate<Exception>() {
547      @Override
548      public boolean evaluate() throws Exception {
549        return !master.getServerManager().areDeadServersInProgress();
550      }
551    });
552
553    assertEquals("Not all servers killed", SLAVES - currentFN.size(),
554      cluster.getLiveRegionServerThreads().size());
555  }
556
557  private void compactTable(TableName tableName) throws IOException {
558    for (JVMClusterUtil.RegionServerThread t : cluster.getRegionServerThreads()) {
559      for (HRegion region : t.getRegionServer().getRegions(tableName)) {
560        region.compact(true);
561      }
562    }
563  }
564}