001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotNull;
022import static org.junit.Assert.assertNull;
023import static org.junit.Assert.assertTrue;
024import static org.junit.Assert.fail;
025
026import java.io.IOException;
027import java.net.InetSocketAddress;
028import java.util.ArrayList;
029import java.util.Arrays;
030import java.util.Collection;
031import java.util.HashMap;
032import java.util.List;
033import java.util.Map;
034import java.util.Random;
035import java.util.concurrent.ThreadLocalRandom;
036import java.util.concurrent.atomic.AtomicInteger;
037import org.apache.hadoop.conf.Configuration;
038import org.apache.hadoop.hbase.HBaseClassTestRule;
039import org.apache.hadoop.hbase.HBaseTestingUtility;
040import org.apache.hadoop.hbase.HColumnDescriptor;
041import org.apache.hadoop.hbase.HConstants;
042import org.apache.hadoop.hbase.HRegionLocation;
043import org.apache.hadoop.hbase.HTableDescriptor;
044import org.apache.hadoop.hbase.MetaTableAccessor;
045import org.apache.hadoop.hbase.MiniHBaseCluster;
046import org.apache.hadoop.hbase.NamespaceDescriptor;
047import org.apache.hadoop.hbase.ServerName;
048import org.apache.hadoop.hbase.TableName;
049import org.apache.hadoop.hbase.client.Admin;
050import org.apache.hadoop.hbase.client.Connection;
051import org.apache.hadoop.hbase.client.RegionInfo;
052import org.apache.hadoop.hbase.client.RegionLocator;
053import org.apache.hadoop.hbase.client.Result;
054import org.apache.hadoop.hbase.client.TableDescriptor;
055import org.apache.hadoop.hbase.favored.FavoredNodeAssignmentHelper;
056import org.apache.hadoop.hbase.favored.FavoredNodeLoadBalancer;
057import org.apache.hadoop.hbase.favored.FavoredNodesPlan;
058import org.apache.hadoop.hbase.favored.FavoredNodesPlan.Position;
059import org.apache.hadoop.hbase.regionserver.HRegion;
060import org.apache.hadoop.hbase.regionserver.HRegionServer;
061import org.apache.hadoop.hbase.regionserver.Region;
062import org.apache.hadoop.hbase.testclassification.MasterTests;
063import org.apache.hadoop.hbase.testclassification.MediumTests;
064import org.apache.hadoop.hbase.util.Bytes;
065import org.apache.hadoop.hbase.util.Pair;
066import org.apache.zookeeper.KeeperException;
067import org.junit.AfterClass;
068import org.junit.BeforeClass;
069import org.junit.ClassRule;
070import org.junit.Ignore;
071import org.junit.Test;
072import org.junit.experimental.categories.Category;
073import org.slf4j.Logger;
074import org.slf4j.LoggerFactory;
075
076@Category({ MasterTests.class, MediumTests.class })
077public class TestRegionPlacement {
078
079  @ClassRule
080  public static final HBaseClassTestRule CLASS_RULE =
081    HBaseClassTestRule.forClass(TestRegionPlacement.class);
082
083  private static final Logger LOG = LoggerFactory.getLogger(TestRegionPlacement.class);
084  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
085  private final static int SLAVES = 10;
086  private static Connection CONNECTION;
087  private static Admin admin;
088  private static RegionPlacementMaintainer rp;
089  private static Position[] positions = Position.values();
090  private int lastRegionOnPrimaryRSCount = 0;
091  private int REGION_NUM = 10;
092  private Map<RegionInfo, ServerName[]> favoredNodesAssignmentPlan = new HashMap<>();
093
094  @BeforeClass
095  public static void setupBeforeClass() throws Exception {
096    Configuration conf = TEST_UTIL.getConfiguration();
097    // Enable the favored nodes based load balancer
098    conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, FavoredNodeLoadBalancer.class,
099      LoadBalancer.class);
100    conf.setBoolean("hbase.tests.use.shortcircuit.reads", false);
101    TEST_UTIL.startMiniCluster(SLAVES);
102    CONNECTION = TEST_UTIL.getConnection();
103    admin = CONNECTION.getAdmin();
104    rp = new RegionPlacementMaintainer(conf);
105  }
106
107  @AfterClass
108  public static void tearDownAfterClass() throws Exception {
109    TEST_UTIL.shutdownMiniCluster();
110  }
111
112  @Ignore("Test for unfinished feature")
113  @Test
114  public void testRegionPlacement() throws Exception {
115    String tableStr = "testRegionAssignment";
116    TableName table = TableName.valueOf(tableStr);
117    // Create a table with REGION_NUM regions.
118    createTable(table, REGION_NUM);
119
120    TEST_UTIL.waitTableAvailable(table);
121
122    // Verify all the user regions are assigned to the primary region server
123    // based on the plan
124    verifyRegionOnPrimaryRS(REGION_NUM);
125
126    FavoredNodesPlan currentPlan = rp.getRegionAssignmentSnapshot().getExistingAssignmentPlan();
127    // Verify all the region server are update with the latest favored nodes
128    verifyRegionServerUpdated(currentPlan);
129    // Test Case 2: To verify whether the region placement tools can
130    // correctly update the new assignment plan to hbase:meta and Region Server.
131    // The new assignment plan is generated by shuffle the existing assignment
132    // plan by switching PRIMARY, SECONDARY and TERTIARY nodes.
133    // Shuffle the plan by switching the secondary region server with
134    // the tertiary.
135
136    // Shuffle the secondary with tertiary favored nodes
137    FavoredNodesPlan shuffledPlan = this.shuffleAssignmentPlan(currentPlan,
138      FavoredNodesPlan.Position.SECONDARY, FavoredNodesPlan.Position.TERTIARY);
139    // Let the region placement update the hbase:meta and Region Servers
140    rp.updateAssignmentPlan(shuffledPlan);
141
142    // Verify the region assignment. There are supposed to no region reassignment
143    // All the regions are still on the primary region server
144    verifyRegionAssignment(shuffledPlan, 0, REGION_NUM);
145
146    // Shuffle the plan by switching the primary with secondary and
147    // verify the region reassignment is consistent with the plan.
148    shuffledPlan = this.shuffleAssignmentPlan(currentPlan, FavoredNodesPlan.Position.PRIMARY,
149      FavoredNodesPlan.Position.SECONDARY);
150
151    // Let the region placement update the hbase:meta and Region Servers
152    rp.updateAssignmentPlan(shuffledPlan);
153
154    verifyRegionAssignment(shuffledPlan, REGION_NUM, REGION_NUM);
155
156    // also verify that the AssignmentVerificationReport has the correct information
157    RegionPlacementMaintainer rp = new RegionPlacementMaintainer(TEST_UTIL.getConfiguration());
158    // we are interested in only one table (and hence one report)
159    rp.setTargetTableName(new String[] { tableStr });
160    List<AssignmentVerificationReport> reports = rp.verifyRegionPlacement(false);
161    AssignmentVerificationReport report = reports.get(0);
162    assertTrue(report.getRegionsWithoutValidFavoredNodes().isEmpty());
163    assertTrue(report.getNonFavoredAssignedRegions().isEmpty());
164    assertTrue(report.getTotalFavoredAssignments() >= REGION_NUM);
165    assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) != 0);
166    assertTrue(
167      report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) == 0);
168    assertTrue(
169      report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY) == 0);
170    assertTrue(report.getUnassignedRegions().isEmpty());
171
172    // Check when a RS stops, the regions get assigned to their secondary/tertiary
173    killRandomServerAndVerifyAssignment();
174
175    // also verify that the AssignmentVerificationReport has the correct information
176    reports = rp.verifyRegionPlacement(false);
177    report = reports.get(0);
178    assertTrue(report.getRegionsWithoutValidFavoredNodes().isEmpty());
179    assertTrue(report.getNonFavoredAssignedRegions().isEmpty());
180    assertTrue(report.getTotalFavoredAssignments() >= REGION_NUM);
181    assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) > 0);
182    assertTrue(
183      "secondary "
184        + report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY)
185        + " tertiary "
186        + report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY),
187      (report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) > 0
188        || report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY) > 0));
189    assertTrue((report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY)
190      + report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY)
191      + report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY))
192        == REGION_NUM);
193    RegionPlacementMaintainer.printAssignmentPlan(currentPlan);
194  }
195
196  private void killRandomServerAndVerifyAssignment()
197    throws IOException, InterruptedException, KeeperException {
198    ServerName serverToKill = null;
199    int killIndex = 0;
200    Random rand = ThreadLocalRandom.current();
201    ServerName metaServer = TEST_UTIL.getHBaseCluster().getServerHoldingMeta();
202    LOG.debug("Server holding meta " + metaServer);
203    boolean isNamespaceServer = false;
204    do {
205      // kill a random non-meta server carrying at least one region
206      killIndex = rand.nextInt(SLAVES);
207      serverToKill = TEST_UTIL.getHBaseCluster().getRegionServer(killIndex).getServerName();
208      Collection<HRegion> regs =
209        TEST_UTIL.getHBaseCluster().getRegionServer(killIndex).getOnlineRegionsLocalContext();
210      isNamespaceServer = false;
211      for (HRegion r : regs) {
212        if (
213          r.getRegionInfo().getTable().getNamespaceAsString()
214            .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)
215        ) {
216          isNamespaceServer = true;
217          break;
218        }
219      }
220    } while (
221      ServerName.isSameAddress(metaServer, serverToKill) || isNamespaceServer
222        || TEST_UTIL.getHBaseCluster().getRegionServer(killIndex).getNumberOfOnlineRegions() == 0
223    );
224    LOG.debug("Stopping RS " + serverToKill);
225    Map<RegionInfo, Pair<ServerName, ServerName>> regionsToVerify = new HashMap<>();
226    // mark the regions to track
227    for (Map.Entry<RegionInfo, ServerName[]> entry : favoredNodesAssignmentPlan.entrySet()) {
228      ServerName s = entry.getValue()[0];
229      if (ServerName.isSameAddress(s, serverToKill)) {
230        regionsToVerify.put(entry.getKey(), new Pair<>(entry.getValue()[1], entry.getValue()[2]));
231        LOG.debug("Adding " + entry.getKey() + " with sedcondary/tertiary " + entry.getValue()[1]
232          + " " + entry.getValue()[2]);
233      }
234    }
235    int orig = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getNumRegionsOpened();
236    TEST_UTIL.getHBaseCluster().stopRegionServer(serverToKill);
237    TEST_UTIL.getHBaseCluster().waitForRegionServerToStop(serverToKill, 60000);
238    int curr = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getNumRegionsOpened();
239    while (curr - orig < regionsToVerify.size()) {
240      LOG.debug("Waiting for " + regionsToVerify.size() + " to come online " + " Current #regions "
241        + curr + " Original #regions " + orig);
242      Thread.sleep(200);
243      curr = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getNumRegionsOpened();
244    }
245    // now verify
246    for (Map.Entry<RegionInfo, Pair<ServerName, ServerName>> entry : regionsToVerify.entrySet()) {
247      ServerName newDestination = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
248        .getRegionStates().getRegionServerOfRegion(entry.getKey());
249      Pair<ServerName, ServerName> secondaryTertiaryServers = entry.getValue();
250      LOG.debug("New destination for region " + entry.getKey().getEncodedName() + " "
251        + newDestination + ". Secondary/Tertiary are " + secondaryTertiaryServers.getFirst() + "/"
252        + secondaryTertiaryServers.getSecond());
253      if (
254        !(ServerName.isSameAddress(newDestination, secondaryTertiaryServers.getFirst())
255          || ServerName.isSameAddress(newDestination, secondaryTertiaryServers.getSecond()))
256      ) {
257        fail("Region " + entry.getKey() + " not present on any of the expected servers");
258      }
259    }
260    // start(reinstate) region server since we killed one before
261    TEST_UTIL.getHBaseCluster().startRegionServer();
262  }
263
264  /**
265   * Used to test the correctness of this class.
266   */
267  @Ignore("Test for unfinished feature")
268  @Test
269  public void testRandomizedMatrix() {
270    int rows = 100;
271    int cols = 100;
272    float[][] matrix = new float[rows][cols];
273    Random rand = ThreadLocalRandom.current();
274    for (int i = 0; i < rows; i++) {
275      for (int j = 0; j < cols; j++) {
276        matrix[i][j] = rand.nextFloat();
277      }
278    }
279
280    // Test that inverting a transformed matrix gives the original matrix.
281    RegionPlacementMaintainer.RandomizedMatrix rm =
282      new RegionPlacementMaintainer.RandomizedMatrix(rows, cols);
283    float[][] transformed = rm.transform(matrix);
284    float[][] invertedTransformed = rm.invert(transformed);
285    for (int i = 0; i < rows; i++) {
286      for (int j = 0; j < cols; j++) {
287        if (matrix[i][j] != invertedTransformed[i][j]) {
288          throw new RuntimeException();
289        }
290      }
291    }
292
293    // Test that the indices on a transformed matrix can be inverted to give
294    // the same values on the original matrix.
295    int[] transformedIndices = new int[rows];
296    for (int i = 0; i < rows; i++) {
297      transformedIndices[i] = rand.nextInt(cols);
298    }
299    int[] invertedTransformedIndices = rm.invertIndices(transformedIndices);
300    float[] transformedValues = new float[rows];
301    float[] invertedTransformedValues = new float[rows];
302    for (int i = 0; i < rows; i++) {
303      transformedValues[i] = transformed[i][transformedIndices[i]];
304      invertedTransformedValues[i] = matrix[i][invertedTransformedIndices[i]];
305    }
306    Arrays.sort(transformedValues);
307    Arrays.sort(invertedTransformedValues);
308    if (!Arrays.equals(transformedValues, invertedTransformedValues)) {
309      throw new RuntimeException();
310    }
311  }
312
313  /**
314   * Shuffle the assignment plan by switching two favored node positions.
315   * @param plan The assignment plan
316   * @param p1   The first switch position
317   * @param p2   The second switch position
318   * @return the shuffled assignment plan
319   */
320  private FavoredNodesPlan shuffleAssignmentPlan(FavoredNodesPlan plan,
321    FavoredNodesPlan.Position p1, FavoredNodesPlan.Position p2) throws IOException {
322    FavoredNodesPlan shuffledPlan = new FavoredNodesPlan();
323
324    Map<String, RegionInfo> regionToHRegion =
325      rp.getRegionAssignmentSnapshot().getRegionNameToRegionInfoMap();
326    for (Map.Entry<String, List<ServerName>> entry : plan.getAssignmentMap().entrySet()) {
327
328      // copy the server list from the original plan
329      List<ServerName> shuffledServerList = new ArrayList<>();
330      shuffledServerList.addAll(entry.getValue());
331
332      // start to shuffle
333      shuffledServerList.set(p1.ordinal(), entry.getValue().get(p2.ordinal()));
334      shuffledServerList.set(p2.ordinal(), entry.getValue().get(p1.ordinal()));
335
336      // update the plan
337      shuffledPlan.updateFavoredNodesMap(regionToHRegion.get(entry.getKey()), shuffledServerList);
338    }
339    return shuffledPlan;
340  }
341
342  /**
343   * To verify the region assignment status. It will check the assignment plan consistency between
344   * hbase:meta and region servers. Also it will verify weather the number of region movement and
345   * the number regions on the primary region server are expected
346   */
347  private void verifyRegionAssignment(FavoredNodesPlan plan, int regionMovementNum,
348    int numRegionsOnPrimaryRS) throws InterruptedException, IOException {
349    // Verify the assignment plan in hbase:meta is consistent with the expected plan.
350    verifyMETAUpdated(plan);
351
352    // Verify the number of region movement is expected
353    verifyRegionMovementNum(regionMovementNum);
354
355    // Verify the number of regions is assigned to the primary region server
356    // based on the plan is expected
357    verifyRegionOnPrimaryRS(numRegionsOnPrimaryRS);
358
359    // Verify all the online region server are updated with the assignment plan
360    verifyRegionServerUpdated(plan);
361  }
362
363  /**
364   * Verify the meta has updated to the latest assignment plan
365   * @param expectedPlan the region assignment plan
366   * @throws IOException if an IO problem is encountered
367   */
368  private void verifyMETAUpdated(FavoredNodesPlan expectedPlan) throws IOException {
369    FavoredNodesPlan planFromMETA = rp.getRegionAssignmentSnapshot().getExistingAssignmentPlan();
370    assertTrue("The assignment plan is NOT consistent with the expected plan ",
371      planFromMETA.equals(expectedPlan));
372  }
373
374  /**
375   * Verify the number of region movement is expected
376   */
377  private void verifyRegionMovementNum(int expected) throws InterruptedException, IOException {
378    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
379    HMaster m = cluster.getMaster();
380    int lastRegionOpenedCount = m.getAssignmentManager().getNumRegionsOpened();
381    // get the assignments start to execute
382    m.balance();
383
384    int retry = 10;
385    long sleep = 3000;
386    int attempt = 0;
387    int currentRegionOpened, regionMovement;
388    do {
389      currentRegionOpened = m.getAssignmentManager().getNumRegionsOpened();
390      regionMovement = currentRegionOpened - lastRegionOpenedCount;
391      LOG.debug("There are " + regionMovement + "/" + expected + " regions moved after " + attempt
392        + " attempts");
393      Thread.sleep((++attempt) * sleep);
394    } while (regionMovement != expected && attempt <= retry);
395
396    // update the lastRegionOpenedCount
397    lastRegionOpenedCount = currentRegionOpened;
398
399    assertEquals("There are only " + regionMovement + " instead of " + expected
400      + " region movement for " + attempt + " attempts", expected, regionMovement);
401  }
402
403  /**
404   * Verify the number of user regions is assigned to the primary region server based on the plan is
405   * expected
406   * @param expectedNum the expected number of assigned regions
407   */
408  private void verifyRegionOnPrimaryRS(int expectedNum) throws IOException {
409    lastRegionOnPrimaryRSCount = getNumRegionisOnPrimaryRS();
410    assertEquals(
411      "Only " + expectedNum + " of user regions running " + "on the primary region server",
412      expectedNum, lastRegionOnPrimaryRSCount);
413  }
414
415  /**
416   * Verify all the online region servers has been updated to the latest assignment plan
417   */
418  private void verifyRegionServerUpdated(FavoredNodesPlan plan) throws IOException {
419    // Verify all region servers contain the correct favored nodes information
420    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
421    for (int i = 0; i < SLAVES; i++) {
422      HRegionServer rs = cluster.getRegionServer(i);
423      for (Region region : rs.getRegions(TableName.valueOf("testRegionAssignment"))) {
424        InetSocketAddress[] favoredSocketAddress =
425          rs.getFavoredNodesForRegion(region.getRegionInfo().getEncodedName());
426        String regionName = region.getRegionInfo().getRegionNameAsString();
427        List<ServerName> favoredServerList = plan.getAssignmentMap().get(regionName);
428
429        // All regions are supposed to have favored nodes,
430        // except for hbase:meta and ROOT
431        if (favoredServerList == null) {
432          TableDescriptor desc = region.getTableDescriptor();
433          // Verify they are ROOT and hbase:meta regions since no favored nodes
434          assertNull(favoredSocketAddress);
435          assertTrue("User region " + region.getTableDescriptor().getTableName()
436            + " should have favored nodes", desc.isMetaRegion());
437        } else {
438          // For user region, the favored nodes in the region server should be
439          // identical to favored nodes in the assignmentPlan
440          assertTrue(favoredSocketAddress.length == favoredServerList.size());
441          assertTrue(favoredServerList.size() > 0);
442          for (int j = 0; j < favoredServerList.size(); j++) {
443            InetSocketAddress addrFromRS = favoredSocketAddress[j];
444            InetSocketAddress addrFromPlan = InetSocketAddress.createUnresolved(
445              favoredServerList.get(j).getHostname(), favoredServerList.get(j).getPort());
446
447            assertNotNull(addrFromRS);
448            assertNotNull(addrFromPlan);
449            assertTrue(
450              "Region server " + rs.getServerName().getAddress() + " has the " + positions[j]
451                + " for region " + region.getRegionInfo().getRegionNameAsString() + " is "
452                + addrFromRS + " which is inconsistent with the plan " + addrFromPlan,
453              addrFromRS.equals(addrFromPlan));
454          }
455        }
456      }
457    }
458  }
459
460  /**
461   * Check whether regions are assigned to servers consistent with the explicit hints that are
462   * persisted in the hbase:meta table. Also keep track of the number of the regions are assigned to
463   * the primary region server.
464   * @return the number of regions are assigned to the primary region server
465   */
466  private int getNumRegionisOnPrimaryRS() throws IOException {
467    final AtomicInteger regionOnPrimaryNum = new AtomicInteger(0);
468    final AtomicInteger totalRegionNum = new AtomicInteger(0);
469    LOG.info("The start of region placement verification");
470    MetaTableAccessor.Visitor visitor = new MetaTableAccessor.Visitor() {
471      @Override
472      public boolean visit(Result result) throws IOException {
473        try {
474          @SuppressWarnings("deprecation")
475          RegionInfo info = MetaTableAccessor.getRegionInfo(result);
476          if (
477            info.getTable().getNamespaceAsString()
478              .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)
479          ) {
480            return true;
481          }
482          byte[] server = result.getValue(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
483          byte[] favoredNodes = result.getValue(HConstants.CATALOG_FAMILY,
484            FavoredNodeAssignmentHelper.FAVOREDNODES_QUALIFIER);
485          // Add the favored nodes into assignment plan
486          ServerName[] favoredServerList =
487            FavoredNodeAssignmentHelper.getFavoredNodesList(favoredNodes);
488          favoredNodesAssignmentPlan.put(info, favoredServerList);
489
490          Position[] positions = Position.values();
491          if (info != null) {
492            totalRegionNum.incrementAndGet();
493            if (server != null) {
494              ServerName serverName = ServerName.valueOf(Bytes.toString(server), -1);
495              if (favoredNodes != null) {
496                String placement = "[NOT FAVORED NODE]";
497                for (int i = 0; i < favoredServerList.length; i++) {
498                  if (favoredServerList[i].equals(serverName)) {
499                    placement = positions[i].toString();
500                    if (i == Position.PRIMARY.ordinal()) {
501                      regionOnPrimaryNum.incrementAndGet();
502                    }
503                    break;
504                  }
505                }
506                LOG.info(info.getRegionNameAsString() + " on " + serverName + " " + placement);
507              } else {
508                LOG.info(info.getRegionNameAsString() + " running on " + serverName
509                  + " but there is no favored region server");
510              }
511            } else {
512              LOG.info(info.getRegionNameAsString() + " not assigned to any server");
513            }
514          }
515          return true;
516        } catch (RuntimeException e) {
517          LOG.error("Result=" + result);
518          throw e;
519        }
520      }
521    };
522    MetaTableAccessor.fullScanRegions(CONNECTION, visitor);
523    LOG.info("There are " + regionOnPrimaryNum.intValue() + " out of " + totalRegionNum.intValue()
524      + " regions running on the primary" + " region servers");
525    return regionOnPrimaryNum.intValue();
526  }
527
528  /**
529   * Create a table with specified table name and region number.
530   * @param tableName the name of the table to be created
531   * @param regionNum number of regions to create
532   */
533  private static void createTable(TableName tableName, int regionNum) throws IOException {
534    int expectedRegions = regionNum;
535    byte[][] splitKeys = new byte[expectedRegions - 1][];
536    for (int i = 1; i < expectedRegions; i++) {
537      byte splitKey = (byte) i;
538      splitKeys[i - 1] = new byte[] { splitKey, splitKey, splitKey };
539    }
540
541    HTableDescriptor desc = new HTableDescriptor(tableName);
542    desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
543    admin.createTable(desc, splitKeys);
544
545    try (RegionLocator r = CONNECTION.getRegionLocator(tableName)) {
546      List<HRegionLocation> regions = r.getAllRegionLocations();
547      assertEquals(
548        "Tried to create " + expectedRegions + " regions " + "but only found " + regions.size(),
549        expectedRegions, regions.size());
550    }
551  }
552}