001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.junit.Assert.*;
021
022import java.io.IOException;
023import java.util.ArrayList;
024import java.util.Collection;
025import org.apache.hadoop.conf.Configuration;
026import org.apache.hadoop.hbase.HBaseClassTestRule;
027import org.apache.hadoop.hbase.HBaseTestingUtil;
028import org.apache.hadoop.hbase.HConstants;
029import org.apache.hadoop.hbase.ServerName;
030import org.apache.hadoop.hbase.TableName;
031import org.apache.hadoop.hbase.client.RegionInfo;
032import org.apache.hadoop.hbase.client.RegionReplicaUtil;
033import org.apache.hadoop.hbase.client.Table;
034import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
035import org.apache.hadoop.hbase.testclassification.MediumTests;
036import org.apache.hadoop.hbase.testclassification.RegionServerTests;
037import org.apache.hadoop.hbase.util.Bytes;
038import org.apache.hadoop.hbase.util.RegionSplitter;
039import org.junit.After;
040import org.junit.AfterClass;
041import org.junit.Before;
042import org.junit.BeforeClass;
043import org.junit.ClassRule;
044import org.junit.Rule;
045import org.junit.Test;
046import org.junit.experimental.categories.Category;
047import org.junit.rules.TestName;
048import org.slf4j.Logger;
049import org.slf4j.LoggerFactory;
050
051@Category({ RegionServerTests.class, MediumTests.class })
052public class TestRegionReplicasWithRestartScenarios {
053
054  @ClassRule
055  public static final HBaseClassTestRule CLASS_RULE =
056    HBaseClassTestRule.forClass(TestRegionReplicasWithRestartScenarios.class);
057
058  private static final Logger LOG =
059    LoggerFactory.getLogger(TestRegionReplicasWithRestartScenarios.class);
060
061  @Rule
062  public TestName name = new TestName();
063
064  private static final int NB_SERVERS = 3;
065  private Table table;
066  private TableName tableName;
067
068  private static final HBaseTestingUtil HTU = new HBaseTestingUtil();
069  private static final byte[] f = HConstants.CATALOG_FAMILY;
070
071  @BeforeClass
072  public static void beforeClass() throws Exception {
073    HTU.getConfiguration().setInt("hbase.master.wait.on.regionservers.mintostart", NB_SERVERS);
074    HTU.startMiniCluster(NB_SERVERS);
075  }
076
077  @Before
078  public void before() throws IOException {
079    this.tableName = TableName.valueOf(this.name.getMethodName());
080    this.table = createTableDirectlyFromHTD(this.tableName);
081  }
082
083  @After
084  public void after() throws IOException {
085    this.table.close();
086    HTU.deleteTable(this.tableName);
087  }
088
089  private static Table createTableDirectlyFromHTD(final TableName tableName) throws IOException {
090    TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName);
091    builder.setRegionReplication(3);
092    return HTU.createTable(builder.build(), new byte[][] { f }, getSplits(20),
093      new Configuration(HTU.getConfiguration()));
094  }
095
096  private static byte[][] getSplits(int numRegions) {
097    RegionSplitter.UniformSplit split = new RegionSplitter.UniformSplit();
098    split.setFirstRow(Bytes.toBytes(0L));
099    split.setLastRow(Bytes.toBytes(Long.MAX_VALUE));
100    return split.split(numRegions);
101  }
102
103  @AfterClass
104  public static void afterClass() throws Exception {
105    HTU.shutdownMiniCluster();
106  }
107
108  private HRegionServer getRS() {
109    return HTU.getMiniHBaseCluster().getRegionServer(0);
110  }
111
112  private HRegionServer getSecondaryRS() {
113    return HTU.getMiniHBaseCluster().getRegionServer(1);
114  }
115
116  private HRegionServer getTertiaryRS() {
117    return HTU.getMiniHBaseCluster().getRegionServer(2);
118  }
119
120  @Test
121  public void testRegionReplicasCreated() throws Exception {
122    assertReplicaDistributed();
123  }
124
125  @Test
126  public void testWhenRestart() throws Exception {
127    // Start Region before stopping other so SCP has three servers to play with when it goes
128    // about assigning instead of two, depending on sequencing of SCP and RS stop/start.
129    // If two only, then it'll be forced to assign replicas alongside primaries.
130    HTU.getHBaseCluster().startRegionServerAndWait(60000).getRegionServer();
131    HRegionServer stopRegionServer = getRS();
132    ServerName serverName = stopRegionServer.getServerName();
133    // Make a copy because this is actual instance from HRegionServer
134    Collection<HRegion> regionsOnStoppedServer =
135      new ArrayList<HRegion>(stopRegionServer.getOnlineRegionsLocalContext());
136    HTU.getHBaseCluster().stopRegionServer(serverName);
137    HTU.getHBaseCluster().waitForRegionServerToStop(serverName, 60000);
138    HTU.waitTableAvailable(this.tableName);
139    assertReplicaDistributed(regionsOnStoppedServer);
140  }
141
142  private void assertReplicaDistributed() throws Exception {
143    assertReplicaDistributed(getRS().getOnlineRegionsLocalContext());
144  }
145
146  private void assertReplicaDistributed(Collection<HRegion> onlineRegions) throws Exception {
147    LOG.info("ASSERT DISTRIBUTED {}", onlineRegions);
148    boolean res = checkDuplicates(onlineRegions);
149    assertFalse(res);
150    Collection<HRegion> onlineRegions2 = getSecondaryRS().getOnlineRegionsLocalContext();
151    res = checkDuplicates(onlineRegions2);
152    assertFalse(res);
153    Collection<HRegion> onlineRegions3 = getTertiaryRS().getOnlineRegionsLocalContext();
154    checkDuplicates(onlineRegions3);
155    assertFalse(res);
156    int totalRegions = HTU.getMiniHBaseCluster().getLiveRegionServerThreads().stream()
157      .mapToInt(l -> l.getRegionServer().getOnlineRegions().size()).sum();
158    assertEquals(61, totalRegions);
159  }
160
161  private boolean checkDuplicates(Collection<HRegion> onlineRegions3) throws Exception {
162    ArrayList<Region> copyOfRegion = new ArrayList<Region>(onlineRegions3);
163    for (Region region : copyOfRegion) {
164      RegionInfo regionInfo = region.getRegionInfo();
165      RegionInfo regionInfoForReplica =
166        RegionReplicaUtil.getRegionInfoForDefaultReplica(regionInfo);
167      int i = 0;
168      for (Region actualRegion : onlineRegions3) {
169        if (
170          regionInfoForReplica
171            .equals(RegionReplicaUtil.getRegionInfoForDefaultReplica(actualRegion.getRegionInfo()))
172        ) {
173          i++;
174          if (i > 1) {
175            LOG.warn("Duplicate found {} and {}", actualRegion.getRegionInfo(),
176              region.getRegionInfo());
177            assertTrue(Bytes.equals(region.getRegionInfo().getStartKey(),
178              actualRegion.getRegionInfo().getStartKey()));
179            assertTrue(Bytes.equals(region.getRegionInfo().getEndKey(),
180              actualRegion.getRegionInfo().getEndKey()));
181            return true;
182          }
183        }
184      }
185    }
186    return false;
187  }
188}