001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.assignment;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertNotEquals;
023import static org.junit.Assert.assertNotNull;
024import static org.junit.Assert.assertNull;
025import static org.junit.Assert.assertTrue;
026
027import java.time.Instant;
028import java.util.Collections;
029import java.util.List;
030import java.util.Map;
031import java.util.Optional;
032import java.util.concurrent.Future;
033import org.apache.hadoop.conf.Configuration;
034import org.apache.hadoop.hbase.HBaseClassTestRule;
035import org.apache.hadoop.hbase.HRegionInfo;
036import org.apache.hadoop.hbase.ServerName;
037import org.apache.hadoop.hbase.TableName;
038import org.apache.hadoop.hbase.client.RegionInfo;
039import org.apache.hadoop.hbase.client.RegionInfoBuilder;
040import org.apache.hadoop.hbase.client.TableState;
041import org.apache.hadoop.hbase.master.TableStateManager;
042import org.apache.hadoop.hbase.master.hbck.HbckChore;
043import org.apache.hadoop.hbase.master.hbck.HbckReport;
044import org.apache.hadoop.hbase.regionserver.HRegion;
045import org.apache.hadoop.hbase.testclassification.MasterTests;
046import org.apache.hadoop.hbase.testclassification.MediumTests;
047import org.apache.hadoop.hbase.util.Bytes;
048import org.apache.hadoop.hbase.util.CommonFSUtils;
049import org.apache.hadoop.hbase.util.FSUtils;
050import org.apache.hadoop.hbase.util.Pair;
051import org.junit.Before;
052import org.junit.ClassRule;
053import org.junit.Test;
054import org.junit.experimental.categories.Category;
055import org.mockito.Mockito;
056import org.slf4j.Logger;
057import org.slf4j.LoggerFactory;
058
059@Category({ MasterTests.class, MediumTests.class })
060public class TestHbckChore extends TestAssignmentManagerBase {
061  private static final Logger LOG = LoggerFactory.getLogger(TestHbckChore.class);
062
063  @ClassRule
064  public static final HBaseClassTestRule CLASS_RULE =
065    HBaseClassTestRule.forClass(TestHbckChore.class);
066
067  private HbckChore hbckChore;
068
069  @Before
070  public void setUp() throws Exception {
071    super.setUp();
072    hbckChore = new HbckChore(master);
073  }
074
075  @Test
076  public void testForMeta() {
077    byte[] metaRegionNameAsBytes = RegionInfoBuilder.FIRST_META_REGIONINFO.getRegionName();
078    String metaRegionName = RegionInfoBuilder.FIRST_META_REGIONINFO.getRegionNameAsString();
079    List<ServerName> serverNames = master.getServerManager().getOnlineServersList();
080    assertEquals(NSERVERS, serverNames.size());
081
082    hbckChore.choreForTesting();
083    Map<String, Pair<ServerName, List<ServerName>>> inconsistentRegions =
084      hbckChore.getLastReport().getInconsistentRegions();
085
086    // Test for case1: Master thought this region opened, but no regionserver reported it.
087    assertTrue(inconsistentRegions.containsKey(metaRegionName));
088    Pair<ServerName, List<ServerName>> pair = inconsistentRegions.get(metaRegionName);
089    ServerName locationInMeta = pair.getFirst();
090    List<ServerName> reportedRegionServers = pair.getSecond();
091    assertTrue(serverNames.contains(locationInMeta));
092    assertEquals(0, reportedRegionServers.size());
093
094    // Reported right region location. Then not in problematic regions.
095    am.reportOnlineRegions(locationInMeta, Collections.singleton(metaRegionNameAsBytes));
096    hbckChore.choreForTesting();
097    inconsistentRegions = hbckChore.getLastReport().getInconsistentRegions();
098    assertFalse(inconsistentRegions.containsKey(metaRegionName));
099  }
100
101  @Test
102  public void testForUserTable() throws Exception {
103    TableName tableName = TableName.valueOf("testForUserTable");
104    RegionInfo hri = createRegionInfo(tableName, 1);
105    String regionName = hri.getRegionNameAsString();
106    rsDispatcher.setMockRsExecutor(new GoodRsExecutor());
107    Future<byte[]> future = submitProcedure(createAssignProcedure(hri));
108    waitOnFuture(future);
109
110    List<ServerName> serverNames = master.getServerManager().getOnlineServersList();
111    assertEquals(NSERVERS, serverNames.size());
112
113    // Test for case1: Master thought this region opened, but no regionserver reported it.
114    hbckChore.choreForTesting();
115    Map<String, Pair<ServerName, List<ServerName>>> inconsistentRegions =
116      hbckChore.getLastReport().getInconsistentRegions();
117    assertTrue(inconsistentRegions.containsKey(regionName));
118    Pair<ServerName, List<ServerName>> pair = inconsistentRegions.get(regionName);
119    ServerName locationInMeta = pair.getFirst();
120    List<ServerName> reportedRegionServers = pair.getSecond();
121    assertTrue(serverNames.contains(locationInMeta));
122    assertEquals(0, reportedRegionServers.size());
123
124    // Test for case2: Master thought this region opened on Server1, but regionserver reported
125    // Server2
126    final ServerName tempLocationInMeta = locationInMeta;
127    final ServerName anotherServer =
128      serverNames.stream().filter(s -> !s.equals(tempLocationInMeta)).findFirst().get();
129    am.reportOnlineRegions(anotherServer, Collections.singleton(hri.getRegionName()));
130    hbckChore.choreForTesting();
131    inconsistentRegions = hbckChore.getLastReport().getInconsistentRegions();
132    assertTrue(inconsistentRegions.containsKey(regionName));
133    pair = inconsistentRegions.get(regionName);
134    locationInMeta = pair.getFirst();
135    reportedRegionServers = pair.getSecond();
136    assertEquals(1, reportedRegionServers.size());
137    assertFalse(reportedRegionServers.contains(locationInMeta));
138    assertTrue(reportedRegionServers.contains(anotherServer));
139
140    // Test for case3: More than one regionservers reported opened this region.
141    am.reportOnlineRegions(locationInMeta, Collections.singleton(hri.getRegionName()));
142    hbckChore.choreForTesting();
143    inconsistentRegions = hbckChore.getLastReport().getInconsistentRegions();
144    assertTrue(inconsistentRegions.containsKey(regionName));
145    pair = inconsistentRegions.get(regionName);
146    locationInMeta = pair.getFirst();
147    reportedRegionServers = pair.getSecond();
148    assertEquals(2, reportedRegionServers.size());
149    assertTrue(reportedRegionServers.contains(locationInMeta));
150    assertTrue(reportedRegionServers.contains(anotherServer));
151
152    // Reported right region location, then not in inconsistent regions.
153    am.reportOnlineRegions(anotherServer, Collections.EMPTY_SET);
154    hbckChore.choreForTesting();
155    inconsistentRegions = hbckChore.getLastReport().getInconsistentRegions();
156    assertFalse(inconsistentRegions.containsKey(regionName));
157
158    // Test for case4: No region location for a previously reported region. Probably due to
159    // TRSP bug or bypass.
160    am.offlineRegion(hri);
161    hbckChore.choreForTesting();
162    inconsistentRegions = hbckChore.getLastReport().getInconsistentRegions();
163    assertTrue(inconsistentRegions.containsKey(regionName));
164  }
165
166  @Test
167  public void testForDisabledTable() throws Exception {
168    TableName tableName = TableName.valueOf("testForDisabledTable");
169    RegionInfo hri = createRegionInfo(tableName, 1);
170    String regionName = hri.getRegionNameAsString();
171    rsDispatcher.setMockRsExecutor(new GoodRsExecutor());
172    Future<byte[]> future = submitProcedure(createAssignProcedure(hri));
173    waitOnFuture(future);
174
175    List<ServerName> serverNames = master.getServerManager().getOnlineServersList();
176    assertEquals(NSERVERS, serverNames.size());
177
178    hbckChore.choreForTesting();
179    Map<String, Pair<ServerName, List<ServerName>>> inconsistentRegions =
180      hbckChore.getLastReport().getInconsistentRegions();
181    assertTrue(inconsistentRegions.containsKey(regionName));
182    Pair<ServerName, List<ServerName>> pair = inconsistentRegions.get(regionName);
183    ServerName locationInMeta = pair.getFirst();
184    List<ServerName> reportedRegionServers = pair.getSecond();
185    assertTrue(serverNames.contains(locationInMeta));
186    assertEquals(0, reportedRegionServers.size());
187
188    // Set table state to disabled, then not in inconsistent regions.
189    TableStateManager tableStateManager = master.getTableStateManager();
190    Mockito.when(tableStateManager.isTableState(tableName, TableState.State.DISABLED))
191      .thenReturn(true);
192    hbckChore.choreForTesting();
193    inconsistentRegions = hbckChore.getLastReport().getInconsistentRegions();
194    assertFalse(inconsistentRegions.containsKey(regionName));
195  }
196
197  @Test
198  public void testForSplitParent() throws Exception {
199    TableName tableName = TableName.valueOf("testForSplitParent");
200    RegionInfo hri = RegionInfoBuilder.newBuilder(tableName).setStartKey(Bytes.toBytes(0))
201      .setEndKey(Bytes.toBytes(1)).setSplit(true).setOffline(true).setRegionId(0).build();
202    String regionName = hri.getEncodedName();
203    rsDispatcher.setMockRsExecutor(new GoodRsExecutor());
204    Future<byte[]> future = submitProcedure(createAssignProcedure(hri));
205    waitOnFuture(future);
206
207    List<ServerName> serverNames = master.getServerManager().getOnlineServersList();
208    assertEquals(NSERVERS, serverNames.size());
209
210    hbckChore.choreForTesting();
211    Map<String, Pair<ServerName, List<ServerName>>> inconsistentRegions =
212      hbckChore.getLastReport().getInconsistentRegions();
213    assertFalse(inconsistentRegions.containsKey(regionName));
214  }
215
216  @Test
217  public void testOrphanRegionsOnFS() throws Exception {
218    TableName tableName = TableName.valueOf("testOrphanRegionsOnFS");
219    RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tableName).build();
220    Configuration conf = util.getConfiguration();
221
222    hbckChore.choreForTesting();
223    assertEquals(0, hbckChore.getLastReport().getOrphanRegionsOnFS().size());
224
225    HRegion.createRegionDir(conf, regionInfo, CommonFSUtils.getRootDir(conf));
226    hbckChore.choreForTesting();
227    assertEquals(1, hbckChore.getLastReport().getOrphanRegionsOnFS().size());
228    assertTrue(
229      hbckChore.getLastReport().getOrphanRegionsOnFS().containsKey(regionInfo.getEncodedName()));
230
231    FSUtils.deleteRegionDir(conf, new HRegionInfo(regionInfo));
232    hbckChore.choreForTesting();
233    assertEquals(0, hbckChore.getLastReport().getOrphanRegionsOnFS().size());
234  }
235
236  @Test
237  public void testChoreDisable() {
238    // The way to disable to chore is to set hbase.master.hbck.chore.interval <= 0
239    // When the interval is > 0, the chore should run.
240    Instant lastRunTime = Optional.ofNullable(hbckChore.getLastReport())
241      .map(HbckReport::getCheckingEndTimestamp).orElse(null);
242    hbckChore.choreForTesting();
243    Instant thisRunTime = Optional.ofNullable(hbckChore.getLastReport())
244      .map(HbckReport::getCheckingEndTimestamp).orElse(null);
245    assertNotNull(thisRunTime);
246    assertNotEquals(lastRunTime, thisRunTime);
247
248    // When the interval <= 0, the chore shouldn't run
249    master.getConfiguration().setInt("hbase.master.hbck.chore.interval", 0);
250    HbckChore hbckChoreWithChangedConf = new HbckChore(master);
251    hbckChoreWithChangedConf.choreForTesting();
252    assertNull(hbckChoreWithChangedConf.getLastReport());
253  }
254}