Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.assignment;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotNull;
022import static org.junit.Assert.assertNull;
023import static org.junit.Assert.assertTrue;
024
025import java.util.Collections;
026import java.util.concurrent.Executors;
027import java.util.concurrent.Future;
028import org.apache.hadoop.hbase.HBaseClassTestRule;
029import org.apache.hadoop.hbase.HBaseTestingUtil;
030import org.apache.hadoop.hbase.MetaTableAccessor;
031import org.apache.hadoop.hbase.TableName;
032import org.apache.hadoop.hbase.client.RegionInfo;
033import org.apache.hadoop.hbase.client.RegionInfoBuilder;
034import org.apache.hadoop.hbase.master.RegionState.State;
035import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
036import org.apache.hadoop.hbase.procedure2.util.StringUtils;
037import org.apache.hadoop.hbase.testclassification.LargeTests;
038import org.apache.hadoop.hbase.testclassification.MasterTests;
039import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
040import org.junit.ClassRule;
041import org.junit.Test;
042import org.junit.experimental.categories.Category;
043import org.slf4j.Logger;
044import org.slf4j.LoggerFactory;
045
046@Category({ MasterTests.class, LargeTests.class })
047public class TestAssignmentManager extends TestAssignmentManagerBase {
048
049  @ClassRule
050  public static final HBaseClassTestRule CLASS_RULE =
051    HBaseClassTestRule.forClass(TestAssignmentManager.class);
052
053  private static final Logger LOG = LoggerFactory.getLogger(TestAssignmentManager.class);
054
055  @Test
056  public void testAssignWithGoodExec() throws Exception {
057    // collect AM metrics before test
058    collectAssignmentManagerMetrics();
059
060    testAssign(new GoodRsExecutor());
061
062    assertEquals(assignSubmittedCount + NREGIONS,
063      assignProcMetrics.getSubmittedCounter().getCount());
064    assertEquals(assignFailedCount, assignProcMetrics.getFailedCounter().getCount());
065  }
066
067  @Test
068  public void testAssignAndCrashBeforeResponse() throws Exception {
069    TableName tableName = TableName.valueOf("testAssignAndCrashBeforeResponse");
070    RegionInfo hri = createRegionInfo(tableName, 1);
071    rsDispatcher.setMockRsExecutor(new HangThenRSCrashExecutor());
072    TransitRegionStateProcedure proc = createAssignProcedure(hri);
073    waitOnFuture(submitProcedure(proc));
074  }
075
076  @Test
077  public void testUnassignAndCrashBeforeResponse() throws Exception {
078    TableName tableName = TableName.valueOf("testAssignAndCrashBeforeResponse");
079    RegionInfo hri = createRegionInfo(tableName, 1);
080    rsDispatcher.setMockRsExecutor(new HangOnCloseThenRSCrashExecutor());
081    for (int i = 0; i < HangOnCloseThenRSCrashExecutor.TYPES_OF_FAILURE; i++) {
082      TransitRegionStateProcedure assign = createAssignProcedure(hri);
083      waitOnFuture(submitProcedure(assign));
084      TransitRegionStateProcedure unassign = createUnassignProcedure(hri);
085      waitOnFuture(submitProcedure(unassign));
086    }
087  }
088
089  @Test
090  public void testAssignSocketTimeout() throws Exception {
091    TableName tableName = TableName.valueOf(this.name.getMethodName());
092    RegionInfo hri = createRegionInfo(tableName, 1);
093
094    // collect AM metrics before test
095    collectAssignmentManagerMetrics();
096
097    rsDispatcher.setMockRsExecutor(new SocketTimeoutRsExecutor(20));
098    waitOnFuture(submitProcedure(createAssignProcedure(hri)));
099
100    // we crashed a rs, so it is possible that there are other regions on the rs which will also be
101    // reassigned, so here we just assert greater than, not the exact number.
102    assertTrue(assignProcMetrics.getSubmittedCounter().getCount() > assignSubmittedCount);
103    assertEquals(assignFailedCount, assignProcMetrics.getFailedCounter().getCount());
104  }
105
106  @Test
107  public void testAssignQueueFullOnce() throws Exception {
108    TableName tableName = TableName.valueOf(this.name.getMethodName());
109    RegionInfo hri = createRegionInfo(tableName, 1);
110
111    // collect AM metrics before test
112    collectAssignmentManagerMetrics();
113
114    rsDispatcher.setMockRsExecutor(new CallQueueTooBigOnceRsExecutor());
115    waitOnFuture(submitProcedure(createAssignProcedure(hri)));
116
117    assertEquals(assignSubmittedCount + 1, assignProcMetrics.getSubmittedCounter().getCount());
118    assertEquals(assignFailedCount, assignProcMetrics.getFailedCounter().getCount());
119  }
120
121  @Test
122  public void testTimeoutThenQueueFull() throws Exception {
123    TableName tableName = TableName.valueOf(this.name.getMethodName());
124    RegionInfo hri = createRegionInfo(tableName, 1);
125
126    // collect AM metrics before test
127    collectAssignmentManagerMetrics();
128
129    rsDispatcher.setMockRsExecutor(new TimeoutThenCallQueueTooBigRsExecutor(10));
130    waitOnFuture(submitProcedure(createAssignProcedure(hri)));
131    rsDispatcher.setMockRsExecutor(new TimeoutThenCallQueueTooBigRsExecutor(15));
132    waitOnFuture(submitProcedure(createUnassignProcedure(hri)));
133
134    assertEquals(assignSubmittedCount + 1, assignProcMetrics.getSubmittedCounter().getCount());
135    assertEquals(assignFailedCount, assignProcMetrics.getFailedCounter().getCount());
136    assertEquals(unassignSubmittedCount + 1, unassignProcMetrics.getSubmittedCounter().getCount());
137    assertEquals(unassignFailedCount, unassignProcMetrics.getFailedCounter().getCount());
138  }
139
140  private void testAssign(final MockRSExecutor executor) throws Exception {
141    testAssign(executor, NREGIONS);
142  }
143
144  private void testAssign(MockRSExecutor executor, int nRegions) throws Exception {
145    rsDispatcher.setMockRsExecutor(executor);
146
147    TransitRegionStateProcedure[] assignments = new TransitRegionStateProcedure[nRegions];
148
149    long st = EnvironmentEdgeManager.currentTime();
150    bulkSubmit(assignments);
151
152    for (int i = 0; i < assignments.length; ++i) {
153      ProcedureTestingUtility.waitProcedure(master.getMasterProcedureExecutor(), assignments[i]);
154      assertTrue(assignments[i].toString(), assignments[i].isSuccess());
155    }
156    long et = EnvironmentEdgeManager.currentTime();
157    float sec = ((et - st) / 1000.0f);
158    LOG.info(String.format("[T] Assigning %dprocs in %s (%.2fproc/sec)", assignments.length,
159      StringUtils.humanTimeDiff(et - st), assignments.length / sec));
160  }
161
162  @Test
163  public void testAssignAnAssignedRegion() throws Exception {
164    final TableName tableName = TableName.valueOf("testAssignAnAssignedRegion");
165    final RegionInfo hri = createRegionInfo(tableName, 1);
166
167    // collect AM metrics before test
168    collectAssignmentManagerMetrics();
169
170    rsDispatcher.setMockRsExecutor(new GoodRsExecutor());
171
172    Future<byte[]> futureA = submitProcedure(createAssignProcedure(hri));
173
174    // wait first assign
175    waitOnFuture(futureA);
176    am.getRegionStates().isRegionInState(hri, State.OPEN);
177    // Second should be a noop. We should recognize region is already OPEN internally
178    // and skip out doing nothing.
179    // wait second assign
180    Future<byte[]> futureB = submitProcedure(createAssignProcedure(hri));
181    waitOnFuture(futureB);
182    am.getRegionStates().isRegionInState(hri, State.OPEN);
183    // TODO: What else can we do to ensure just a noop.
184
185    // TODO: Though second assign is noop, it's considered success, can noop be handled in a
186    // better way?
187    assertEquals(assignSubmittedCount + 2, assignProcMetrics.getSubmittedCounter().getCount());
188    assertEquals(assignFailedCount, assignProcMetrics.getFailedCounter().getCount());
189  }
190
191  @Test
192  public void testUnassignAnUnassignedRegion() throws Exception {
193    final TableName tableName = TableName.valueOf("testUnassignAnUnassignedRegion");
194    final RegionInfo hri = createRegionInfo(tableName, 1);
195
196    // collect AM metrics before test
197    collectAssignmentManagerMetrics();
198
199    rsDispatcher.setMockRsExecutor(new GoodRsExecutor());
200
201    // assign the region first
202    waitOnFuture(submitProcedure(createAssignProcedure(hri)));
203
204    final Future<byte[]> futureA = submitProcedure(createUnassignProcedure(hri));
205
206    // Wait first unassign.
207    waitOnFuture(futureA);
208    am.getRegionStates().isRegionInState(hri, State.CLOSED);
209    // Second should be a noop. We should recognize region is already CLOSED internally
210    // and skip out doing nothing.
211    final Future<byte[]> futureB = submitProcedure(createUnassignProcedure(hri));
212    waitOnFuture(futureB);
213    // Ensure we are still CLOSED.
214    am.getRegionStates().isRegionInState(hri, State.CLOSED);
215    // TODO: What else can we do to ensure just a noop.
216
217    assertEquals(assignSubmittedCount + 1, assignProcMetrics.getSubmittedCounter().getCount());
218    assertEquals(assignFailedCount, assignProcMetrics.getFailedCounter().getCount());
219    // TODO: Though second unassign is noop, it's considered success, can noop be handled in a
220    // better way?
221    assertEquals(unassignSubmittedCount + 2, unassignProcMetrics.getSubmittedCounter().getCount());
222    assertEquals(unassignFailedCount, unassignProcMetrics.getFailedCounter().getCount());
223  }
224
225  /**
226   * It is possible that when AM send assign meta request to a RS successfully, but RS can not send
227   * back any response, which cause master startup hangs forever
228   */
229  @Test
230  public void testAssignMetaAndCrashBeforeResponse() throws Exception {
231    tearDown();
232    // See setUp(), start HBase until set up meta
233    util = new HBaseTestingUtil();
234    this.executor = Executors.newSingleThreadScheduledExecutor();
235    setupConfiguration(util.getConfiguration());
236    master = new MockMasterServices(util.getConfiguration());
237    rsDispatcher = new MockRSProcedureDispatcher(master);
238    master.start(NSERVERS, rsDispatcher);
239    am = master.getAssignmentManager();
240
241    // Assign meta
242    rsDispatcher.setMockRsExecutor(new HangThenRSRestartExecutor());
243    am.assign(RegionInfoBuilder.FIRST_META_REGIONINFO);
244    assertEquals(true, am.isMetaAssigned());
245
246    // set it back as default, see setUpMeta()
247    am.wakeMetaLoadedEvent();
248  }
249
250  private void assertCloseThenOpen() {
251    assertEquals(closeSubmittedCount + 1, closeProcMetrics.getSubmittedCounter().getCount());
252    assertEquals(closeFailedCount, closeProcMetrics.getFailedCounter().getCount());
253    assertEquals(openSubmittedCount + 1, openProcMetrics.getSubmittedCounter().getCount());
254    assertEquals(openFailedCount, openProcMetrics.getFailedCounter().getCount());
255  }
256
257  @Test
258  public void testMove() throws Exception {
259    TableName tableName = TableName.valueOf("testMove");
260    RegionInfo hri = createRegionInfo(tableName, 1);
261    rsDispatcher.setMockRsExecutor(new GoodRsExecutor());
262    am.assign(hri);
263
264    // collect AM metrics before test
265    collectAssignmentManagerMetrics();
266
267    am.move(hri);
268
269    assertEquals(moveSubmittedCount + 1, moveProcMetrics.getSubmittedCounter().getCount());
270    assertEquals(moveFailedCount, moveProcMetrics.getFailedCounter().getCount());
271    assertCloseThenOpen();
272  }
273
274  @Test
275  public void testReopen() throws Exception {
276    TableName tableName = TableName.valueOf("testReopen");
277    RegionInfo hri = createRegionInfo(tableName, 1);
278    rsDispatcher.setMockRsExecutor(new GoodRsExecutor());
279    am.assign(hri);
280
281    // collect AM metrics before test
282    collectAssignmentManagerMetrics();
283
284    TransitRegionStateProcedure proc =
285      TransitRegionStateProcedure.reopen(master.getMasterProcedureExecutor().getEnvironment(), hri);
286    am.getRegionStates().getRegionStateNode(hri).setProcedure(proc);
287    waitOnFuture(submitProcedure(proc));
288
289    assertEquals(reopenSubmittedCount + 1, reopenProcMetrics.getSubmittedCounter().getCount());
290    assertEquals(reopenFailedCount, reopenProcMetrics.getFailedCounter().getCount());
291    assertCloseThenOpen();
292  }
293
294  @Test
295  public void testLoadRegionFromMetaAfterRegionManuallyAdded() throws Exception {
296    try {
297      this.util.startMiniCluster();
298      final AssignmentManager am = this.util.getHBaseCluster().getMaster().getAssignmentManager();
299      final TableName tableName =
300        TableName.valueOf("testLoadRegionFromMetaAfterRegionManuallyAdded");
301      this.util.createTable(tableName, "f");
302      RegionInfo hri = createRegionInfo(tableName, 1);
303      assertNull("RegionInfo was just instantiated by the test, but "
304        + "shouldn't be in AM regionStates yet.", am.getRegionStates().getRegionState(hri));
305      MetaTableAccessor.addRegionsToMeta(this.util.getConnection(), Collections.singletonList(hri),
306        1);
307      // TODO: is there a race here -- no other thread else will refresh the table states behind
308      // the scenes?
309      assertNull("RegionInfo was manually added in META, but shouldn't be in AM regionStates yet.",
310        am.getRegionStates().getRegionState(hri));
311      am.populateRegionStatesFromMeta(hri.getEncodedName());
312      assertNotNull(am.getRegionInfo(hri.getRegionName()));
313      assertNotNull(am.getRegionInfo(hri.getEncodedName()));
314    } finally {
315      this.util.killMiniHBaseCluster();
316    }
317  }
318
319  @Test
320  public void testLoadRegionFromMetaRegionNotInMeta() throws Exception {
321    try {
322      this.util.startMiniCluster();
323      final AssignmentManager am = this.util.getHBaseCluster().getMaster().getAssignmentManager();
324      final TableName tableName = TableName.valueOf("testLoadRegionFromMetaRegionNotInMeta");
325      this.util.createTable(tableName, "f");
326      final RegionInfo hri = createRegionInfo(tableName, 1);
327      assertNull("Bogus RegionInfo discovered in RegionStates.",
328        am.getRegionStates().getRegionState(hri));
329      am.populateRegionStatesFromMeta(hri.getEncodedName());
330      assertNull("RegionInfo was never added in META", am.getRegionStates().getRegionState(hri));
331    } finally {
332      this.util.killMiniHBaseCluster();
333    }
334  }
335}