001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.normalizer;
019
020import static java.lang.String.format;
021import static org.apache.hadoop.hbase.master.normalizer.RegionNormalizerWorker.CUMULATIVE_SIZE_LIMIT_MB_KEY;
022import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.DEFAULT_MERGE_MIN_REGION_AGE_DAYS;
023import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_ENABLED_KEY;
024import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_MIN_REGION_AGE_DAYS_KEY;
025import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_MIN_REGION_COUNT_KEY;
026import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_MIN_REGION_SIZE_MB_KEY;
027import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY;
028import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MIN_REGION_COUNT_KEY;
029import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.SPLIT_ENABLED_KEY;
030import static org.hamcrest.MatcherAssert.assertThat;
031import static org.hamcrest.Matchers.contains;
032import static org.hamcrest.Matchers.empty;
033import static org.hamcrest.Matchers.everyItem;
034import static org.hamcrest.Matchers.greaterThanOrEqualTo;
035import static org.hamcrest.Matchers.hasSize;
036import static org.hamcrest.Matchers.instanceOf;
037import static org.hamcrest.Matchers.not;
038import static org.junit.Assert.assertEquals;
039import static org.junit.Assert.assertFalse;
040import static org.junit.Assert.assertTrue;
041import static org.mockito.ArgumentMatchers.any;
042import static org.mockito.ArgumentMatchers.anyList;
043import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
044import static org.mockito.Mockito.spy;
045import static org.mockito.Mockito.times;
046import static org.mockito.Mockito.verify;
047import static org.mockito.Mockito.when;
048
049import java.time.Instant;
050import java.time.Period;
051import java.util.ArrayList;
052import java.util.Collections;
053import java.util.HashMap;
054import java.util.List;
055import java.util.Map;
056import org.apache.hadoop.conf.Configuration;
057import org.apache.hadoop.hbase.HBaseClassTestRule;
058import org.apache.hadoop.hbase.HBaseConfiguration;
059import org.apache.hadoop.hbase.RegionMetrics;
060import org.apache.hadoop.hbase.ServerName;
061import org.apache.hadoop.hbase.Size;
062import org.apache.hadoop.hbase.TableName;
063import org.apache.hadoop.hbase.TableNameTestRule;
064import org.apache.hadoop.hbase.client.RegionInfo;
065import org.apache.hadoop.hbase.client.RegionInfoBuilder;
066import org.apache.hadoop.hbase.client.TableDescriptor;
067import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
068import org.apache.hadoop.hbase.master.MasterServices;
069import org.apache.hadoop.hbase.master.RegionState;
070import org.apache.hadoop.hbase.testclassification.MasterTests;
071import org.apache.hadoop.hbase.testclassification.SmallTests;
072import org.apache.hadoop.hbase.util.Bytes;
073import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
074import org.junit.Before;
075import org.junit.ClassRule;
076import org.junit.Rule;
077import org.junit.Test;
078import org.junit.experimental.categories.Category;
079import org.mockito.Mockito;
080
081/**
082 * Tests logic of {@link SimpleRegionNormalizer}.
083 */
084@Category({ MasterTests.class, SmallTests.class })
085public class TestSimpleRegionNormalizer {
086
087  @ClassRule
088  public static final HBaseClassTestRule CLASS_RULE =
089    HBaseClassTestRule.forClass(TestSimpleRegionNormalizer.class);
090
091  private Configuration conf;
092  private SimpleRegionNormalizer normalizer;
093  private MasterServices masterServices;
094  private TableDescriptor tableDescriptor;
095
096  @Rule
097  public TableNameTestRule name = new TableNameTestRule();
098
099  @Before
100  public void before() {
101    conf = HBaseConfiguration.create();
102    tableDescriptor = TableDescriptorBuilder.newBuilder(name.getTableName()).build();
103  }
104
105  @Test
106  public void testNoNormalizationForMetaTable() {
107    TableName testTable = TableName.META_TABLE_NAME;
108    TableDescriptor testMetaTd = TableDescriptorBuilder.newBuilder(testTable).build();
109    List<RegionInfo> RegionInfo = new ArrayList<>();
110    Map<byte[], Integer> regionSizes = new HashMap<>();
111
112    setupMocksForNormalizer(regionSizes, RegionInfo);
113    List<NormalizationPlan> plans = normalizer.computePlansForTable(testMetaTd);
114    assertThat(plans, empty());
115  }
116
117  @Test
118  public void testNoNormalizationIfTooFewRegions() {
119    final TableName tableName = name.getTableName();
120    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 2);
121    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 10, 15);
122    setupMocksForNormalizer(regionSizes, regionInfos);
123
124    List<NormalizationPlan> plans = normalizer.computePlansForTable(tableDescriptor);
125    assertThat(plans, empty());
126  }
127
128  @Test
129  public void testNoNormalizationOnNormalizedCluster() {
130    final TableName tableName = name.getTableName();
131    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 4);
132    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 10, 15, 8, 10);
133    setupMocksForNormalizer(regionSizes, regionInfos);
134
135    List<NormalizationPlan> plans = normalizer.computePlansForTable(tableDescriptor);
136    assertThat(plans, empty());
137  }
138
139  private void noNormalizationOnTransitioningRegions(final RegionState.State state) {
140    final TableName tableName = name.getTableName();
141    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 3);
142    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 10, 1, 100);
143
144    setupMocksForNormalizer(regionSizes, regionInfos);
145    when(
146      masterServices.getAssignmentManager().getRegionStates().getRegionState(any(RegionInfo.class)))
147        .thenReturn(RegionState.createForTesting(null, state));
148    assertThat(normalizer.getMergeMinRegionCount(), greaterThanOrEqualTo(regionInfos.size()));
149
150    List<NormalizationPlan> plans = normalizer.computePlansForTable(tableDescriptor);
151    assertThat(format("Unexpected plans for RegionState %s", state), plans, empty());
152  }
153
154  @Test
155  public void testNoNormalizationOnMergingNewRegions() {
156    noNormalizationOnTransitioningRegions(RegionState.State.MERGING_NEW);
157  }
158
159  @Test
160  public void testNoNormalizationOnMergingRegions() {
161    noNormalizationOnTransitioningRegions(RegionState.State.MERGING);
162  }
163
164  @Test
165  public void testNoNormalizationOnMergedRegions() {
166    noNormalizationOnTransitioningRegions(RegionState.State.MERGED);
167  }
168
169  @Test
170  public void testNoNormalizationOnSplittingNewRegions() {
171    noNormalizationOnTransitioningRegions(RegionState.State.SPLITTING_NEW);
172  }
173
174  @Test
175  public void testNoNormalizationOnSplittingRegions() {
176    noNormalizationOnTransitioningRegions(RegionState.State.SPLITTING);
177  }
178
179  @Test
180  public void testNoNormalizationOnSplitRegions() {
181    noNormalizationOnTransitioningRegions(RegionState.State.SPLIT);
182  }
183
184  @Test
185  public void testMergeOfSmallRegions() {
186    final TableName tableName = name.getTableName();
187    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 5);
188    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 15, 5, 5, 15, 16);
189    setupMocksForNormalizer(regionSizes, regionInfos);
190
191    assertThat(normalizer.computePlansForTable(tableDescriptor),
192      contains(new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(1), 5)
193        .addTarget(regionInfos.get(2), 5).build()));
194  }
195
196  // Test for situation illustrated in HBASE-14867
197  @Test
198  public void testMergeOfSecondSmallestRegions() {
199    final TableName tableName = name.getTableName();
200    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 6);
201    final Map<byte[], Integer> regionSizes =
202      createRegionSizesMap(regionInfos, 1, 10000, 10000, 10000, 2700, 2700);
203    setupMocksForNormalizer(regionSizes, regionInfos);
204
205    assertThat(normalizer.computePlansForTable(tableDescriptor),
206      contains(new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(4), 2700)
207        .addTarget(regionInfos.get(5), 2700).build()));
208  }
209
210  @Test
211  public void testMergeOfSmallNonAdjacentRegions() {
212    final TableName tableName = name.getTableName();
213    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 5);
214    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 15, 5, 16, 15, 5);
215    setupMocksForNormalizer(regionSizes, regionInfos);
216
217    List<NormalizationPlan> plans = normalizer.computePlansForTable(tableDescriptor);
218    assertThat(plans, empty());
219  }
220
221  @Test
222  public void testSplitOfLargeRegion() {
223    final TableName tableName = name.getTableName();
224    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 4);
225    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 8, 6, 10, 30);
226    setupMocksForNormalizer(regionSizes, regionInfos);
227
228    assertThat(normalizer.computePlansForTable(tableDescriptor),
229      contains(new SplitNormalizationPlan(regionInfos.get(3), 30)));
230  }
231
232  @Test
233  public void testWithTargetRegionSize() throws Exception {
234    final TableName tableName = name.getTableName();
235    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 6);
236    final Map<byte[], Integer> regionSizes =
237      createRegionSizesMap(regionInfos, 20, 40, 60, 80, 100, 120);
238    setupMocksForNormalizer(regionSizes, regionInfos);
239
240    // test when target region size is 20
241    when(tableDescriptor.getNormalizerTargetRegionSize()).thenReturn(20L);
242    assertThat(normalizer.computePlansForTable(tableDescriptor),
243      contains(new SplitNormalizationPlan(regionInfos.get(2), 60),
244        new SplitNormalizationPlan(regionInfos.get(3), 80),
245        new SplitNormalizationPlan(regionInfos.get(4), 100),
246        new SplitNormalizationPlan(regionInfos.get(5), 120)));
247
248    // test when target region size is 200
249    when(tableDescriptor.getNormalizerTargetRegionSize()).thenReturn(200L);
250    assertThat(normalizer.computePlansForTable(tableDescriptor),
251      contains(new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(0), 20)
252        .addTarget(regionInfos.get(1), 40).addTarget(regionInfos.get(2), 60)
253        .addTarget(regionInfos.get(3), 80).build()));
254  }
255
256  @Test
257  public void testSplitWithTargetRegionCount() throws Exception {
258    final TableName tableName = name.getTableName();
259    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 4);
260    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 20, 40, 60, 80);
261    setupMocksForNormalizer(regionSizes, regionInfos);
262
263    // test when target region count is 8
264    when(tableDescriptor.getNormalizerTargetRegionCount()).thenReturn(8);
265    assertThat(normalizer.computePlansForTable(tableDescriptor),
266      contains(new SplitNormalizationPlan(regionInfos.get(2), 60),
267        new SplitNormalizationPlan(regionInfos.get(3), 80)));
268
269    // test when target region count is 3
270    when(tableDescriptor.getNormalizerTargetRegionCount()).thenReturn(3);
271    assertThat(normalizer.computePlansForTable(tableDescriptor),
272      contains(new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(0), 20)
273        .addTarget(regionInfos.get(1), 40).build()));
274  }
275
276  @Test
277  public void testHonorsSplitEnabled() {
278    conf.setBoolean(SPLIT_ENABLED_KEY, true);
279    final TableName tableName = name.getTableName();
280    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 5);
281    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 5, 5, 20, 5, 5);
282    setupMocksForNormalizer(regionSizes, regionInfos);
283    assertThat(normalizer.computePlansForTable(tableDescriptor),
284      contains(instanceOf(SplitNormalizationPlan.class)));
285
286    conf.setBoolean(SPLIT_ENABLED_KEY, false);
287    setupMocksForNormalizer(regionSizes, regionInfos);
288    assertThat(normalizer.computePlansForTable(tableDescriptor), empty());
289  }
290
291  @Test
292  public void testHonorsSplitEnabledInTD() {
293    conf.setBoolean(SPLIT_ENABLED_KEY, true);
294    final TableName tableName = name.getTableName();
295    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 5);
296    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 5, 5, 20, 5, 5);
297    setupMocksForNormalizer(regionSizes, regionInfos);
298    assertThat(normalizer.computePlansForTable(tableDescriptor),
299      contains(instanceOf(SplitNormalizationPlan.class)));
300
301    // When hbase.normalizer.split.enabled is true in configuration, but false in table descriptor
302    when(tableDescriptor.getValue(SPLIT_ENABLED_KEY)).thenReturn("false");
303    assertThat(normalizer.computePlansForTable(tableDescriptor), empty());
304
305    // When hbase.normalizer.split.enabled is false in configuration, but true in table descriptor
306    conf.setBoolean(SPLIT_ENABLED_KEY, false);
307    setupMocksForNormalizer(regionSizes, regionInfos);
308    when(tableDescriptor.getValue(SPLIT_ENABLED_KEY)).thenReturn("true");
309    assertThat(normalizer.computePlansForTable(tableDescriptor),
310      contains(instanceOf(SplitNormalizationPlan.class)));
311  }
312
313  @Test
314  public void testHonorsMergeEnabled() {
315    conf.setBoolean(MERGE_ENABLED_KEY, true);
316    final TableName tableName = name.getTableName();
317    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 5);
318    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 20, 5, 5, 20, 20);
319    setupMocksForNormalizer(regionSizes, regionInfos);
320    assertThat(normalizer.computePlansForTable(tableDescriptor),
321      contains(instanceOf(MergeNormalizationPlan.class)));
322
323    conf.setBoolean(MERGE_ENABLED_KEY, false);
324    setupMocksForNormalizer(regionSizes, regionInfos);
325    assertThat(normalizer.computePlansForTable(tableDescriptor), empty());
326  }
327
328  @Test
329  public void testHonorsMergeEnabledInTD() {
330    conf.setBoolean(MERGE_ENABLED_KEY, true);
331    final TableName tableName = name.getTableName();
332    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 5);
333    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 20, 5, 5, 20, 20);
334    setupMocksForNormalizer(regionSizes, regionInfos);
335    assertThat(normalizer.computePlansForTable(tableDescriptor),
336      contains(instanceOf(MergeNormalizationPlan.class)));
337
338    // When hbase.normalizer.merge.enabled is true in configuration, but false in table descriptor
339    when(tableDescriptor.getValue(MERGE_ENABLED_KEY)).thenReturn("false");
340    assertThat(normalizer.computePlansForTable(tableDescriptor), empty());
341
342    // When hbase.normalizer.merge.enabled is false in configuration, but true in table descriptor
343    conf.setBoolean(MERGE_ENABLED_KEY, false);
344    setupMocksForNormalizer(regionSizes, regionInfos);
345    when(tableDescriptor.getValue(MERGE_ENABLED_KEY)).thenReturn("true");
346    assertThat(normalizer.computePlansForTable(tableDescriptor),
347      contains(instanceOf(MergeNormalizationPlan.class)));
348  }
349
350  @Test
351  public void testHonorsMinimumRegionCount() {
352    honorsMinimumRegionCount(MERGE_MIN_REGION_COUNT_KEY);
353  }
354
355  /**
356   * Test the backward compatibility of the deprecated MIN_REGION_COUNT_KEY configuration.
357   */
358  @Test
359  public void testHonorsOldMinimumRegionCount() {
360    honorsMinimumRegionCount(MIN_REGION_COUNT_KEY);
361  }
362
363  private void honorsMinimumRegionCount(String confKey) {
364    conf.setInt(confKey, 1);
365    final TableName tableName = name.getTableName();
366    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 3);
367    // create a table topology that results in both a merge plan and a split plan. Assert that the
368    // merge is only created when the when the number of table regions is above the region count
369    // threshold, and that the split plan is create in both cases.
370    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 1, 1, 10);
371    setupMocksForNormalizer(regionSizes, regionInfos);
372    assertEquals(1, normalizer.getMergeMinRegionCount());
373
374    List<NormalizationPlan> plans = normalizer.computePlansForTable(tableDescriptor);
375    assertThat(plans,
376      contains(new SplitNormalizationPlan(regionInfos.get(2), 10),
377        new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(0), 1)
378          .addTarget(regionInfos.get(1), 1).build()));
379
380    // have to call setupMocks again because we don't have dynamic config update on normalizer.
381    conf.setInt(confKey, 4);
382    setupMocksForNormalizer(regionSizes, regionInfos);
383    assertEquals(4, normalizer.getMergeMinRegionCount());
384    assertThat(normalizer.computePlansForTable(tableDescriptor),
385      contains(new SplitNormalizationPlan(regionInfos.get(2), 10)));
386  }
387
388  @Test
389  public void testHonorsMinimumRegionCountInTD() {
390    honorsOldMinimumRegionCountInTD(MERGE_MIN_REGION_COUNT_KEY);
391  }
392
393  /**
394   * Test the backward compatibility of the deprecated MIN_REGION_COUNT_KEY configuration in table
395   * descriptor.
396   */
397  @Test
398  public void testHonorsOldMinimumRegionCountInTD() {
399    honorsOldMinimumRegionCountInTD(MIN_REGION_COUNT_KEY);
400  }
401
402  private void honorsOldMinimumRegionCountInTD(String confKey) {
403    conf.setInt(confKey, 1);
404    final TableName tableName = name.getTableName();
405    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 3);
406    // create a table topology that results in both a merge plan and a split plan. Assert that the
407    // merge is only created when the when the number of table regions is above the region count
408    // threshold, and that the split plan is create in both cases.
409    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 1, 1, 10);
410    setupMocksForNormalizer(regionSizes, regionInfos);
411    assertEquals(1, normalizer.getMergeMinRegionCount());
412
413    List<NormalizationPlan> plans = normalizer.computePlansForTable(tableDescriptor);
414    assertThat(plans,
415      contains(new SplitNormalizationPlan(regionInfos.get(2), 10),
416        new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(0), 1)
417          .addTarget(regionInfos.get(1), 1).build()));
418
419    when(tableDescriptor.getValue(confKey)).thenReturn("4");
420    assertThat(normalizer.computePlansForTable(tableDescriptor),
421      contains(new SplitNormalizationPlan(regionInfos.get(2), 10)));
422  }
423
424  @Test
425  public void testHonorsMergeMinRegionAge() {
426    conf.setInt(MERGE_MIN_REGION_AGE_DAYS_KEY, 7);
427    final TableName tableName = name.getTableName();
428    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 4);
429    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 1, 1, 10, 10);
430    setupMocksForNormalizer(regionSizes, regionInfos);
431    assertEquals(Period.ofDays(7), normalizer.getMergeMinRegionAge());
432    assertThat(normalizer.computePlansForTable(tableDescriptor),
433      everyItem(not(instanceOf(MergeNormalizationPlan.class))));
434
435    // have to call setupMocks again because we don't have dynamic config update on normalizer.
436    conf.unset(MERGE_MIN_REGION_AGE_DAYS_KEY);
437    setupMocksForNormalizer(regionSizes, regionInfos);
438    assertEquals(Period.ofDays(DEFAULT_MERGE_MIN_REGION_AGE_DAYS),
439      normalizer.getMergeMinRegionAge());
440    final List<NormalizationPlan> plans = normalizer.computePlansForTable(tableDescriptor);
441    assertThat(plans, not(empty()));
442    assertThat(plans, everyItem(instanceOf(MergeNormalizationPlan.class)));
443  }
444
445  @Test
446  public void testHonorsMergeMinRegionAgeInTD() {
447    conf.setInt(MERGE_MIN_REGION_AGE_DAYS_KEY, 7);
448    final TableName tableName = name.getTableName();
449    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 4);
450    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 1, 1, 10, 10);
451    setupMocksForNormalizer(regionSizes, regionInfos);
452    assertEquals(Period.ofDays(7), normalizer.getMergeMinRegionAge());
453    assertThat(normalizer.computePlansForTable(tableDescriptor),
454      everyItem(not(instanceOf(MergeNormalizationPlan.class))));
455
456    conf.unset(MERGE_MIN_REGION_AGE_DAYS_KEY);
457    setupMocksForNormalizer(regionSizes, regionInfos);
458    when(tableDescriptor.getValue(MERGE_MIN_REGION_AGE_DAYS_KEY)).thenReturn("-1");
459    List<NormalizationPlan> plans = normalizer.computePlansForTable(tableDescriptor);
460    assertThat(plans, not(empty()));
461    assertThat(plans, everyItem(instanceOf(MergeNormalizationPlan.class)));
462
463    when(tableDescriptor.getValue(MERGE_MIN_REGION_AGE_DAYS_KEY)).thenReturn("5");
464    plans = normalizer.computePlansForTable(tableDescriptor);
465    assertThat(plans, empty());
466    assertThat(plans, everyItem(not(instanceOf(MergeNormalizationPlan.class))));
467  }
468
469  @Test
470  public void testHonorsMergeMinRegionSize() {
471    conf.setBoolean(SPLIT_ENABLED_KEY, false);
472    final TableName tableName = name.getTableName();
473    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 5);
474    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 1, 2, 0, 10, 10);
475    setupMocksForNormalizer(regionSizes, regionInfos);
476
477    assertFalse(normalizer.isSplitEnabled());
478    assertEquals(1, normalizer.getMergeMinRegionSizeMb());
479    assertThat(normalizer.computePlansForTable(tableDescriptor),
480      contains(new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(0), 1)
481        .addTarget(regionInfos.get(1), 2).build()));
482
483    conf.setInt(MERGE_MIN_REGION_SIZE_MB_KEY, 3);
484    setupMocksForNormalizer(regionSizes, regionInfos);
485    assertEquals(3, normalizer.getMergeMinRegionSizeMb());
486    assertThat(normalizer.computePlansForTable(tableDescriptor), empty());
487  }
488
489  @Test
490  public void testHonorsMergeMinRegionSizeInTD() {
491    conf.setBoolean(SPLIT_ENABLED_KEY, false);
492    final TableName tableName = name.getTableName();
493    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 5);
494    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 1, 2, 0, 10, 10);
495    setupMocksForNormalizer(regionSizes, regionInfos);
496
497    assertFalse(normalizer.isSplitEnabled());
498    assertEquals(1, normalizer.getMergeMinRegionSizeMb());
499    assertThat(normalizer.computePlansForTable(tableDescriptor),
500      contains(new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(0), 1)
501        .addTarget(regionInfos.get(1), 2).build()));
502
503    when(tableDescriptor.getValue(MERGE_MIN_REGION_SIZE_MB_KEY)).thenReturn("3");
504    assertThat(normalizer.computePlansForTable(tableDescriptor), empty());
505  }
506
507  @Test
508  public void testHonorsMergeRequestMaxNumberOfRegionsCount() {
509    conf.setBoolean(SPLIT_ENABLED_KEY, false);
510    conf.setInt(MERGE_MIN_REGION_COUNT_KEY, 1);
511    conf.setInt(MERGE_MIN_REGION_SIZE_MB_KEY, 0);
512    conf.setInt(MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY, 3);
513    final TableName tableName = name.getTableName();
514    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 5);
515    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 0, 1, 0, 1, 0);
516    setupMocksForNormalizer(regionSizes, regionInfos);
517    assertEquals(3, normalizer.getMergeRequestMaxNumberOfRegionsCount());
518    List<NormalizationPlan> plans = normalizer.computePlansForTable(tableDescriptor);
519    assertThat(plans,
520      contains(
521        new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(0), 0)
522          .addTarget(regionInfos.get(1), 1).addTarget(regionInfos.get(2), 0).build(),
523        new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(3), 1)
524          .addTarget(regionInfos.get(4), 0).build()));
525  }
526
527  @Test
528  public void testHonorsMergeRequestMaxNumberOfRegionsCountDefault() {
529    conf.setBoolean(SPLIT_ENABLED_KEY, false);
530    conf.setInt(MERGE_MIN_REGION_COUNT_KEY, 1);
531    conf.setInt(MERGE_MIN_REGION_SIZE_MB_KEY, 0);
532    final TableName tableName = name.getTableName();
533    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 3);
534    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 0, 0, 0);
535    setupMocksForNormalizer(regionSizes, regionInfos);
536    assertEquals(100, normalizer.getMergeRequestMaxNumberOfRegionsCount());
537    List<NormalizationPlan> plans = normalizer.computePlansForTable(tableDescriptor);
538    assertThat(plans, contains(new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(0), 0)
539      .addTarget(regionInfos.get(1), 0).addTarget(regionInfos.get(2), 0).build()));
540  }
541
542  @Test
543  public void testMergeEmptyRegions0() {
544    conf.setBoolean(SPLIT_ENABLED_KEY, false);
545    conf.setInt(MERGE_MIN_REGION_SIZE_MB_KEY, 0);
546    final TableName tableName = name.getTableName();
547    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 7);
548    final Map<byte[], Integer> regionSizes =
549      createRegionSizesMap(regionInfos, 0, 1, 10, 0, 9, 10, 0);
550    setupMocksForNormalizer(regionSizes, regionInfos);
551
552    assertFalse(normalizer.isSplitEnabled());
553    assertEquals(0, normalizer.getMergeMinRegionSizeMb());
554    assertThat(normalizer.computePlansForTable(tableDescriptor),
555      contains(
556        new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(0), 0)
557          .addTarget(regionInfos.get(1), 1).build(),
558        new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(2), 10)
559          .addTarget(regionInfos.get(3), 0).build(),
560        new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(5), 10)
561          .addTarget(regionInfos.get(6), 0).build()));
562  }
563
564  @Test
565  public void testMergeEmptyRegions1() {
566    conf.setBoolean(SPLIT_ENABLED_KEY, false);
567    conf.setInt(MERGE_MIN_REGION_SIZE_MB_KEY, 0);
568    final TableName tableName = name.getTableName();
569    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 8);
570    final Map<byte[], Integer> regionSizes =
571      createRegionSizesMap(regionInfos, 0, 1, 10, 0, 9, 0, 10, 0);
572    setupMocksForNormalizer(regionSizes, regionInfos);
573
574    assertFalse(normalizer.isSplitEnabled());
575    assertEquals(0, normalizer.getMergeMinRegionSizeMb());
576    assertThat(normalizer.computePlansForTable(tableDescriptor),
577      contains(
578        new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(0), 0)
579          .addTarget(regionInfos.get(1), 1).build(),
580        new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(2), 10)
581          .addTarget(regionInfos.get(3), 0).build(),
582        new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(4), 9)
583          .addTarget(regionInfos.get(5), 0).build(),
584        new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(6), 10)
585          .addTarget(regionInfos.get(7), 0).build()));
586  }
587
588  @Test
589  public void testMergeEmptyRegions2() {
590    conf.setBoolean(SPLIT_ENABLED_KEY, false);
591    conf.setInt(MERGE_MIN_REGION_SIZE_MB_KEY, 0);
592    final TableName tableName = name.getTableName();
593    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 8);
594    final Map<byte[], Integer> regionSizes =
595      createRegionSizesMap(regionInfos, 0, 10, 1, 0, 9, 0, 10, 0);
596    setupMocksForNormalizer(regionSizes, regionInfos);
597
598    assertFalse(normalizer.isSplitEnabled());
599    assertEquals(0, normalizer.getMergeMinRegionSizeMb());
600    List<NormalizationPlan> plans = normalizer.computePlansForTable(tableDescriptor);
601    assertThat(plans,
602      contains(
603        new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(0), 0)
604          .addTarget(regionInfos.get(1), 10).build(),
605        new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(2), 1)
606          .addTarget(regionInfos.get(3), 0).build(),
607        new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(4), 9)
608          .addTarget(regionInfos.get(5), 0).build(),
609        new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(6), 10)
610          .addTarget(regionInfos.get(7), 0).build()));
611  }
612
613  @Test
614  public void testSplitAndMultiMerge() {
615    conf.setInt(MERGE_MIN_REGION_SIZE_MB_KEY, 0);
616    final TableName tableName = name.getTableName();
617    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 8);
618    final Map<byte[], Integer> regionSizes =
619      createRegionSizesMap(regionInfos, 3, 1, 1, 30, 9, 3, 1, 0);
620    setupMocksForNormalizer(regionSizes, regionInfos);
621
622    assertTrue(normalizer.isMergeEnabled());
623    assertTrue(normalizer.isSplitEnabled());
624    assertEquals(0, normalizer.getMergeMinRegionSizeMb());
625    assertThat(normalizer.computePlansForTable(tableDescriptor),
626      contains(new SplitNormalizationPlan(regionInfos.get(3), 30),
627        new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(0), 3)
628          .addTarget(regionInfos.get(1), 1).addTarget(regionInfos.get(2), 1).build(),
629        new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(5), 3)
630          .addTarget(regionInfos.get(6), 1).addTarget(regionInfos.get(7), 0).build()));
631  }
632
633  // This test is to make sure that normalizer is only going to merge adjacent regions.
634  @Test
635  public void testNormalizerCannotMergeNonAdjacentRegions() {
636    final TableName tableName = name.getTableName();
637    // create 5 regions with sizes to trigger merge of small regions. region ranges are:
638    // [, "aa"), ["aa", "aa1"), ["aa1", "aa1!"), ["aa1!", "aa2"), ["aa2", )
639    // Region ["aa", "aa1") and ["aa1!", "aa2") are not adjacent, they are not supposed to
640    // merged.
641    final byte[][] keys = { null, Bytes.toBytes("aa"), Bytes.toBytes("aa1!"), Bytes.toBytes("aa1"),
642      Bytes.toBytes("aa2"), null, };
643    final List<RegionInfo> regionInfos = createRegionInfos(tableName, keys);
644    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 3, 1, 1, 3, 5);
645    setupMocksForNormalizer(regionSizes, regionInfos);
646
647    // Compute the plan, no merge plan returned as they are not adjacent.
648    List<NormalizationPlan> plans = normalizer.computePlansForTable(tableDescriptor);
649    assertThat(plans, empty());
650  }
651
652  @Test
653  public void testSizeLimitShufflesPlans() {
654    conf.setLong(CUMULATIVE_SIZE_LIMIT_MB_KEY, 10);
655    final TableName tableName = name.getTableName();
656    final List<RegionInfo> regionInfos = createRegionInfos(tableName, 4);
657    final Map<byte[], Integer> regionSizes = createRegionSizesMap(regionInfos, 3, 3, 3, 3);
658    setupMocksForNormalizer(regionSizes, regionInfos);
659    when(tableDescriptor.getNormalizerTargetRegionSize()).thenReturn(1L);
660    normalizer = spy(normalizer);
661
662    assertTrue(normalizer.isSplitEnabled());
663    assertTrue(normalizer.isMergeEnabled());
664    List<NormalizationPlan> computedPlans = normalizer.computePlansForTable(tableDescriptor);
665    assertThat(computedPlans, hasSize(4));
666    verify(normalizer, times(1)).shuffleNormalizationPlans(anyList());
667  }
668
669  @SuppressWarnings("MockitoCast")
670  private void setupMocksForNormalizer(Map<byte[], Integer> regionSizes,
671    List<RegionInfo> regionInfoList) {
672    masterServices = Mockito.mock(MasterServices.class, RETURNS_DEEP_STUBS);
673    tableDescriptor = Mockito.mock(TableDescriptor.class, RETURNS_DEEP_STUBS);
674
675    // for simplicity all regions are assumed to be on one server; doesn't matter to us
676    ServerName sn = ServerName.valueOf("localhost", 0, 0L);
677    when(masterServices.getAssignmentManager().getRegionStates().getRegionsOfTable(any()))
678      .thenReturn(regionInfoList);
679    when(masterServices.getAssignmentManager().getRegionStates().getRegionServerOfRegion(any()))
680      .thenReturn(sn);
681    when(
682      masterServices.getAssignmentManager().getRegionStates().getRegionState(any(RegionInfo.class)))
683        .thenReturn(RegionState.createForTesting(null, RegionState.State.OPEN));
684
685    for (Map.Entry<byte[], Integer> region : regionSizes.entrySet()) {
686      RegionMetrics regionLoad = Mockito.mock(RegionMetrics.class);
687      when(regionLoad.getRegionName()).thenReturn(region.getKey());
688      when(regionLoad.getStoreFileSize())
689        .thenReturn(new Size(region.getValue(), Size.Unit.MEGABYTE));
690
691      // this is possibly broken with jdk9, unclear if false positive or not
692      // suppress it for now, fix it when we get to running tests on 9
693      // see: http://errorprone.info/bugpattern/MockitoCast
694      when((Object) masterServices.getServerManager().getLoad(sn).getRegionMetrics()
695        .get(region.getKey())).thenReturn(regionLoad);
696    }
697
698    when(masterServices.isSplitOrMergeEnabled(any())).thenReturn(true);
699    when(tableDescriptor.getTableName()).thenReturn(name.getTableName());
700
701    normalizer = new SimpleRegionNormalizer();
702    normalizer.setConf(conf);
703    normalizer.setMasterServices(masterServices);
704  }
705
706  /**
707   * Create a list of {@link RegionInfo}s that represent a region chain of the specified length.
708   */
709  private static List<RegionInfo> createRegionInfos(final TableName tableName, final int length) {
710    if (length < 1) {
711      throw new IllegalStateException("length must be greater than or equal to 1.");
712    }
713
714    final byte[] startKey = Bytes.toBytes("aaaaa");
715    final byte[] endKey = Bytes.toBytes("zzzzz");
716    if (length == 1) {
717      return Collections.singletonList(createRegionInfo(tableName, startKey, endKey));
718    }
719
720    final byte[][] splitKeys = Bytes.split(startKey, endKey, length - 1);
721    final List<RegionInfo> ret = new ArrayList<>(length);
722    for (int i = 0; i < splitKeys.length - 1; i++) {
723      ret.add(createRegionInfo(tableName, splitKeys[i], splitKeys[i + 1]));
724    }
725    return ret;
726  }
727
728  private static RegionInfo createRegionInfo(final TableName tableName, final byte[] startKey,
729    final byte[] endKey) {
730    return RegionInfoBuilder.newBuilder(tableName).setStartKey(startKey).setEndKey(endKey)
731      .setRegionId(generateRegionId()).build();
732  }
733
734  private static long generateRegionId() {
735    return Instant.ofEpochMilli(EnvironmentEdgeManager.currentTime())
736      .minus(Period.ofDays(DEFAULT_MERGE_MIN_REGION_AGE_DAYS + 1)).toEpochMilli();
737  }
738
739  private static List<RegionInfo> createRegionInfos(final TableName tableName,
740    final byte[][] splitKeys) {
741    final List<RegionInfo> ret = new ArrayList<>(splitKeys.length);
742    for (int i = 0; i < splitKeys.length - 1; i++) {
743      ret.add(createRegionInfo(tableName, splitKeys[i], splitKeys[i + 1]));
744    }
745    return ret;
746  }
747
748  private static Map<byte[], Integer> createRegionSizesMap(final List<RegionInfo> regionInfos,
749    int... sizes) {
750    if (regionInfos.size() != sizes.length) {
751      throw new IllegalStateException("Parameter lengths must match.");
752    }
753
754    final Map<byte[], Integer> ret = new HashMap<>(regionInfos.size());
755    for (int i = 0; i < regionInfos.size(); i++) {
756      ret.put(regionInfos.get(i).getRegionName(), sizes[i]);
757    }
758    return ret;
759  }
760}