001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.regionserver;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertNull;
023import static org.junit.Assert.assertTrue;
024import static org.mockito.Mockito.doReturn;
025import static org.mockito.Mockito.mock;
026import static org.mockito.Mockito.when;
027
028import java.io.IOException;
029import java.util.ArrayList;
030import java.util.List;
031import java.util.Optional;
032import org.apache.hadoop.conf.Configuration;
033import org.apache.hadoop.hbase.HBaseClassTestRule;
034import org.apache.hadoop.hbase.HBaseConfiguration;
035import org.apache.hadoop.hbase.HConstants;
036import org.apache.hadoop.hbase.TableName;
037import org.apache.hadoop.hbase.client.RegionInfo;
038import org.apache.hadoop.hbase.client.RegionInfoBuilder;
039import org.apache.hadoop.hbase.client.TableDescriptor;
040import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
041import org.apache.hadoop.hbase.testclassification.RegionServerTests;
042import org.apache.hadoop.hbase.testclassification.SmallTests;
043import org.apache.hadoop.hbase.util.Bytes;
044import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
045import org.junit.Before;
046import org.junit.ClassRule;
047import org.junit.Test;
048import org.junit.experimental.categories.Category;
049
050@Category({ RegionServerTests.class, SmallTests.class })
051public class TestRegionSplitPolicy {
052
053  @ClassRule
054  public static final HBaseClassTestRule CLASS_RULE =
055    HBaseClassTestRule.forClass(TestRegionSplitPolicy.class);
056
057  private Configuration conf;
058  private HRegion mockRegion;
059  private List<HStore> stores;
060  private static final TableName TABLENAME = TableName.valueOf("t");
061
062  @Before
063  public void setupMocks() {
064    conf = HBaseConfiguration.create();
065    RegionInfo hri = RegionInfoBuilder.newBuilder(TABLENAME).build();
066    mockRegion = mock(HRegion.class);
067    doReturn(hri).when(mockRegion).getRegionInfo();
068    doReturn(true).when(mockRegion).isAvailable();
069    stores = new ArrayList<>();
070    doReturn(stores).when(mockRegion).getStores();
071  }
072
073  @Test
074  public void testForceSplitRegionWithReference() throws IOException {
075    TableDescriptor td = TableDescriptorBuilder.newBuilder(TABLENAME).setMaxFileSize(1024L).build();
076    doReturn(td).when(mockRegion).getTableDescriptor();
077    // Add a store above the requisite size. Should split.
078    HStore mockStore = mock(HStore.class);
079    doReturn(2000L).when(mockStore).getSize();
080    // Act as if there's a reference file or some other reason it can't split.
081    // This should prevent splitting even though it's big enough.
082    doReturn(false).when(mockStore).canSplit();
083    stores.add(mockStore);
084
085    conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
086      ConstantSizeRegionSplitPolicy.class.getName());
087    ConstantSizeRegionSplitPolicy policy =
088      (ConstantSizeRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
089    assertFalse(policy.shouldSplit());
090
091    conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
092      IncreasingToUpperBoundRegionSplitPolicy.class.getName());
093    policy = (IncreasingToUpperBoundRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
094    assertFalse(policy.shouldSplit());
095  }
096
097  @Test
098  public void testIncreasingToUpperBoundRegionSplitPolicy() throws IOException {
099    // Configure IncreasingToUpperBoundRegionSplitPolicy as our split policy
100    conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
101      IncreasingToUpperBoundRegionSplitPolicy.class.getName());
102    // Now make it so the mock region has a RegionServerService that will
103    // return 'online regions'.
104    RegionServerServices rss = mock(RegionServerServices.class);
105    final List<HRegion> regions = new ArrayList<>();
106    doReturn(regions).when(rss).getRegions(TABLENAME);
107    when(mockRegion.getRegionServerServices()).thenReturn(rss);
108    // Set max size for this 'table'.
109    long maxSplitSize = 1024L;
110    // Set flush size to 1/8. IncreasingToUpperBoundRegionSplitPolicy
111    // grows by the cube of the number of regions times flushsize each time.
112    long flushSize = maxSplitSize / 8;
113    conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, flushSize);
114    TableDescriptor td = TableDescriptorBuilder.newBuilder(TABLENAME).setMaxFileSize(maxSplitSize)
115      .setMemStoreFlushSize(flushSize).build();
116    doReturn(td).when(mockRegion).getTableDescriptor();
117    // If RegionServerService with no regions in it -- 'online regions' == 0 --
118    // then IncreasingToUpperBoundRegionSplitPolicy should act like a
119    // ConstantSizePolicy
120    IncreasingToUpperBoundRegionSplitPolicy policy =
121      (IncreasingToUpperBoundRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
122    doConstantSizePolicyTests(policy);
123
124    // Add a store in excess of split size. Because there are "no regions"
125    // on this server -- rss.getOnlineRegions is 0 -- then we should split
126    // like a constantsizeregionsplitpolicy would
127    HStore mockStore = mock(HStore.class);
128    doReturn(2000L).when(mockStore).getSize();
129    doReturn(true).when(mockStore).canSplit();
130    stores.add(mockStore);
131    // It should split
132    assertTrue(policy.shouldSplit());
133
134    // Now test that we increase our split size as online regions for a table
135    // grows. With one region, split size should be flushsize.
136    regions.add(mockRegion);
137    doReturn(flushSize).when(mockStore).getSize();
138    // Should not split since store is flush size.
139    assertFalse(policy.shouldSplit());
140    // Set size of store to be > 2*flush size and we should split
141    doReturn(flushSize * 2 + 1).when(mockStore).getSize();
142    assertTrue(policy.shouldSplit());
143    // Add another region to the 'online regions' on this server and we should
144    // now be no longer be splittable since split size has gone up.
145    regions.add(mockRegion);
146    assertFalse(policy.shouldSplit());
147    // make sure its just over; verify it'll split
148    doReturn((long) (maxSplitSize * 1.25 + 1)).when(mockStore).getSize();
149    assertTrue(policy.shouldSplit());
150
151    // Finally assert that even if loads of regions, we'll split at max size
152    assertWithinJitter(maxSplitSize, policy.getSizeToCheck(1000));
153    // Assert same is true if count of regions is zero.
154    assertWithinJitter(maxSplitSize, policy.getSizeToCheck(0));
155  }
156
157  @Test
158  public void testIsExceedSize() throws IOException {
159    // Configure SteppingAllStoresSizeSplitPolicy as our split policy
160    conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
161      ConstantSizeRegionSplitPolicy.class.getName());
162    conf.set(HConstants.OVERALL_HREGION_FILES, "true");
163    // Now make it so the mock region has a RegionServerService that will
164    // return 'online regions'.
165    RegionServerServices rss = mock(RegionServerServices.class);
166    final List<HRegion> regions = new ArrayList<>();
167    doReturn(regions).when(rss).getRegions(TABLENAME);
168    when(mockRegion.getRegionServerServices()).thenReturn(rss);
169
170    TableDescriptor td = TableDescriptorBuilder.newBuilder(TABLENAME).build();
171    doReturn(td).when(mockRegion).getTableDescriptor();
172    ConstantSizeRegionSplitPolicy policy =
173      (ConstantSizeRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
174    regions.add(mockRegion);
175
176    HStore mockStore1 = mock(HStore.class);
177    doReturn(100L).when(mockStore1).getSize();
178    HStore mockStore2 = mock(HStore.class);
179    doReturn(924L).when(mockStore2).getSize();
180    HStore mockStore3 = mock(HStore.class);
181    doReturn(925L).when(mockStore3).getSize();
182
183    // test sum of store's size not greater than sizeToCheck
184    stores.add(mockStore1);
185    stores.add(mockStore2);
186    assertFalse(policy.isExceedSize(1024));
187    stores.clear();
188
189    // test sum of store's size greater than sizeToCheck
190    stores.add(mockStore1);
191    stores.add(mockStore3);
192    assertTrue(policy.isExceedSize(1024));
193  }
194
195  @Test
196  public void testBusyRegionSplitPolicy() throws Exception {
197    doReturn(TableDescriptorBuilder.newBuilder(TABLENAME).build()).when(mockRegion)
198      .getTableDescriptor();
199    conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, BusyRegionSplitPolicy.class.getName());
200    conf.setLong("hbase.busy.policy.minAge", 1000000L);
201    conf.setFloat("hbase.busy.policy.blockedRequests", 0.1f);
202
203    RegionServerServices rss = mock(RegionServerServices.class);
204    final List<HRegion> regions = new ArrayList<>();
205    doReturn(regions).when(rss).getRegions(TABLENAME);
206    when(mockRegion.getRegionServerServices()).thenReturn(rss);
207    when(mockRegion.getBlockedRequestsCount()).thenReturn(0L);
208    when(mockRegion.getWriteRequestsCount()).thenReturn(0L);
209
210    BusyRegionSplitPolicy policy =
211      (BusyRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
212
213    when(mockRegion.getBlockedRequestsCount()).thenReturn(10L);
214    when(mockRegion.getWriteRequestsCount()).thenReturn(10L);
215    // Not enough time since region came online
216    assertFalse(policy.shouldSplit());
217
218    // Reset min age for split to zero
219    conf.setLong("hbase.busy.policy.minAge", 0L);
220    // Aggregate over 500 ms periods
221    conf.setLong("hbase.busy.policy.aggWindow", 500L);
222    policy = (BusyRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
223    long start = EnvironmentEdgeManager.currentTime();
224    when(mockRegion.getBlockedRequestsCount()).thenReturn(10L);
225    when(mockRegion.getWriteRequestsCount()).thenReturn(20L);
226    Thread.sleep(300);
227    assertFalse(policy.shouldSplit());
228    when(mockRegion.getBlockedRequestsCount()).thenReturn(12L);
229    when(mockRegion.getWriteRequestsCount()).thenReturn(30L);
230    Thread.sleep(2);
231    // Enough blocked requests since last time, but aggregate blocked request
232    // rate over last 500 ms is still low, because major portion of the window is constituted
233    // by the previous zero blocked request period which lasted at least 300 ms off last 500 ms.
234    if (EnvironmentEdgeManager.currentTime() - start < 500) {
235      assertFalse(policy.shouldSplit());
236    }
237    when(mockRegion.getBlockedRequestsCount()).thenReturn(14L);
238    when(mockRegion.getWriteRequestsCount()).thenReturn(40L);
239    Thread.sleep(200);
240    assertTrue(policy.shouldSplit());
241  }
242
243  private void assertWithinJitter(long maxSplitSize, long sizeToCheck) {
244    assertTrue("Size greater than lower bound of jitter",
245      (long) (maxSplitSize * 0.75) <= sizeToCheck);
246    assertTrue("Size less than upper bound of jitter", (long) (maxSplitSize * 1.25) >= sizeToCheck);
247  }
248
249  @Test
250  public void testCreateDefault() throws IOException {
251    conf.setLong(HConstants.HREGION_MAX_FILESIZE, 1234L);
252    TableDescriptor td = TableDescriptorBuilder.newBuilder(TABLENAME).build();
253    doReturn(td).when(mockRegion).getTableDescriptor();
254    // Using a default HTD, should pick up the file size from
255    // configuration.
256    ConstantSizeRegionSplitPolicy policy =
257      (ConstantSizeRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
258    assertWithinJitter(1234L, policy.getDesiredMaxFileSize());
259
260    // If specified in HTD, should use that
261    td = TableDescriptorBuilder.newBuilder(TABLENAME).setMaxFileSize(9999L).build();
262    doReturn(td).when(mockRegion).getTableDescriptor();
263    policy = (ConstantSizeRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
264    assertWithinJitter(9999L, policy.getDesiredMaxFileSize());
265  }
266
267  /**
268   * Test setting up a customized split policy
269   */
270  @Test
271  public void testCustomPolicy() throws IOException {
272    TableDescriptor td = TableDescriptorBuilder.newBuilder(TABLENAME)
273      .setRegionSplitPolicyClassName(KeyPrefixRegionSplitPolicy.class.getName())
274      .setValue(KeyPrefixRegionSplitPolicy.PREFIX_LENGTH_KEY, "2").build();
275
276    doReturn(td).when(mockRegion).getTableDescriptor();
277
278    HStore mockStore = mock(HStore.class);
279    doReturn(2000L).when(mockStore).getSize();
280    doReturn(true).when(mockStore).canSplit();
281    doReturn(Optional.of(Bytes.toBytes("abcd"))).when(mockStore).getSplitPoint();
282    stores.add(mockStore);
283
284    KeyPrefixRegionSplitPolicy policy =
285      (KeyPrefixRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
286
287    assertEquals("ab", Bytes.toString(policy.getSplitPoint()));
288  }
289
290  @Test
291  public void testConstantSizePolicy() throws IOException {
292    TableDescriptor td = TableDescriptorBuilder.newBuilder(TABLENAME).setMaxFileSize(1024L).build();
293    doReturn(td).when(mockRegion).getTableDescriptor();
294    ConstantSizeRegionSplitPolicy policy =
295      (ConstantSizeRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
296    doConstantSizePolicyTests(policy);
297  }
298
299  /**
300   * Run through tests for a ConstantSizeRegionSplitPolicy
301   */
302  private void doConstantSizePolicyTests(final ConstantSizeRegionSplitPolicy policy) {
303    // For no stores, should not split
304    assertFalse(policy.shouldSplit());
305
306    // Add a store above the requisite size. Should split.
307    HStore mockStore = mock(HStore.class);
308    doReturn(2000L).when(mockStore).getSize();
309    doReturn(true).when(mockStore).canSplit();
310    stores.add(mockStore);
311
312    assertTrue(policy.shouldSplit());
313
314    // Act as if there's a reference file or some other reason it can't split.
315    // This should prevent splitting even though it's big enough.
316    doReturn(false).when(mockStore).canSplit();
317    assertFalse(policy.shouldSplit());
318
319    // Reset splittability after above
320    doReturn(true).when(mockStore).canSplit();
321
322    // Set to a small size, should not split
323    doReturn(100L).when(mockStore).getSize();
324    assertFalse(policy.shouldSplit());
325
326    // Clear families we added above
327    stores.clear();
328  }
329
330  @Test
331  public void testGetSplitPoint() throws IOException {
332    TableDescriptor td = TableDescriptorBuilder.newBuilder(TABLENAME).build();
333    doReturn(td).when(mockRegion).getTableDescriptor();
334
335    ConstantSizeRegionSplitPolicy policy =
336      (ConstantSizeRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
337
338    // For no stores, should not split
339    assertFalse(policy.shouldSplit());
340    assertNull(policy.getSplitPoint());
341
342    // Add a store above the requisite size. Should split.
343    HStore mockStore = mock(HStore.class);
344    doReturn(2000L).when(mockStore).getSize();
345    doReturn(true).when(mockStore).canSplit();
346    doReturn(Optional.of(Bytes.toBytes("store 1 split"))).when(mockStore).getSplitPoint();
347    stores.add(mockStore);
348
349    assertEquals("store 1 split", Bytes.toString(policy.getSplitPoint()));
350
351    // Add a bigger store. The split point should come from that one
352    HStore mockStore2 = mock(HStore.class);
353    doReturn(4000L).when(mockStore2).getSize();
354    doReturn(true).when(mockStore2).canSplit();
355    doReturn(Optional.of(Bytes.toBytes("store 2 split"))).when(mockStore2).getSplitPoint();
356    stores.add(mockStore2);
357
358    assertEquals("store 2 split", Bytes.toString(policy.getSplitPoint()));
359  }
360
361  @Test
362  public void testDelimitedKeyPrefixRegionSplitPolicy() throws IOException {
363    TableDescriptor td = TableDescriptorBuilder.newBuilder(TABLENAME)
364      .setRegionSplitPolicyClassName(DelimitedKeyPrefixRegionSplitPolicy.class.getName())
365      .setValue(DelimitedKeyPrefixRegionSplitPolicy.DELIMITER_KEY, ",").build();
366
367    doReturn(td).when(mockRegion).getTableDescriptor();
368    doReturn(stores).when(mockRegion).getStores();
369
370    HStore mockStore = mock(HStore.class);
371    doReturn(2000L).when(mockStore).getSize();
372    doReturn(true).when(mockStore).canSplit();
373    doReturn(Optional.of(Bytes.toBytes("ab,cd"))).when(mockStore).getSplitPoint();
374    stores.add(mockStore);
375
376    DelimitedKeyPrefixRegionSplitPolicy policy =
377      (DelimitedKeyPrefixRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
378
379    assertEquals("ab", Bytes.toString(policy.getSplitPoint()));
380
381    doReturn(Optional.of(Bytes.toBytes("ijk"))).when(mockStore).getSplitPoint();
382    assertEquals("ijk", Bytes.toString(policy.getSplitPoint()));
383  }
384
385  @Test
386  public void testConstantSizePolicyWithJitter() throws IOException {
387    conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
388      ConstantSizeRegionSplitPolicy.class.getName());
389    TableDescriptor td =
390      TableDescriptorBuilder.newBuilder(TABLENAME).setMaxFileSize(Long.MAX_VALUE).build();
391    doReturn(td).when(mockRegion).getTableDescriptor();
392    boolean positiveJitter = false;
393    ConstantSizeRegionSplitPolicy policy = null;
394    while (!positiveJitter) {
395      policy = (ConstantSizeRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
396      positiveJitter = policy.positiveJitterRate();
397    }
398    // add a store
399    HStore mockStore = mock(HStore.class);
400    doReturn(2000L).when(mockStore).getSize();
401    doReturn(true).when(mockStore).canSplit();
402    stores.add(mockStore);
403    // Jitter shouldn't cause overflow when HTableDescriptor.MAX_FILESIZE set to Long.MAX_VALUE
404    assertFalse(policy.shouldSplit());
405  }
406}