Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.apache.hadoop.hbase.TableName.META_TABLE_NAME;
021import static org.hamcrest.CoreMatchers.instanceOf;
022import static org.hamcrest.MatcherAssert.assertThat;
023import static org.junit.Assert.assertEquals;
024import static org.junit.Assert.assertFalse;
025import static org.junit.Assert.assertNotEquals;
026import static org.junit.Assert.assertNotNull;
027import static org.junit.Assert.assertTrue;
028import static org.junit.Assert.fail;
029
030import java.io.IOException;
031import java.util.ArrayList;
032import java.util.Collections;
033import java.util.List;
034import java.util.concurrent.ExecutionException;
035import java.util.stream.Collectors;
036import org.apache.hadoop.hbase.AsyncMetaTableAccessor;
037import org.apache.hadoop.hbase.HBaseClassTestRule;
038import org.apache.hadoop.hbase.HConstants;
039import org.apache.hadoop.hbase.HRegionLocation;
040import org.apache.hadoop.hbase.TableName;
041import org.apache.hadoop.hbase.master.HMaster;
042import org.apache.hadoop.hbase.master.assignment.AssignmentTestingUtil;
043import org.apache.hadoop.hbase.master.janitor.CatalogJanitor;
044import org.apache.hadoop.hbase.testclassification.ClientTests;
045import org.apache.hadoop.hbase.testclassification.LargeTests;
046import org.apache.hadoop.hbase.util.Bytes;
047import org.apache.hadoop.hbase.util.Threads;
048import org.junit.ClassRule;
049import org.junit.Ignore;
050import org.junit.Test;
051import org.junit.experimental.categories.Category;
052import org.junit.runner.RunWith;
053import org.junit.runners.Parameterized;
054
055/**
056 * Class to test asynchronous region admin operations.
057 * @see TestAsyncRegionAdminApi This test and it used to be joined it was taking longer than our ten
058 *      minute timeout so they were split.
059 */
060@RunWith(Parameterized.class)
061@Category({ LargeTests.class, ClientTests.class })
062public class TestAsyncRegionAdminApi2 extends TestAsyncAdminBase {
063
064  @ClassRule
065  public static final HBaseClassTestRule CLASS_RULE =
066    HBaseClassTestRule.forClass(TestAsyncRegionAdminApi2.class);
067
068  @Test
069  public void testGetRegionLocation() throws Exception {
070    RawAsyncHBaseAdmin rawAdmin = (RawAsyncHBaseAdmin) ASYNC_CONN.getAdmin();
071    TEST_UTIL.createMultiRegionTable(tableName, HConstants.CATALOG_FAMILY);
072    AsyncTableRegionLocator locator = ASYNC_CONN.getRegionLocator(tableName);
073    HRegionLocation regionLocation = locator.getRegionLocation(Bytes.toBytes("mmm")).get();
074    RegionInfo region = regionLocation.getRegion();
075    byte[] regionName = regionLocation.getRegion().getRegionName();
076    HRegionLocation location = rawAdmin.getRegionLocation(regionName).get();
077    assertTrue(Bytes.equals(regionName, location.getRegion().getRegionName()));
078    location = rawAdmin.getRegionLocation(region.getEncodedNameAsBytes()).get();
079    assertTrue(Bytes.equals(regionName, location.getRegion().getRegionName()));
080  }
081
082  @Test
083  public void testSplitSwitch() throws Exception {
084    createTableWithDefaultConf(tableName);
085    byte[][] families = { FAMILY };
086    final int rows = 10000;
087    TestAsyncRegionAdminApi.loadData(tableName, families, rows);
088
089    AsyncTable<AdvancedScanResultConsumer> metaTable = ASYNC_CONN.getTable(META_TABLE_NAME);
090    List<HRegionLocation> regionLocations =
091      AsyncMetaTableAccessor.getTableHRegionLocations(metaTable, tableName).get();
092    int originalCount = regionLocations.size();
093
094    initSplitMergeSwitch();
095    assertTrue(admin.splitSwitch(false).get());
096    try {
097      admin.split(tableName, Bytes.toBytes(rows / 2)).join();
098    } catch (Exception e) {
099      // Expected
100    }
101    int count = admin.getRegions(tableName).get().size();
102    assertTrue(originalCount == count);
103
104    assertFalse(admin.splitSwitch(true).get());
105    admin.split(tableName).join();
106    while ((count = admin.getRegions(tableName).get().size()) == originalCount) {
107      Threads.sleep(100);
108    }
109    assertTrue(originalCount < count);
110  }
111
112  @Test
113  @Ignore
114  // It was ignored in TestSplitOrMergeStatus, too
115  public void testMergeSwitch() throws Exception {
116    createTableWithDefaultConf(tableName);
117    byte[][] families = { FAMILY };
118    TestAsyncRegionAdminApi.loadData(tableName, families, 1000);
119
120    AsyncTable<AdvancedScanResultConsumer> metaTable = ASYNC_CONN.getTable(META_TABLE_NAME);
121    List<HRegionLocation> regionLocations =
122      AsyncMetaTableAccessor.getTableHRegionLocations(metaTable, tableName).get();
123    int originalCount = regionLocations.size();
124
125    initSplitMergeSwitch();
126    admin.split(tableName).join();
127    int postSplitCount = originalCount;
128    while ((postSplitCount = admin.getRegions(tableName).get().size()) == originalCount) {
129      Threads.sleep(100);
130    }
131    assertTrue("originalCount=" + originalCount + ", postSplitCount=" + postSplitCount,
132      originalCount != postSplitCount);
133
134    // Merge switch is off so merge should NOT succeed.
135    assertTrue(admin.mergeSwitch(false).get());
136    List<RegionInfo> regions = admin.getRegions(tableName).get();
137    assertTrue(regions.size() > 1);
138    admin.mergeRegions(regions.get(0).getRegionName(), regions.get(1).getRegionName(), true).join();
139    int count = admin.getRegions(tableName).get().size();
140    assertTrue("postSplitCount=" + postSplitCount + ", count=" + count, postSplitCount == count);
141
142    // Merge switch is on so merge should succeed.
143    assertFalse(admin.mergeSwitch(true).get());
144    admin.mergeRegions(regions.get(0).getRegionName(), regions.get(1).getRegionName(), true).join();
145    count = admin.getRegions(tableName).get().size();
146    assertTrue((postSplitCount / 2) == count);
147  }
148
149  private void initSplitMergeSwitch() throws Exception {
150    if (!admin.isSplitEnabled().get()) {
151      admin.splitSwitch(true).get();
152    }
153    if (!admin.isMergeEnabled().get()) {
154      admin.mergeSwitch(true).get();
155    }
156    assertTrue(admin.isSplitEnabled().get());
157    assertTrue(admin.isMergeEnabled().get());
158  }
159
160  @Test
161  public void testMergeRegions() throws Exception {
162    byte[][] splitRows = new byte[][] { Bytes.toBytes("3"), Bytes.toBytes("6") };
163    createTableWithDefaultConf(tableName, splitRows);
164
165    AsyncTable<AdvancedScanResultConsumer> metaTable = ASYNC_CONN.getTable(META_TABLE_NAME);
166    List<HRegionLocation> regionLocations =
167      AsyncMetaTableAccessor.getTableHRegionLocations(metaTable, tableName).get();
168    RegionInfo regionA;
169    RegionInfo regionB;
170    RegionInfo regionC;
171    RegionInfo mergedChildRegion = null;
172
173    // merge with full name
174    assertEquals(3, regionLocations.size());
175    regionA = regionLocations.get(0).getRegion();
176    regionB = regionLocations.get(1).getRegion();
177    regionC = regionLocations.get(2).getRegion();
178    admin.mergeRegions(regionA.getRegionName(), regionB.getRegionName(), false).get();
179
180    regionLocations = AsyncMetaTableAccessor.getTableHRegionLocations(metaTable, tableName).get();
181
182    assertEquals(2, regionLocations.size());
183    for (HRegionLocation rl : regionLocations) {
184      if (regionC.compareTo(rl.getRegion()) != 0) {
185        mergedChildRegion = rl.getRegion();
186        break;
187      }
188    }
189
190    assertNotNull(mergedChildRegion);
191    // Need to wait GC for merged child region is done.
192    HMaster services = TEST_UTIL.getHBaseCluster().getMaster();
193    CatalogJanitor cj = services.getCatalogJanitor();
194    assertTrue(cj.scan() > 0);
195    // Wait until all procedures settled down
196    while (!services.getMasterProcedureExecutor().getActiveProcIds().isEmpty()) {
197      Thread.sleep(200);
198    }
199    // merge with encoded name
200    admin.mergeRegions(regionC.getRegionName(), mergedChildRegion.getRegionName(), false).get();
201
202    regionLocations = AsyncMetaTableAccessor.getTableHRegionLocations(metaTable, tableName).get();
203    assertEquals(1, regionLocations.size());
204  }
205
206  @Test
207  public void testMergeRegionsInvalidRegionCount() throws Exception {
208    byte[][] splitRows = new byte[][] { Bytes.toBytes("3"), Bytes.toBytes("6") };
209    createTableWithDefaultConf(tableName, splitRows);
210    List<RegionInfo> regions = admin.getRegions(tableName).join();
211    // 0
212    try {
213      admin.mergeRegions(Collections.emptyList(), false).get();
214      fail();
215    } catch (ExecutionException e) {
216      // expected
217      assertThat(e.getCause(), instanceOf(IllegalArgumentException.class));
218    }
219    // 1
220    try {
221      admin.mergeRegions(regions.stream().limit(1).map(RegionInfo::getEncodedNameAsBytes)
222        .collect(Collectors.toList()), false).get();
223      fail();
224    } catch (ExecutionException e) {
225      // expected
226      assertThat(e.getCause(), instanceOf(IllegalArgumentException.class));
227    }
228  }
229
230  @Test
231  public void testSplitTable() throws Exception {
232    initSplitMergeSwitch();
233    splitTest(TableName.valueOf("testSplitTable"), 3000, false, null);
234    splitTest(TableName.valueOf("testSplitTableWithSplitPoint"), 3000, false, Bytes.toBytes("3"));
235    splitTest(TableName.valueOf("testSplitTableRegion"), 3000, true, null);
236    splitTest(TableName.valueOf("testSplitTableRegionWithSplitPoint2"), 3000, true,
237      Bytes.toBytes("3"));
238  }
239
240  private void splitTest(TableName tableName, int rowCount, boolean isSplitRegion,
241    byte[] splitPoint) throws Exception {
242    // create table
243    createTableWithDefaultConf(tableName);
244
245    AsyncTable<AdvancedScanResultConsumer> metaTable = ASYNC_CONN.getTable(META_TABLE_NAME);
246    List<HRegionLocation> regionLocations =
247      AsyncMetaTableAccessor.getTableHRegionLocations(metaTable, tableName).get();
248    assertEquals(1, regionLocations.size());
249
250    AsyncTable<?> table = ASYNC_CONN.getTable(tableName);
251    List<Put> puts = new ArrayList<>();
252    for (int i = 0; i < rowCount; i++) {
253      Put put = new Put(Bytes.toBytes(i));
254      put.addColumn(FAMILY, null, Bytes.toBytes("value" + i));
255      puts.add(put);
256    }
257    table.putAll(puts).join();
258
259    if (isSplitRegion) {
260      if (splitPoint == null) {
261        admin.splitRegion(regionLocations.get(0).getRegion().getRegionName()).get();
262      } else {
263        admin.splitRegion(regionLocations.get(0).getRegion().getRegionName(), splitPoint).get();
264      }
265    } else {
266      if (splitPoint == null) {
267        admin.split(tableName).get();
268      } else {
269        admin.split(tableName, splitPoint).get();
270      }
271    }
272
273    int count = 0;
274    for (int i = 0; i < 45; i++) {
275      try {
276        regionLocations =
277          AsyncMetaTableAccessor.getTableHRegionLocations(metaTable, tableName).get();
278        count = regionLocations.size();
279        if (count >= 2) {
280          break;
281        }
282        Thread.sleep(1000L);
283      } catch (Exception e) {
284        LOG.error(e.toString(), e);
285      }
286    }
287    assertEquals(2, count);
288  }
289
290  @Test
291  public void testTruncateRegion() throws Exception {
292    // Arrange - Create table, insert data, identify region to truncate.
293    final byte[][] splitKeys =
294      new byte[][] { Bytes.toBytes("30"), Bytes.toBytes("60"), Bytes.toBytes("90") };
295    String family1 = "f1";
296    String family2 = "f2";
297
298    final String[] sFamilies = new String[] { family1, family2 };
299    final byte[][] bFamilies = new byte[][] { Bytes.toBytes(family1), Bytes.toBytes(family2) };
300    createTableWithDefaultConf(tableName, splitKeys, bFamilies);
301
302    AsyncTable<AdvancedScanResultConsumer> metaTable = ASYNC_CONN.getTable(META_TABLE_NAME);
303    List<HRegionLocation> regionLocations =
304      AsyncMetaTableAccessor.getTableHRegionLocations(metaTable, tableName).get();
305    RegionInfo regionToBeTruncated = regionLocations.get(0).getRegion();
306
307    assertEquals(4, regionLocations.size());
308
309    AssignmentTestingUtil.insertData(TEST_UTIL, tableName, 2, 21, sFamilies);
310    AssignmentTestingUtil.insertData(TEST_UTIL, tableName, 2, 31, sFamilies);
311    AssignmentTestingUtil.insertData(TEST_UTIL, tableName, 2, 61, sFamilies);
312    AssignmentTestingUtil.insertData(TEST_UTIL, tableName, 2, 91, sFamilies);
313    int rowCountBeforeTruncate = TEST_UTIL.countRows(tableName);
314
315    // Act - Truncate the first region
316    admin.truncateRegion(regionToBeTruncated.getRegionName()).get();
317
318    // Assert
319    int rowCountAfterTruncate = TEST_UTIL.countRows(tableName);
320    assertNotEquals(rowCountBeforeTruncate, rowCountAfterTruncate);
321    int expectedRowCount = rowCountBeforeTruncate - 2;// Since region with 2 rows was truncated.
322    assertEquals(expectedRowCount, rowCountAfterTruncate);
323  }
324
325  @Test
326  public void testTruncateReplicaRegionNotAllowed() throws Exception {
327    // Arrange - Create table, insert data, identify region to truncate.
328    final byte[][] splitKeys =
329      new byte[][] { Bytes.toBytes("30"), Bytes.toBytes("60"), Bytes.toBytes("90") };
330    String family1 = "f1";
331    String family2 = "f2";
332
333    final byte[][] bFamilies = new byte[][] { Bytes.toBytes(family1), Bytes.toBytes(family2) };
334    createTableWithDefaultConf(tableName, 2, splitKeys, bFamilies);
335
336    AsyncTable<AdvancedScanResultConsumer> metaTable = ASYNC_CONN.getTable(META_TABLE_NAME);
337    List<HRegionLocation> regionLocations =
338      AsyncMetaTableAccessor.getTableHRegionLocations(metaTable, tableName).get();
339    RegionInfo primaryRegion = regionLocations.get(0).getRegion();
340
341    RegionInfo firstReplica = RegionReplicaUtil.getRegionInfoForReplica(primaryRegion, 1);
342
343    // Act - Truncate the first region
344    try {
345      admin.truncateRegion(firstReplica.getRegionName()).get();
346    } catch (Exception e) {
347      // Assert
348      assertEquals("Expected message is different",
349        "Can't truncate replicas directly.Replicas are auto-truncated "
350          + "when their primary is truncated.",
351        e.getCause().getMessage());
352    }
353  }
354
355  @Test
356  public void testTruncateRegionsMetaTableRegionsNotAllowed() throws Exception {
357    AsyncTableRegionLocator locator = ASYNC_CONN.getRegionLocator(META_TABLE_NAME);
358    List<HRegionLocation> regionLocations = locator.getAllRegionLocations().get();
359    HRegionLocation regionToBeTruncated = regionLocations.get(0);
360    try {
361      admin.truncateRegion(regionToBeTruncated.getRegion().getRegionName()).get();
362      fail();
363    } catch (ExecutionException e) {
364      // expected
365      assertThat(e.getCause(), instanceOf(IOException.class));
366      assertEquals("Can't truncate region in catalog tables", e.getCause().getMessage());
367    }
368  }
369}