Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotEquals;
022import static org.junit.Assert.assertNotNull;
023import static org.junit.Assert.assertTrue;
024
025import java.io.IOException;
026import java.nio.charset.StandardCharsets;
027import java.util.ArrayList;
028import java.util.Collection;
029import java.util.EnumSet;
030import java.util.HashMap;
031import java.util.HashSet;
032import java.util.List;
033import java.util.Map;
034import java.util.Set;
035import java.util.concurrent.atomic.AtomicInteger;
036import org.apache.hadoop.conf.Configuration;
037import org.apache.hadoop.hbase.ClusterMetrics.Option;
038import org.apache.hadoop.hbase.HBaseClassTestRule;
039import org.apache.hadoop.hbase.HBaseTestingUtility;
040import org.apache.hadoop.hbase.HConstants;
041import org.apache.hadoop.hbase.HRegionLocation;
042import org.apache.hadoop.hbase.MetaTableAccessor;
043import org.apache.hadoop.hbase.MetaTableAccessor.Visitor;
044import org.apache.hadoop.hbase.RegionLocations;
045import org.apache.hadoop.hbase.ServerName;
046import org.apache.hadoop.hbase.StartMiniClusterOption;
047import org.apache.hadoop.hbase.TableName;
048import org.apache.hadoop.hbase.client.Admin;
049import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
050import org.apache.hadoop.hbase.client.Connection;
051import org.apache.hadoop.hbase.client.ConnectionFactory;
052import org.apache.hadoop.hbase.client.Delete;
053import org.apache.hadoop.hbase.client.RegionInfo;
054import org.apache.hadoop.hbase.client.RegionReplicaUtil;
055import org.apache.hadoop.hbase.client.Result;
056import org.apache.hadoop.hbase.client.Table;
057import org.apache.hadoop.hbase.client.TableDescriptor;
058import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
059import org.apache.hadoop.hbase.testclassification.MasterTests;
060import org.apache.hadoop.hbase.testclassification.MediumTests;
061import org.apache.hadoop.hbase.util.Bytes;
062import org.apache.hadoop.hbase.util.JVMClusterUtil;
063import org.junit.AfterClass;
064import org.junit.BeforeClass;
065import org.junit.ClassRule;
066import org.junit.Rule;
067import org.junit.Test;
068import org.junit.experimental.categories.Category;
069import org.junit.rules.TestName;
070import org.slf4j.Logger;
071import org.slf4j.LoggerFactory;
072
073import org.apache.hbase.thirdparty.com.google.common.io.Closeables;
074
075@Category({ MasterTests.class, MediumTests.class })
076public class TestMasterOperationsForRegionReplicas {
077
078  @ClassRule
079  public static final HBaseClassTestRule CLASS_RULE =
080    HBaseClassTestRule.forClass(TestMasterOperationsForRegionReplicas.class);
081
082  private static final Logger LOG = LoggerFactory.getLogger(TestRegionPlacement.class);
083  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
084  private static Connection CONNECTION = null;
085  private static Admin ADMIN;
086  private static int numSlaves = 2;
087  private final static StartMiniClusterOption option = StartMiniClusterOption.builder()
088    .numRegionServers(numSlaves).numMasters(1).numAlwaysStandByMasters(1).build();
089  private static Configuration conf;
090
091  @Rule
092  public TestName name = new TestName();
093
094  @BeforeClass
095  public static void setupBeforeClass() throws Exception {
096    conf = TEST_UTIL.getConfiguration();
097    conf.setBoolean("hbase.tests.use.shortcircuit.reads", false);
098    TEST_UTIL.startMiniCluster(option);
099    TEST_UTIL.getAdmin().balancerSwitch(false, true);
100    resetConnections();
101    while (
102      ADMIN.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics().size()
103          < numSlaves
104    ) {
105      Thread.sleep(100);
106    }
107  }
108
109  private static void resetConnections() throws IOException {
110    Closeables.close(ADMIN, true);
111    Closeables.close(CONNECTION, true);
112    CONNECTION = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
113    ADMIN = CONNECTION.getAdmin();
114  }
115
116  @AfterClass
117  public static void tearDownAfterClass() throws Exception {
118    Closeables.close(ADMIN, true);
119    Closeables.close(CONNECTION, true);
120    TEST_UTIL.shutdownMiniCluster();
121  }
122
123  @Test
124  public void testCreateTableWithSingleReplica() throws Exception {
125    final int numRegions = 3;
126    final int numReplica = 1;
127    final TableName tableName = TableName.valueOf(name.getMethodName());
128    try {
129      TableDescriptor desc =
130        TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(numReplica)
131          .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build();
132      ADMIN.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), numRegions);
133      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
134      TEST_UTIL.waitUntilNoRegionsInTransition();
135
136      validateNumberOfRowsInMeta(tableName, numRegions, ADMIN.getConnection());
137      List<RegionInfo> hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
138      assertEquals(numRegions * numReplica, hris.size());
139    } finally {
140      ADMIN.disableTable(tableName);
141      ADMIN.deleteTable(tableName);
142    }
143  }
144
145  @Test
146  public void testCreateTableWithMultipleReplicas() throws Exception {
147    final TableName tableName = TableName.valueOf(name.getMethodName());
148    final int numRegions = 3;
149    final int numReplica = 2;
150    try {
151      TableDescriptor desc =
152        TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(numReplica)
153          .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build();
154      ADMIN.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), numRegions);
155      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
156      TEST_UTIL.waitUntilNoRegionsInTransition();
157      validateNumberOfRowsInMeta(tableName, numRegions, ADMIN.getConnection());
158
159      List<RegionInfo> hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
160      assertEquals(numRegions * numReplica, hris.size());
161      assertRegionStateNotNull(hris, numRegions, numReplica);
162
163      List<Result> metaRows = MetaTableAccessor.fullScanRegions(ADMIN.getConnection());
164      int numRows = 0;
165      for (Result result : metaRows) {
166        RegionLocations locations = MetaTableAccessor.getRegionLocations(result);
167        RegionInfo hri = locations.getRegionLocation().getRegion();
168        if (!hri.getTable().equals(tableName)) continue;
169        numRows += 1;
170        HRegionLocation[] servers = locations.getRegionLocations();
171        // have two locations for the replicas of a region, and the locations should be different
172        assertEquals(2, servers.length);
173        assertNotEquals(servers[1], servers[0]);
174      }
175      assertEquals(numRegions, numRows);
176
177      // The same verification of the meta as above but with the SnapshotOfRegionAssignmentFromMeta
178      // class
179      validateFromSnapshotFromMeta(TEST_UTIL, tableName, numRegions, numReplica,
180        ADMIN.getConnection());
181
182      // Now kill the master, restart it and see if the assignments are kept
183      ServerName master = TEST_UTIL.getHBaseClusterInterface().getClusterMetrics().getMasterName();
184      TEST_UTIL.getHBaseClusterInterface().stopMaster(master);
185      TEST_UTIL.getHBaseClusterInterface().waitForMasterToStop(master, 30000);
186      TEST_UTIL.getHBaseClusterInterface().startMaster(master.getHostname(), master.getPort());
187      TEST_UTIL.getHBaseClusterInterface().waitForActiveAndReadyMaster();
188      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
189      TEST_UTIL.waitUntilNoRegionsInTransition();
190      assertRegionStateNotNull(hris, numRegions, numReplica);
191      validateFromSnapshotFromMeta(TEST_UTIL, tableName, numRegions, numReplica,
192        ADMIN.getConnection());
193
194      // Now shut the whole cluster down, and verify the assignments are kept so that the
195      // availability constraints are met. MiniHBaseCluster chooses arbitrary ports on each
196      // restart. This messes with our being able to test that we retain locality. Therefore,
197      // figure current cluster ports and pass them in on next cluster start so new cluster comes
198      // up at same coordinates -- and the assignment retention logic has a chance to cut in.
199      List<Integer> rsports = new ArrayList<>();
200      for (JVMClusterUtil.RegionServerThread rst : TEST_UTIL.getHBaseCluster()
201        .getLiveRegionServerThreads()) {
202        rsports.add(rst.getRegionServer().getRpcServer().getListenerAddress().getPort());
203      }
204      TEST_UTIL.shutdownMiniHBaseCluster();
205      StartMiniClusterOption option =
206        StartMiniClusterOption.builder().numRegionServers(numSlaves).rsPorts(rsports).build();
207      TEST_UTIL.startMiniHBaseCluster(option);
208      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
209      TEST_UTIL.waitUntilNoRegionsInTransition();
210      resetConnections();
211      validateFromSnapshotFromMeta(TEST_UTIL, tableName, numRegions, numReplica,
212        ADMIN.getConnection());
213
214      // Now shut the whole cluster down, and verify regions are assigned even if there is only
215      // one server running
216      TEST_UTIL.shutdownMiniHBaseCluster();
217      TEST_UTIL.startMiniHBaseCluster();
218      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
219      TEST_UTIL.waitUntilNoRegionsInTransition();
220      resetConnections();
221      validateSingleRegionServerAssignment(ADMIN.getConnection(), numRegions, numReplica);
222      for (int i = 1; i < numSlaves; i++) { // restore the cluster
223        TEST_UTIL.getMiniHBaseCluster().startRegionServer();
224      }
225
226      // Check on alter table
227      ADMIN.disableTable(tableName);
228      assertTrue(ADMIN.isTableDisabled(tableName));
229      // increase the replica
230      ADMIN.modifyTable(
231        TableDescriptorBuilder.newBuilder(desc).setRegionReplication(numReplica + 1).build());
232      ADMIN.enableTable(tableName);
233      LOG.info(ADMIN.getDescriptor(tableName).toString());
234      assertTrue(ADMIN.isTableEnabled(tableName));
235      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
236      TEST_UTIL.waitUntilNoRegionsInTransition();
237      List<RegionInfo> regions = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager()
238        .getRegionStates().getRegionsOfTable(tableName);
239      assertTrue("regions.size=" + regions.size() + ", numRegions=" + numRegions + ", numReplica="
240        + numReplica, regions.size() == numRegions * (numReplica + 1));
241
242      // decrease the replica(earlier, table was modified to have a replica count of numReplica + 1)
243      ADMIN.disableTable(tableName);
244      ADMIN.modifyTable(
245        TableDescriptorBuilder.newBuilder(desc).setRegionReplication(numReplica).build());
246      ADMIN.enableTable(tableName);
247      assertTrue(ADMIN.isTableEnabled(tableName));
248      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
249      TEST_UTIL.waitUntilNoRegionsInTransition();
250      regions = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates()
251        .getRegionsOfTable(tableName);
252      assertEquals(numRegions * numReplica, regions.size());
253      // also make sure the meta table has the replica locations removed
254      hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
255      assertEquals(numRegions * numReplica, hris.size());
256      // just check that the number of default replica regions in the meta table are the same
257      // as the number of regions the table was created with, and the count of the
258      // replicas is numReplica for each region
259      Map<RegionInfo, Integer> defaultReplicas = new HashMap<>();
260      for (RegionInfo hri : hris) {
261        RegionInfo regionReplica0 = RegionReplicaUtil.getRegionInfoForDefaultReplica(hri);
262        Integer i = defaultReplicas.get(regionReplica0);
263        defaultReplicas.put(regionReplica0, i == null ? 1 : i + 1);
264      }
265      assertEquals(numRegions, defaultReplicas.size());
266      Collection<Integer> counts = new HashSet<>(defaultReplicas.values());
267      assertEquals(1, counts.size());
268      assertTrue(counts.contains(numReplica));
269    } finally {
270      ADMIN.disableTable(tableName);
271      ADMIN.deleteTable(tableName);
272    }
273  }
274
275  private void assertRegionStateNotNull(List<RegionInfo> hris, int numRegions, int numReplica) {
276    // check that the master created expected number of RegionState objects
277    for (int i = 0; i < numRegions; i++) {
278      for (int j = 0; j < numReplica; j++) {
279        RegionInfo replica = RegionReplicaUtil.getRegionInfoForReplica(hris.get(i), j);
280        RegionState state = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
281          .getRegionStates().getRegionState(replica);
282        assertNotNull(state);
283      }
284    }
285  }
286
287  @Test
288  public void testIncompleteMetaTableReplicaInformation() throws Exception {
289    final TableName tableName = TableName.valueOf(name.getMethodName());
290    final int numRegions = 3;
291    final int numReplica = 2;
292    try {
293      // Create a table and let the meta table be updated with the location of the
294      // region locations.
295      TableDescriptor desc =
296        TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(numReplica)
297          .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build();
298      ADMIN.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), numRegions);
299      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
300      TEST_UTIL.waitUntilNoRegionsInTransition();
301      Set<byte[]> tableRows = new HashSet<>();
302      List<RegionInfo> hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
303      for (RegionInfo hri : hris) {
304        tableRows.add(hri.getRegionName());
305      }
306      ADMIN.disableTable(tableName);
307      // now delete one replica info from all the rows
308      // this is to make the meta appear to be only partially updated
309      Table metaTable = ADMIN.getConnection().getTable(TableName.META_TABLE_NAME);
310      for (byte[] row : tableRows) {
311        Delete deleteOneReplicaLocation = new Delete(row);
312        deleteOneReplicaLocation.addColumns(HConstants.CATALOG_FAMILY,
313          MetaTableAccessor.getServerColumn(1));
314        deleteOneReplicaLocation.addColumns(HConstants.CATALOG_FAMILY,
315          MetaTableAccessor.getSeqNumColumn(1));
316        deleteOneReplicaLocation.addColumns(HConstants.CATALOG_FAMILY,
317          MetaTableAccessor.getStartCodeColumn(1));
318        metaTable.delete(deleteOneReplicaLocation);
319      }
320      metaTable.close();
321      // even if the meta table is partly updated, when we re-enable the table, we should
322      // get back the desired number of replicas for the regions
323      ADMIN.enableTable(tableName);
324      assertTrue(ADMIN.isTableEnabled(tableName));
325      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
326      TEST_UTIL.waitUntilNoRegionsInTransition();
327      List<RegionInfo> regions = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager()
328        .getRegionStates().getRegionsOfTable(tableName);
329      assertEquals(numRegions * numReplica, regions.size());
330    } finally {
331      ADMIN.disableTable(tableName);
332      ADMIN.deleteTable(tableName);
333    }
334  }
335
336  private void validateNumberOfRowsInMeta(final TableName table, int numRegions,
337    Connection connection) throws IOException {
338    assert (ADMIN.tableExists(table));
339    final AtomicInteger count = new AtomicInteger();
340    Visitor visitor = new Visitor() {
341      @Override
342      public boolean visit(Result r) throws IOException {
343        if (MetaTableAccessor.getRegionInfo(r).getTable().equals(table)) count.incrementAndGet();
344        return true;
345      }
346    };
347    MetaTableAccessor.fullScanRegions(connection, visitor);
348    assertEquals(numRegions, count.get());
349  }
350
351  private void validateFromSnapshotFromMeta(HBaseTestingUtility util, TableName table,
352    int numRegions, int numReplica, Connection connection) throws IOException {
353    SnapshotOfRegionAssignmentFromMeta snapshot =
354      new SnapshotOfRegionAssignmentFromMeta(connection);
355    snapshot.initialize();
356    Map<RegionInfo, ServerName> regionToServerMap = snapshot.getRegionToRegionServerMap();
357    assert (regionToServerMap.size() == numRegions * numReplica + 1); // '1' for the namespace
358    Map<ServerName, List<RegionInfo>> serverToRegionMap = snapshot.getRegionServerToRegionMap();
359    for (Map.Entry<ServerName, List<RegionInfo>> entry : serverToRegionMap.entrySet()) {
360      if (entry.getKey().equals(util.getHBaseCluster().getMaster().getServerName())) {
361        continue;
362      }
363      List<RegionInfo> regions = entry.getValue();
364      Set<byte[]> setOfStartKeys = new HashSet<>();
365      for (RegionInfo region : regions) {
366        byte[] startKey = region.getStartKey();
367        if (region.getTable().equals(table)) {
368          setOfStartKeys.add(startKey); // ignore other tables
369          LOG.info("--STARTKEY {}--", new String(startKey, StandardCharsets.UTF_8));
370        }
371      }
372      // the number of startkeys will be equal to the number of regions hosted in each server
373      // (each server will be hosting one replica of a region)
374      assertEquals(numRegions, setOfStartKeys.size());
375    }
376  }
377
378  private void validateSingleRegionServerAssignment(Connection connection, int numRegions,
379    int numReplica) throws IOException {
380    SnapshotOfRegionAssignmentFromMeta snapshot =
381      new SnapshotOfRegionAssignmentFromMeta(connection);
382    snapshot.initialize();
383    Map<RegionInfo, ServerName> regionToServerMap = snapshot.getRegionToRegionServerMap();
384    assertEquals(regionToServerMap.size(), numRegions * numReplica + 1);
385    Map<ServerName, List<RegionInfo>> serverToRegionMap = snapshot.getRegionServerToRegionMap();
386    assertEquals("One Region Only", 1, serverToRegionMap.keySet().size());
387    for (Map.Entry<ServerName, List<RegionInfo>> entry : serverToRegionMap.entrySet()) {
388      if (entry.getKey().equals(TEST_UTIL.getHBaseCluster().getMaster().getServerName())) {
389        continue;
390      }
391      assertEquals(entry.getValue().size(), numRegions * numReplica + 1);
392    }
393  }
394}