001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertNotEquals;
022import static org.junit.Assert.assertNotNull;
023import static org.junit.Assert.assertTrue;
024
025import java.io.IOException;
026import java.nio.charset.StandardCharsets;
027import java.util.ArrayList;
028import java.util.Collection;
029import java.util.EnumSet;
030import java.util.HashMap;
031import java.util.HashSet;
032import java.util.List;
033import java.util.Map;
034import java.util.Set;
035import java.util.concurrent.atomic.AtomicInteger;
036import org.apache.hadoop.conf.Configuration;
037import org.apache.hadoop.hbase.CatalogFamilyFormat;
038import org.apache.hadoop.hbase.ClientMetaTableAccessor;
039import org.apache.hadoop.hbase.ClusterMetrics.Option;
040import org.apache.hadoop.hbase.HBaseClassTestRule;
041import org.apache.hadoop.hbase.HBaseTestingUtil;
042import org.apache.hadoop.hbase.HConstants;
043import org.apache.hadoop.hbase.HRegionLocation;
044import org.apache.hadoop.hbase.MetaTableAccessor;
045import org.apache.hadoop.hbase.RegionLocations;
046import org.apache.hadoop.hbase.ServerName;
047import org.apache.hadoop.hbase.StartTestingClusterOption;
048import org.apache.hadoop.hbase.TableName;
049import org.apache.hadoop.hbase.client.Admin;
050import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
051import org.apache.hadoop.hbase.client.Connection;
052import org.apache.hadoop.hbase.client.ConnectionFactory;
053import org.apache.hadoop.hbase.client.Delete;
054import org.apache.hadoop.hbase.client.RegionInfo;
055import org.apache.hadoop.hbase.client.RegionReplicaUtil;
056import org.apache.hadoop.hbase.client.Result;
057import org.apache.hadoop.hbase.client.Table;
058import org.apache.hadoop.hbase.client.TableDescriptor;
059import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
060import org.apache.hadoop.hbase.testclassification.MasterTests;
061import org.apache.hadoop.hbase.testclassification.MediumTests;
062import org.apache.hadoop.hbase.util.Bytes;
063import org.apache.hadoop.hbase.util.JVMClusterUtil;
064import org.junit.AfterClass;
065import org.junit.BeforeClass;
066import org.junit.ClassRule;
067import org.junit.Rule;
068import org.junit.Test;
069import org.junit.experimental.categories.Category;
070import org.junit.rules.TestName;
071import org.slf4j.Logger;
072import org.slf4j.LoggerFactory;
073
074import org.apache.hbase.thirdparty.com.google.common.io.Closeables;
075
076@Category({ MasterTests.class, MediumTests.class })
077public class TestMasterOperationsForRegionReplicas {
078
079  @ClassRule
080  public static final HBaseClassTestRule CLASS_RULE =
081    HBaseClassTestRule.forClass(TestMasterOperationsForRegionReplicas.class);
082
083  private static final Logger LOG = LoggerFactory.getLogger(TestRegionPlacement.class);
084  private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
085  private static Connection CONNECTION = null;
086  private static Admin ADMIN;
087  private static int numSlaves = 2;
088  private final static StartTestingClusterOption option = StartTestingClusterOption.builder()
089    .numRegionServers(numSlaves).numMasters(1).numAlwaysStandByMasters(1).build();
090  private static Configuration conf;
091
092  @Rule
093  public TestName name = new TestName();
094
095  @BeforeClass
096  public static void setupBeforeClass() throws Exception {
097    conf = TEST_UTIL.getConfiguration();
098    conf.setBoolean("hbase.tests.use.shortcircuit.reads", false);
099    TEST_UTIL.startMiniCluster(option);
100    TEST_UTIL.getAdmin().balancerSwitch(false, true);
101    resetConnections();
102    while (
103      ADMIN.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics().size()
104          < numSlaves
105    ) {
106      Thread.sleep(100);
107    }
108  }
109
110  private static void resetConnections() throws IOException {
111    Closeables.close(ADMIN, true);
112    Closeables.close(CONNECTION, true);
113    CONNECTION = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
114    ADMIN = CONNECTION.getAdmin();
115  }
116
117  @AfterClass
118  public static void tearDownAfterClass() throws Exception {
119    Closeables.close(ADMIN, true);
120    Closeables.close(CONNECTION, true);
121    TEST_UTIL.shutdownMiniCluster();
122  }
123
124  @Test
125  public void testCreateTableWithSingleReplica() throws Exception {
126    final int numRegions = 3;
127    final int numReplica = 1;
128    final TableName tableName = TableName.valueOf(name.getMethodName());
129    try {
130      TableDescriptor desc =
131        TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(numReplica)
132          .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build();
133      ADMIN.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), numRegions);
134      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
135      TEST_UTIL.waitUntilNoRegionsInTransition();
136
137      validateNumberOfRowsInMeta(tableName, numRegions, ADMIN.getConnection());
138      List<RegionInfo> hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
139      assertEquals(numRegions * numReplica, hris.size());
140    } finally {
141      ADMIN.disableTable(tableName);
142      ADMIN.deleteTable(tableName);
143    }
144  }
145
146  @Test
147  public void testCreateTableWithMultipleReplicas() throws Exception {
148    final TableName tableName = TableName.valueOf(name.getMethodName());
149    final int numRegions = 3;
150    final int numReplica = 2;
151    try {
152      TableDescriptor desc =
153        TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(numReplica)
154          .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build();
155      ADMIN.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), numRegions);
156      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
157      TEST_UTIL.waitUntilNoRegionsInTransition();
158      validateNumberOfRowsInMeta(tableName, numRegions, ADMIN.getConnection());
159
160      List<RegionInfo> hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
161      assertEquals(numRegions * numReplica, hris.size());
162      assertRegionStateNotNull(hris, numRegions, numReplica);
163
164      List<Result> metaRows = MetaTableAccessor.fullScanRegions(ADMIN.getConnection());
165      int numRows = 0;
166      for (Result result : metaRows) {
167        RegionLocations locations = CatalogFamilyFormat.getRegionLocations(result);
168        RegionInfo hri = locations.getRegionLocation().getRegion();
169        if (!hri.getTable().equals(tableName)) continue;
170        numRows += 1;
171        HRegionLocation[] servers = locations.getRegionLocations();
172        // have two locations for the replicas of a region, and the locations should be different
173        assertEquals(2, servers.length);
174        assertNotEquals(servers[1], servers[0]);
175      }
176      assertEquals(numRegions, numRows);
177
178      // The same verification of the meta as above but with the SnapshotOfRegionAssignmentFromMeta
179      // class
180      validateFromSnapshotFromMeta(TEST_UTIL, tableName, numRegions, numReplica,
181        ADMIN.getConnection());
182
183      // Now kill the master, restart it and see if the assignments are kept
184      ServerName master = TEST_UTIL.getHBaseClusterInterface().getClusterMetrics().getMasterName();
185      TEST_UTIL.getHBaseClusterInterface().stopMaster(master);
186      TEST_UTIL.getHBaseClusterInterface().waitForMasterToStop(master, 30000);
187      TEST_UTIL.getHBaseClusterInterface().startMaster(master.getHostname(), master.getPort());
188      TEST_UTIL.getHBaseClusterInterface().waitForActiveAndReadyMaster();
189      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
190      TEST_UTIL.waitUntilNoRegionsInTransition();
191      assertRegionStateNotNull(hris, numRegions, numReplica);
192      validateFromSnapshotFromMeta(TEST_UTIL, tableName, numRegions, numReplica,
193        ADMIN.getConnection());
194
195      // Now shut the whole cluster down, and verify the assignments are kept so that the
196      // availability constraints are met. MiniHBaseCluster chooses arbitrary ports on each
197      // restart. This messes with our being able to test that we retain locality. Therefore,
198      // figure current cluster ports and pass them in on next cluster start so new cluster comes
199      // up at same coordinates -- and the assignment retention logic has a chance to cut in.
200      List<Integer> rsports = new ArrayList<>();
201      for (JVMClusterUtil.RegionServerThread rst : TEST_UTIL.getHBaseCluster()
202        .getLiveRegionServerThreads()) {
203        rsports.add(rst.getRegionServer().getRpcServer().getListenerAddress().getPort());
204      }
205      TEST_UTIL.shutdownMiniHBaseCluster();
206      StartTestingClusterOption option =
207        StartTestingClusterOption.builder().numRegionServers(numSlaves).rsPorts(rsports).build();
208      TEST_UTIL.startMiniHBaseCluster(option);
209      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
210      TEST_UTIL.waitUntilNoRegionsInTransition();
211      resetConnections();
212      validateFromSnapshotFromMeta(TEST_UTIL, tableName, numRegions, numReplica,
213        ADMIN.getConnection());
214
215      // Now shut the whole cluster down, and verify regions are assigned even if there is only
216      // one server running
217      TEST_UTIL.shutdownMiniHBaseCluster();
218      TEST_UTIL.startMiniHBaseCluster();
219      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
220      TEST_UTIL.waitUntilNoRegionsInTransition();
221      resetConnections();
222      validateSingleRegionServerAssignment(ADMIN.getConnection(), numRegions, numReplica);
223      for (int i = 1; i < numSlaves; i++) { // restore the cluster
224        TEST_UTIL.getMiniHBaseCluster().startRegionServer();
225      }
226
227      // Check on alter table
228      ADMIN.disableTable(tableName);
229      assertTrue(ADMIN.isTableDisabled(tableName));
230      // increase the replica
231      ADMIN.modifyTable(
232        TableDescriptorBuilder.newBuilder(desc).setRegionReplication(numReplica + 1).build());
233      ADMIN.enableTable(tableName);
234      LOG.info(ADMIN.getDescriptor(tableName).toString());
235      assertTrue(ADMIN.isTableEnabled(tableName));
236      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
237      TEST_UTIL.waitUntilNoRegionsInTransition();
238      List<RegionInfo> regions = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager()
239        .getRegionStates().getRegionsOfTable(tableName);
240      assertTrue("regions.size=" + regions.size() + ", numRegions=" + numRegions + ", numReplica="
241        + numReplica, regions.size() == numRegions * (numReplica + 1));
242
243      // decrease the replica(earlier, table was modified to have a replica count of numReplica + 1)
244      ADMIN.disableTable(tableName);
245      ADMIN.modifyTable(
246        TableDescriptorBuilder.newBuilder(desc).setRegionReplication(numReplica).build());
247      ADMIN.enableTable(tableName);
248      assertTrue(ADMIN.isTableEnabled(tableName));
249      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
250      TEST_UTIL.waitUntilNoRegionsInTransition();
251      regions = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates()
252        .getRegionsOfTable(tableName);
253      assertEquals(numRegions * numReplica, regions.size());
254      // also make sure the meta table has the replica locations removed
255      hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
256      assertEquals(numRegions * numReplica, hris.size());
257      // just check that the number of default replica regions in the meta table are the same
258      // as the number of regions the table was created with, and the count of the
259      // replicas is numReplica for each region
260      Map<RegionInfo, Integer> defaultReplicas = new HashMap<>();
261      for (RegionInfo hri : hris) {
262        RegionInfo regionReplica0 = RegionReplicaUtil.getRegionInfoForDefaultReplica(hri);
263        Integer i = defaultReplicas.get(regionReplica0);
264        defaultReplicas.put(regionReplica0, i == null ? 1 : i + 1);
265      }
266      assertEquals(numRegions, defaultReplicas.size());
267      Collection<Integer> counts = new HashSet<>(defaultReplicas.values());
268      assertEquals(1, counts.size());
269      assertTrue(counts.contains(numReplica));
270    } finally {
271      ADMIN.disableTable(tableName);
272      ADMIN.deleteTable(tableName);
273    }
274  }
275
276  private void assertRegionStateNotNull(List<RegionInfo> hris, int numRegions, int numReplica) {
277    // check that the master created expected number of RegionState objects
278    for (int i = 0; i < numRegions; i++) {
279      for (int j = 0; j < numReplica; j++) {
280        RegionInfo replica = RegionReplicaUtil.getRegionInfoForReplica(hris.get(i), j);
281        RegionState state = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
282          .getRegionStates().getRegionState(replica);
283        assertNotNull(state);
284      }
285    }
286  }
287
288  @Test
289  public void testIncompleteMetaTableReplicaInformation() throws Exception {
290    final TableName tableName = TableName.valueOf(name.getMethodName());
291    final int numRegions = 3;
292    final int numReplica = 2;
293    try {
294      // Create a table and let the meta table be updated with the location of the
295      // region locations.
296      TableDescriptor desc =
297        TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(numReplica)
298          .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build();
299      ADMIN.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), numRegions);
300      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
301      TEST_UTIL.waitUntilNoRegionsInTransition();
302      Set<byte[]> tableRows = new HashSet<>();
303      List<RegionInfo> hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
304      for (RegionInfo hri : hris) {
305        tableRows.add(hri.getRegionName());
306      }
307      ADMIN.disableTable(tableName);
308      // now delete one replica info from all the rows
309      // this is to make the meta appear to be only partially updated
310      Table metaTable = ADMIN.getConnection().getTable(TableName.META_TABLE_NAME);
311      for (byte[] row : tableRows) {
312        Delete deleteOneReplicaLocation = new Delete(row);
313        deleteOneReplicaLocation.addColumns(HConstants.CATALOG_FAMILY,
314          CatalogFamilyFormat.getServerColumn(1));
315        deleteOneReplicaLocation.addColumns(HConstants.CATALOG_FAMILY,
316          CatalogFamilyFormat.getSeqNumColumn(1));
317        deleteOneReplicaLocation.addColumns(HConstants.CATALOG_FAMILY,
318          CatalogFamilyFormat.getStartCodeColumn(1));
319        metaTable.delete(deleteOneReplicaLocation);
320      }
321      metaTable.close();
322      // even if the meta table is partly updated, when we re-enable the table, we should
323      // get back the desired number of replicas for the regions
324      ADMIN.enableTable(tableName);
325      assertTrue(ADMIN.isTableEnabled(tableName));
326      TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
327      TEST_UTIL.waitUntilNoRegionsInTransition();
328      List<RegionInfo> regions = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager()
329        .getRegionStates().getRegionsOfTable(tableName);
330      assertEquals(numRegions * numReplica, regions.size());
331    } finally {
332      ADMIN.disableTable(tableName);
333      ADMIN.deleteTable(tableName);
334    }
335  }
336
337  private void validateNumberOfRowsInMeta(final TableName table, int numRegions,
338    Connection connection) throws IOException {
339    assert (ADMIN.tableExists(table));
340    final AtomicInteger count = new AtomicInteger();
341    ClientMetaTableAccessor.Visitor visitor = new ClientMetaTableAccessor.Visitor() {
342      @Override
343      public boolean visit(Result r) throws IOException {
344        if (CatalogFamilyFormat.getRegionInfo(r).getTable().equals(table)) {
345          count.incrementAndGet();
346        }
347        return true;
348      }
349    };
350    MetaTableAccessor.fullScanRegions(connection, visitor);
351    assertEquals(numRegions, count.get());
352  }
353
354  private void validateFromSnapshotFromMeta(HBaseTestingUtil util, TableName table, int numRegions,
355    int numReplica, Connection connection) throws IOException {
356    SnapshotOfRegionAssignmentFromMeta snapshot =
357      new SnapshotOfRegionAssignmentFromMeta(connection);
358    snapshot.initialize();
359    Map<RegionInfo, ServerName> regionToServerMap = snapshot.getRegionToRegionServerMap();
360    assert (regionToServerMap.size() == numRegions * numReplica);
361    Map<ServerName, List<RegionInfo>> serverToRegionMap = snapshot.getRegionServerToRegionMap();
362    for (Map.Entry<ServerName, List<RegionInfo>> entry : serverToRegionMap.entrySet()) {
363      if (entry.getKey().equals(util.getHBaseCluster().getMaster().getServerName())) {
364        continue;
365      }
366      List<RegionInfo> regions = entry.getValue();
367      Set<byte[]> setOfStartKeys = new HashSet<>();
368      for (RegionInfo region : regions) {
369        byte[] startKey = region.getStartKey();
370        if (region.getTable().equals(table)) {
371          setOfStartKeys.add(startKey); // ignore other tables
372          LOG.info("--STARTKEY {}--", new String(startKey, StandardCharsets.UTF_8));
373        }
374      }
375      // the number of startkeys will be equal to the number of regions hosted in each server
376      // (each server will be hosting one replica of a region)
377      assertEquals(numRegions, setOfStartKeys.size());
378    }
379  }
380
381  private void validateSingleRegionServerAssignment(Connection connection, int numRegions,
382    int numReplica) throws IOException {
383    SnapshotOfRegionAssignmentFromMeta snapshot =
384      new SnapshotOfRegionAssignmentFromMeta(connection);
385    snapshot.initialize();
386    Map<RegionInfo, ServerName> regionToServerMap = snapshot.getRegionToRegionServerMap();
387    assertEquals(regionToServerMap.size(), numRegions * numReplica);
388    Map<ServerName, List<RegionInfo>> serverToRegionMap = snapshot.getRegionServerToRegionMap();
389    assertEquals("One Region Only", 1, serverToRegionMap.keySet().size());
390    for (Map.Entry<ServerName, List<RegionInfo>> entry : serverToRegionMap.entrySet()) {
391      if (entry.getKey().equals(TEST_UTIL.getHBaseCluster().getMaster().getServerName())) {
392        continue;
393      }
394      assertEquals(entry.getValue().size(), numRegions * numReplica);
395    }
396  }
397}