Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.namespace;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertNotNull;
023import static org.junit.Assert.assertNull;
024import static org.junit.Assert.assertTrue;
025import static org.junit.Assert.fail;
026
027import java.io.IOException;
028import java.util.Collections;
029import java.util.List;
030import java.util.Optional;
031import java.util.concurrent.CountDownLatch;
032import java.util.concurrent.ExecutionException;
033import java.util.concurrent.Future;
034import java.util.concurrent.TimeUnit;
035import org.apache.hadoop.conf.Configuration;
036import org.apache.hadoop.fs.FileSystem;
037import org.apache.hadoop.fs.Path;
038import org.apache.hadoop.hbase.Coprocessor;
039import org.apache.hadoop.hbase.CoprocessorEnvironment;
040import org.apache.hadoop.hbase.DoNotRetryIOException;
041import org.apache.hadoop.hbase.HBaseClassTestRule;
042import org.apache.hadoop.hbase.HBaseTestingUtil;
043import org.apache.hadoop.hbase.HConstants;
044import org.apache.hadoop.hbase.NamespaceDescriptor;
045import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
046import org.apache.hadoop.hbase.StartTestingClusterOption;
047import org.apache.hadoop.hbase.TableName;
048import org.apache.hadoop.hbase.Waiter;
049import org.apache.hadoop.hbase.client.Admin;
050import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
051import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
052import org.apache.hadoop.hbase.client.CompactionState;
053import org.apache.hadoop.hbase.client.Connection;
054import org.apache.hadoop.hbase.client.ConnectionFactory;
055import org.apache.hadoop.hbase.client.RegionInfo;
056import org.apache.hadoop.hbase.client.RegionLocator;
057import org.apache.hadoop.hbase.client.Table;
058import org.apache.hadoop.hbase.client.TableDescriptor;
059import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
060import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
061import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor;
062import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
063import org.apache.hadoop.hbase.coprocessor.MasterObserver;
064import org.apache.hadoop.hbase.coprocessor.ObserverContext;
065import org.apache.hadoop.hbase.coprocessor.RegionCoprocessor;
066import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
067import org.apache.hadoop.hbase.coprocessor.RegionObserver;
068import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessor;
069import org.apache.hadoop.hbase.coprocessor.RegionServerObserver;
070import org.apache.hadoop.hbase.master.HMaster;
071import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
072import org.apache.hadoop.hbase.master.TableNamespaceManager;
073import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
074import org.apache.hadoop.hbase.quotas.QuotaExceededException;
075import org.apache.hadoop.hbase.quotas.QuotaUtil;
076import org.apache.hadoop.hbase.regionserver.HRegion;
077import org.apache.hadoop.hbase.regionserver.Store;
078import org.apache.hadoop.hbase.regionserver.StoreFile;
079import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
080import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
081import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
082import org.apache.hadoop.hbase.testclassification.LargeTests;
083import org.apache.hadoop.hbase.util.Bytes;
084import org.apache.hadoop.hbase.util.CommonFSUtils;
085import org.apache.zookeeper.KeeperException;
086import org.junit.After;
087import org.junit.AfterClass;
088import org.junit.BeforeClass;
089import org.junit.ClassRule;
090import org.junit.Test;
091import org.junit.experimental.categories.Category;
092import org.slf4j.Logger;
093import org.slf4j.LoggerFactory;
094
095@Category(LargeTests.class)
096public class TestNamespaceAuditor {
097
098  @ClassRule
099  public static final HBaseClassTestRule CLASS_RULE =
100    HBaseClassTestRule.forClass(TestNamespaceAuditor.class);
101
102  private static final Logger LOG = LoggerFactory.getLogger(TestNamespaceAuditor.class);
103  private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
104  private static Admin ADMIN;
105  private String prefix = "TestNamespaceAuditor";
106
107  @BeforeClass
108  public static void before() throws Exception {
109    Configuration conf = UTIL.getConfiguration();
110    conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, CustomObserver.class.getName());
111    conf.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, MasterSyncObserver.class.getName(),
112      CPMasterObserver.class.getName());
113    conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 5);
114    conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
115    conf.setClass("hbase.coprocessor.regionserver.classes", CPRegionServerObserver.class,
116      RegionServerObserver.class);
117    StartTestingClusterOption option = StartTestingClusterOption.builder().numMasters(2).build();
118    UTIL.startMiniCluster(option);
119    waitForQuotaInitialize(UTIL);
120    ADMIN = UTIL.getAdmin();
121  }
122
123  @AfterClass
124  public static void tearDown() throws Exception {
125    UTIL.shutdownMiniCluster();
126  }
127
128  @After
129  public void cleanup() throws Exception, KeeperException {
130    for (TableDescriptor table : ADMIN.listTableDescriptors()) {
131      ADMIN.disableTable(table.getTableName());
132      deleteTable(table.getTableName());
133    }
134    for (NamespaceDescriptor ns : ADMIN.listNamespaceDescriptors()) {
135      if (ns.getName().startsWith(prefix)) {
136        ADMIN.deleteNamespace(ns.getName());
137      }
138    }
139    assertTrue("Quota manager not initialized",
140      UTIL.getHBaseCluster().getMaster().getMasterQuotaManager().isQuotaInitialized());
141  }
142
143  @Test
144  public void testTableOperations() throws Exception {
145    String nsp = prefix + "_np2";
146    NamespaceDescriptor nspDesc =
147      NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "5")
148        .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
149    ADMIN.createNamespace(nspDesc);
150    assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
151    assertEquals(3, ADMIN.listNamespaceDescriptors().length);
152    ColumnFamilyDescriptor columnFamilyDescriptor =
153      ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("fam1")).build();
154
155    TableDescriptorBuilder tableDescOne = TableDescriptorBuilder
156      .newBuilder(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1"));
157    tableDescOne.setColumnFamily(columnFamilyDescriptor);
158    TableDescriptorBuilder tableDescTwo = TableDescriptorBuilder
159      .newBuilder(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2"));
160    tableDescTwo.setColumnFamily(columnFamilyDescriptor);
161    TableDescriptorBuilder tableDescThree = TableDescriptorBuilder
162      .newBuilder(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table3"));
163    tableDescThree.setColumnFamily(columnFamilyDescriptor);
164    ADMIN.createTable(tableDescOne.build());
165    boolean constraintViolated = false;
166    try {
167      ADMIN.createTable(tableDescTwo.build(), Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 5);
168    } catch (Exception exp) {
169      assertTrue(exp instanceof IOException);
170      constraintViolated = true;
171    } finally {
172      assertTrue("Constraint not violated for table " + tableDescTwo.build().getTableName(),
173        constraintViolated);
174    }
175    ADMIN.createTable(tableDescTwo.build(), Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
176    NamespaceTableAndRegionInfo nspState = getQuotaManager().getState(nsp);
177    assertNotNull(nspState);
178    assertTrue(nspState.getTables().size() == 2);
179    assertTrue(nspState.getRegionCount() == 5);
180    constraintViolated = false;
181    try {
182      ADMIN.createTable(tableDescThree.build());
183    } catch (Exception exp) {
184      assertTrue(exp instanceof IOException);
185      constraintViolated = true;
186    } finally {
187      assertTrue("Constraint not violated for table " + tableDescThree.build().getTableName(),
188        constraintViolated);
189    }
190  }
191
192  @Test
193  public void testValidQuotas() throws Exception {
194    boolean exceptionCaught = false;
195    FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
196    Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
197    NamespaceDescriptor nspDesc = NamespaceDescriptor.create(prefix + "vq1")
198      .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "hihdufh")
199      .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
200    try {
201      ADMIN.createNamespace(nspDesc);
202    } catch (Exception exp) {
203      LOG.warn(exp.toString(), exp);
204      exceptionCaught = true;
205    } finally {
206      assertTrue(exceptionCaught);
207      assertFalse(fs.exists(CommonFSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
208    }
209    nspDesc = NamespaceDescriptor.create(prefix + "vq2")
210      .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "-456")
211      .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
212    try {
213      ADMIN.createNamespace(nspDesc);
214    } catch (Exception exp) {
215      LOG.warn(exp.toString(), exp);
216      exceptionCaught = true;
217    } finally {
218      assertTrue(exceptionCaught);
219      assertFalse(fs.exists(CommonFSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
220    }
221    nspDesc = NamespaceDescriptor.create(prefix + "vq3")
222      .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10")
223      .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "sciigd").build();
224    try {
225      ADMIN.createNamespace(nspDesc);
226    } catch (Exception exp) {
227      LOG.warn(exp.toString(), exp);
228      exceptionCaught = true;
229    } finally {
230      assertTrue(exceptionCaught);
231      assertFalse(fs.exists(CommonFSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
232    }
233    nspDesc = NamespaceDescriptor.create(prefix + "vq4")
234      .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10")
235      .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "-1500").build();
236    try {
237      ADMIN.createNamespace(nspDesc);
238    } catch (Exception exp) {
239      LOG.warn(exp.toString(), exp);
240      exceptionCaught = true;
241    } finally {
242      assertTrue(exceptionCaught);
243      assertFalse(fs.exists(CommonFSUtils.getNamespaceDir(rootDir, nspDesc.getName())));
244    }
245  }
246
247  @Test
248  public void testDeleteTable() throws Exception {
249    String namespace = prefix + "_dummy";
250    NamespaceDescriptor nspDesc = NamespaceDescriptor.create(namespace)
251      .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "100")
252      .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "3").build();
253    ADMIN.createNamespace(nspDesc);
254    assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(namespace));
255    NamespaceTableAndRegionInfo stateInfo = getNamespaceState(nspDesc.getName());
256    assertNotNull("Namespace state found null for " + namespace, stateInfo);
257    ColumnFamilyDescriptor columnFamilyDescriptor =
258      ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("fam1")).build();
259    TableDescriptorBuilder tableDescOne = TableDescriptorBuilder
260      .newBuilder(TableName.valueOf(namespace + TableName.NAMESPACE_DELIM + "table1"));
261    tableDescOne.setColumnFamily(columnFamilyDescriptor);
262    TableDescriptorBuilder tableDescTwo = TableDescriptorBuilder
263      .newBuilder(TableName.valueOf(namespace + TableName.NAMESPACE_DELIM + "table2"));
264    tableDescTwo.setColumnFamily(columnFamilyDescriptor);
265    ADMIN.createTable(tableDescOne.build());
266    ADMIN.createTable(tableDescTwo.build(), Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 5);
267    stateInfo = getNamespaceState(nspDesc.getName());
268    assertNotNull("Namespace state found to be null.", stateInfo);
269    assertEquals(2, stateInfo.getTables().size());
270    assertEquals(5, stateInfo.getRegionCountOfTable(tableDescTwo.build().getTableName()));
271    assertEquals(6, stateInfo.getRegionCount());
272    ADMIN.disableTable(tableDescOne.build().getTableName());
273    deleteTable(tableDescOne.build().getTableName());
274    stateInfo = getNamespaceState(nspDesc.getName());
275    assertNotNull("Namespace state found to be null.", stateInfo);
276    assertEquals(5, stateInfo.getRegionCount());
277    assertEquals(1, stateInfo.getTables().size());
278    ADMIN.disableTable(tableDescTwo.build().getTableName());
279    deleteTable(tableDescTwo.build().getTableName());
280    ADMIN.deleteNamespace(namespace);
281    stateInfo = getNamespaceState(namespace);
282    assertNull("Namespace state not found to be null.", stateInfo);
283  }
284
285  public static class CPRegionServerObserver
286    implements RegionServerCoprocessor, RegionServerObserver {
287    private volatile boolean shouldFailMerge = false;
288
289    public void failMerge(boolean fail) {
290      shouldFailMerge = fail;
291    }
292
293    private boolean triggered = false;
294
295    public synchronized void waitUtilTriggered() throws InterruptedException {
296      while (!triggered) {
297        wait();
298      }
299    }
300
301    @Override
302    public Optional<RegionServerObserver> getRegionServerObserver() {
303      return Optional.of(this);
304    }
305  }
306
307  public static class CPMasterObserver implements MasterCoprocessor, MasterObserver {
308    private volatile boolean shouldFailMerge = false;
309
310    public void failMerge(boolean fail) {
311      shouldFailMerge = fail;
312    }
313
314    @Override
315    public Optional<MasterObserver> getMasterObserver() {
316      return Optional.of(this);
317    }
318
319    @Override
320    public synchronized void preMergeRegionsAction(
321      final ObserverContext<MasterCoprocessorEnvironment> ctx, final RegionInfo[] regionsToMerge)
322      throws IOException {
323      notifyAll();
324      if (shouldFailMerge) {
325        throw new IOException("fail merge");
326      }
327    }
328  }
329
330  @Test
331  public void testRegionMerge() throws Exception {
332    String nsp1 = prefix + "_regiontest";
333    final int initialRegions = 3;
334    NamespaceDescriptor nspDesc = NamespaceDescriptor.create(nsp1)
335      .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "" + initialRegions)
336      .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
337    ADMIN.createNamespace(nspDesc);
338    final TableName tableTwo = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table2");
339    byte[] columnFamily = Bytes.toBytes("info");
340    TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableTwo)
341      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(columnFamily)).build();
342    ADMIN.createTable(tableDescriptor, Bytes.toBytes("0"), Bytes.toBytes("9"), initialRegions);
343    Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
344    try (Table table = connection.getTable(tableTwo)) {
345      UTIL.loadNumericRows(table, Bytes.toBytes("info"), 1000, 1999);
346    }
347    ADMIN.flush(tableTwo);
348    List<RegionInfo> hris = ADMIN.getRegions(tableTwo);
349    assertEquals(initialRegions, hris.size());
350    Collections.sort(hris, RegionInfo.COMPARATOR);
351    Future<?> f = ADMIN.mergeRegionsAsync(hris.get(0).getEncodedNameAsBytes(),
352      hris.get(1).getEncodedNameAsBytes(), false);
353    f.get(10, TimeUnit.SECONDS);
354
355    hris = ADMIN.getRegions(tableTwo);
356    assertEquals(initialRegions - 1, hris.size());
357    Collections.sort(hris, RegionInfo.COMPARATOR);
358    byte[] splitKey = Bytes.toBytes("3");
359    HRegion regionToSplit = UTIL.getMiniHBaseCluster().getRegions(tableTwo).stream()
360      .filter(r -> r.getRegionInfo().containsRow(splitKey)).findFirst().get();
361    regionToSplit.compact(true);
362    // Waiting for compaction to finish
363    UTIL.waitFor(30000, new Waiter.Predicate<Exception>() {
364      @Override
365      public boolean evaluate() throws Exception {
366        return (CompactionState.NONE
367            == ADMIN.getCompactionStateForRegion(regionToSplit.getRegionInfo().getRegionName()));
368      }
369    });
370
371    // Cleaning compacted references for split to proceed
372    regionToSplit.getStores().stream().forEach(s -> {
373      try {
374        s.closeAndArchiveCompactedFiles();
375      } catch (IOException e1) {
376        LOG.error("Error whiling cleaning compacted file");
377      }
378    });
379    // the above compact may quit immediately if there is a compaction ongoing, so here we need to
380    // wait a while to let the ongoing compaction finish.
381    UTIL.waitFor(10000, regionToSplit::isSplittable);
382    ADMIN.splitRegionAsync(regionToSplit.getRegionInfo().getRegionName(), splitKey).get(10,
383      TimeUnit.SECONDS);
384    hris = ADMIN.getRegions(tableTwo);
385    assertEquals(initialRegions, hris.size());
386    Collections.sort(hris, RegionInfo.COMPARATOR);
387
388    // Fail region merge through Coprocessor hook
389    SingleProcessHBaseCluster cluster = UTIL.getHBaseCluster();
390    MasterCoprocessorHost cpHost = cluster.getMaster().getMasterCoprocessorHost();
391    Coprocessor coprocessor = cpHost.findCoprocessor(CPMasterObserver.class);
392    CPMasterObserver masterObserver = (CPMasterObserver) coprocessor;
393    masterObserver.failMerge(true);
394
395    f = ADMIN.mergeRegionsAsync(hris.get(1).getEncodedNameAsBytes(),
396      hris.get(2).getEncodedNameAsBytes(), false);
397    try {
398      f.get(10, TimeUnit.SECONDS);
399      fail("Merge was supposed to fail!");
400    } catch (ExecutionException ee) {
401      // Expected.
402    }
403    hris = ADMIN.getRegions(tableTwo);
404    assertEquals(initialRegions, hris.size());
405    Collections.sort(hris, RegionInfo.COMPARATOR);
406    // verify that we cannot split
407    try {
408      ADMIN.split(tableTwo, Bytes.toBytes("6"));
409      fail();
410    } catch (DoNotRetryIOException e) {
411      // Expected
412    }
413    Thread.sleep(2000);
414    assertEquals(initialRegions, ADMIN.getRegions(tableTwo).size());
415  }
416
417  /*
418   * Create a table and make sure that the table creation fails after adding this table entry into
419   * namespace quota cache. Now correct the failure and recreate the table with same name.
420   * HBASE-13394
421   */
422  @Test
423  public void testRecreateTableWithSameNameAfterFirstTimeFailure() throws Exception {
424    String nsp1 = prefix + "_testRecreateTable";
425    NamespaceDescriptor nspDesc =
426      NamespaceDescriptor.create(nsp1).addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20")
427        .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1").build();
428    ADMIN.createNamespace(nspDesc);
429    final TableName tableOne = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table1");
430    byte[] columnFamily = Bytes.toBytes("info");
431    TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableOne)
432      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(columnFamily)).build();
433    MasterSyncObserver.throwExceptionInPreCreateTableAction = true;
434    try {
435      try {
436        ADMIN.createTable(tableDescriptor);
437        fail("Table " + tableOne.toString() + "creation should fail.");
438      } catch (Exception exp) {
439        LOG.error(exp.toString(), exp);
440      }
441      assertFalse(ADMIN.tableExists(tableOne));
442
443      NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp1);
444      assertEquals("First table creation failed in namespace so number of tables in namespace "
445        + "should be 0.", 0, nstate.getTables().size());
446
447      MasterSyncObserver.throwExceptionInPreCreateTableAction = false;
448      try {
449        ADMIN.createTable(tableDescriptor);
450      } catch (Exception e) {
451        fail("Table " + tableOne.toString() + "creation should succeed.");
452        LOG.error(e.toString(), e);
453      }
454      assertTrue(ADMIN.tableExists(tableOne));
455      nstate = getNamespaceState(nsp1);
456      assertEquals(
457        "First table was created successfully so table size in namespace should " + "be one now.",
458        1, nstate.getTables().size());
459    } finally {
460      MasterSyncObserver.throwExceptionInPreCreateTableAction = false;
461      if (ADMIN.tableExists(tableOne)) {
462        ADMIN.disableTable(tableOne);
463        deleteTable(tableOne);
464      }
465      ADMIN.deleteNamespace(nsp1);
466    }
467  }
468
469  private NamespaceTableAndRegionInfo getNamespaceState(String namespace)
470    throws KeeperException, IOException {
471    return getQuotaManager().getState(namespace);
472  }
473
474  public static class CustomObserver implements RegionCoprocessor, RegionObserver {
475    volatile CountDownLatch postCompact;
476
477    @Override
478    public void postCompact(ObserverContext<? extends RegionCoprocessorEnvironment> e, Store store,
479      StoreFile resultFile, CompactionLifeCycleTracker tracker, CompactionRequest request)
480      throws IOException {
481      postCompact.countDown();
482    }
483
484    @Override
485    public void start(CoprocessorEnvironment e) throws IOException {
486      postCompact = new CountDownLatch(1);
487    }
488
489    @Override
490    public Optional<RegionObserver> getRegionObserver() {
491      return Optional.of(this);
492    }
493  }
494
495  @Test
496  public void testStatePreserve() throws Exception {
497    final String nsp1 = prefix + "_testStatePreserve";
498    NamespaceDescriptor nspDesc =
499      NamespaceDescriptor.create(nsp1).addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20")
500        .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "10").build();
501    ADMIN.createNamespace(nspDesc);
502    TableName tableOne = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table1");
503    TableName tableTwo = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table2");
504    TableName tableThree = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table3");
505    ColumnFamilyDescriptor columnFamilyDescriptor =
506      ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("fam1")).build();
507    TableDescriptorBuilder tableDescOne = TableDescriptorBuilder.newBuilder(tableOne);
508    tableDescOne.setColumnFamily(columnFamilyDescriptor);
509    TableDescriptorBuilder tableDescTwo = TableDescriptorBuilder.newBuilder(tableTwo);
510    tableDescTwo.setColumnFamily(columnFamilyDescriptor);
511    TableDescriptorBuilder tableDescThree = TableDescriptorBuilder.newBuilder(tableThree);
512    tableDescThree.setColumnFamily(columnFamilyDescriptor);
513
514    ADMIN.createTable(tableDescOne.build(), Bytes.toBytes("1"), Bytes.toBytes("1000"), 3);
515    ADMIN.createTable(tableDescTwo.build(), Bytes.toBytes("1"), Bytes.toBytes("1000"), 3);
516    ADMIN.createTable(tableDescThree.build(), Bytes.toBytes("1"), Bytes.toBytes("1000"), 4);
517    ADMIN.disableTable(tableThree);
518    deleteTable(tableThree);
519    // wait for chore to complete
520    UTIL.waitFor(1000, new Waiter.Predicate<Exception>() {
521      @Override
522      public boolean evaluate() throws Exception {
523        return (getNamespaceState(nsp1).getTables().size() == 2);
524      }
525    });
526    NamespaceTableAndRegionInfo before = getNamespaceState(nsp1);
527    killActiveMaster();
528    NamespaceTableAndRegionInfo after = getNamespaceState(nsp1);
529    assertEquals("Expected: " + before.getTables() + " Found: " + after.getTables(),
530      before.getTables().size(), after.getTables().size());
531  }
532
533  public static void waitForQuotaInitialize(final HBaseTestingUtil util) throws Exception {
534    util.waitFor(60000, new Waiter.Predicate<Exception>() {
535      @Override
536      public boolean evaluate() throws Exception {
537        HMaster master = util.getHBaseCluster().getMaster();
538        if (master == null) {
539          return false;
540        }
541        MasterQuotaManager quotaManager = master.getMasterQuotaManager();
542        return quotaManager != null && quotaManager.isQuotaInitialized();
543      }
544    });
545  }
546
547  private void killActiveMaster() throws Exception {
548    UTIL.getHBaseCluster().getMaster(0).stop("Stopping to start again");
549    UTIL.getHBaseCluster().waitOnMaster(0);
550    waitForQuotaInitialize(UTIL);
551  }
552
553  private NamespaceAuditor getQuotaManager() {
554    return UTIL.getHBaseCluster().getMaster().getMasterQuotaManager().getNamespaceQuotaManager();
555  }
556
557  public static class MasterSyncObserver implements MasterCoprocessor, MasterObserver {
558    volatile CountDownLatch tableDeletionLatch;
559    static boolean throwExceptionInPreCreateTableAction;
560
561    @Override
562    public Optional<MasterObserver> getMasterObserver() {
563      return Optional.of(this);
564    }
565
566    @Override
567    public void preDeleteTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
568      TableName tableName) throws IOException {
569      tableDeletionLatch = new CountDownLatch(1);
570    }
571
572    @Override
573    public void postCompletedDeleteTableAction(
574      final ObserverContext<MasterCoprocessorEnvironment> ctx, final TableName tableName)
575      throws IOException {
576      tableDeletionLatch.countDown();
577    }
578
579    @Override
580    public void preCreateTableAction(ObserverContext<MasterCoprocessorEnvironment> ctx,
581      TableDescriptor desc, RegionInfo[] regions) throws IOException {
582      if (throwExceptionInPreCreateTableAction) {
583        throw new IOException("Throw exception as it is demanded.");
584      }
585    }
586  }
587
588  private void deleteTable(final TableName tableName) throws Exception {
589    // NOTE: We need a latch because admin is not sync,
590    // so the postOp coprocessor method may be called after the admin operation returned.
591    MasterSyncObserver observer = UTIL.getHBaseCluster().getMaster().getMasterCoprocessorHost()
592      .findCoprocessor(MasterSyncObserver.class);
593    ADMIN.deleteTable(tableName);
594    observer.tableDeletionLatch.await();
595  }
596
597  @Test(expected = QuotaExceededException.class)
598  public void testExceedTableQuotaInNamespace() throws Exception {
599    String nsp = prefix + "_testExceedTableQuotaInNamespace";
600    NamespaceDescriptor nspDesc = NamespaceDescriptor.create(nsp)
601      .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1").build();
602    ADMIN.createNamespace(nspDesc);
603    assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
604    assertEquals(3, ADMIN.listNamespaceDescriptors().length);
605    ColumnFamilyDescriptor columnFamilyDescriptor =
606      ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("fam1")).build();
607    TableDescriptorBuilder tableDescOne = TableDescriptorBuilder
608      .newBuilder(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1"));
609    tableDescOne.setColumnFamily(columnFamilyDescriptor);
610    TableDescriptorBuilder tableDescTwo = TableDescriptorBuilder
611      .newBuilder(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2"));
612    tableDescTwo.setColumnFamily(columnFamilyDescriptor);
613    ADMIN.createTable(tableDescOne.build());
614    ADMIN.createTable(tableDescTwo.build(), Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
615  }
616
617  @Test(expected = QuotaExceededException.class)
618  public void testCloneSnapshotQuotaExceed() throws Exception {
619    String nsp = prefix + "_testTableQuotaExceedWithCloneSnapshot";
620    NamespaceDescriptor nspDesc = NamespaceDescriptor.create(nsp)
621      .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1").build();
622    ADMIN.createNamespace(nspDesc);
623    assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
624    TableName tableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
625    TableName cloneTableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2");
626    ColumnFamilyDescriptor columnFamilyDescriptor =
627      ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("fam1")).build();
628    TableDescriptorBuilder tableDescOne = TableDescriptorBuilder.newBuilder(tableName);
629    tableDescOne.setColumnFamily(columnFamilyDescriptor);
630    ADMIN.createTable(tableDescOne.build());
631    String snapshot = "snapshot_testTableQuotaExceedWithCloneSnapshot";
632    ADMIN.snapshot(snapshot, tableName);
633    ADMIN.cloneSnapshot(snapshot, cloneTableName);
634    ADMIN.deleteSnapshot(snapshot);
635  }
636
637  @Test
638  public void testCloneSnapshot() throws Exception {
639    String nsp = prefix + "_testCloneSnapshot";
640    NamespaceDescriptor nspDesc =
641      NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2")
642        .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20").build();
643    ADMIN.createNamespace(nspDesc);
644    assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
645    TableName tableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
646    TableName cloneTableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2");
647
648    ColumnFamilyDescriptor columnFamilyDescriptor =
649      ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("fam1")).build();
650    TableDescriptorBuilder tableDescOne = TableDescriptorBuilder.newBuilder(tableName);
651    tableDescOne.setColumnFamily(columnFamilyDescriptor);
652
653    ADMIN.createTable(tableDescOne.build(), Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
654    String snapshot = "snapshot_testCloneSnapshot";
655    ADMIN.snapshot(snapshot, tableName);
656    ADMIN.cloneSnapshot(snapshot, cloneTableName);
657
658    int tableLength;
659    try (RegionLocator locator = ADMIN.getConnection().getRegionLocator(tableName)) {
660      tableLength = locator.getStartKeys().length;
661    }
662    assertEquals(tableName.getNameAsString() + " should have four regions.", 4, tableLength);
663
664    try (RegionLocator locator = ADMIN.getConnection().getRegionLocator(cloneTableName)) {
665      tableLength = locator.getStartKeys().length;
666    }
667    assertEquals(cloneTableName.getNameAsString() + " should have four regions.", 4, tableLength);
668
669    NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
670    assertEquals("Total tables count should be 2.", 2, nstate.getTables().size());
671    assertEquals("Total regions count should be.", 8, nstate.getRegionCount());
672
673    ADMIN.deleteSnapshot(snapshot);
674  }
675
676  @Test
677  public void testRestoreSnapshot() throws Exception {
678    String nsp = prefix + "_testRestoreSnapshot";
679    NamespaceDescriptor nspDesc = NamespaceDescriptor.create(nsp)
680      .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10").build();
681    ADMIN.createNamespace(nspDesc);
682    assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
683    TableName tableName1 = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
684    ColumnFamilyDescriptor columnFamilyDescriptor =
685      ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("fam1")).build();
686    TableDescriptorBuilder tableDescOne = TableDescriptorBuilder.newBuilder(tableName1);
687    tableDescOne.setColumnFamily(columnFamilyDescriptor);
688    ADMIN.createTable(tableDescOne.build(), Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
689
690    NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
691    assertEquals("Intial region count should be 4.", 4, nstate.getRegionCount());
692
693    String snapshot = "snapshot_testRestoreSnapshot";
694    ADMIN.snapshot(snapshot, tableName1);
695
696    List<RegionInfo> regions = ADMIN.getRegions(tableName1);
697    Collections.sort(regions, RegionInfo.COMPARATOR);
698
699    ADMIN.split(tableName1, Bytes.toBytes("JJJ"));
700    Thread.sleep(2000);
701    assertEquals("Total regions count should be 5.", 5, nstate.getRegionCount());
702
703    ADMIN.disableTable(tableName1);
704    ADMIN.restoreSnapshot(snapshot);
705
706    assertEquals("Total regions count should be 4 after restore.", 4, nstate.getRegionCount());
707
708    ADMIN.enableTable(tableName1);
709    ADMIN.deleteSnapshot(snapshot);
710  }
711
712  @Test
713  public void testRestoreSnapshotQuotaExceed() throws Exception {
714    String nsp = prefix + "_testRestoreSnapshotQuotaExceed";
715    NamespaceDescriptor nspDesc = NamespaceDescriptor.create(nsp)
716      .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10").build();
717    ADMIN.createNamespace(nspDesc);
718    NamespaceDescriptor ndesc = ADMIN.getNamespaceDescriptor(nsp);
719    assertNotNull("Namespace descriptor found null.", ndesc);
720    TableName tableName1 = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
721    ColumnFamilyDescriptor columnFamilyDescriptor =
722      ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("fam1")).build();
723    TableDescriptorBuilder tableDescOne = TableDescriptorBuilder.newBuilder(tableName1);
724    tableDescOne.setColumnFamily(columnFamilyDescriptor);
725
726    ADMIN.createTable(tableDescOne.build(), Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
727
728    NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
729    assertEquals("Intial region count should be 4.", 4, nstate.getRegionCount());
730
731    String snapshot = "snapshot_testRestoreSnapshotQuotaExceed";
732    // snapshot has 4 regions
733    ADMIN.snapshot(snapshot, tableName1);
734    // recreate table with 1 region and set max regions to 3 for namespace
735    ADMIN.disableTable(tableName1);
736    ADMIN.deleteTable(tableName1);
737    ADMIN.createTable(tableDescOne.build());
738    ndesc.setConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "3");
739    ADMIN.modifyNamespace(ndesc);
740
741    ADMIN.disableTable(tableName1);
742    try {
743      ADMIN.restoreSnapshot(snapshot);
744      fail("Region quota is exceeded so QuotaExceededException should be thrown but HBaseAdmin"
745        + " wraps IOException into RestoreSnapshotException");
746    } catch (RestoreSnapshotException ignore) {
747      assertTrue(ignore.getCause() instanceof QuotaExceededException);
748    }
749    assertEquals(1, getNamespaceState(nsp).getRegionCount());
750    ADMIN.enableTable(tableName1);
751    ADMIN.deleteSnapshot(snapshot);
752  }
753}