Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertTrue;
023import static org.junit.Assert.fail;
024
025import java.util.ArrayList;
026import java.util.Arrays;
027import java.util.List;
028import java.util.regex.Pattern;
029import org.apache.hadoop.conf.Configuration;
030import org.apache.hadoop.fs.FileSystem;
031import org.apache.hadoop.fs.Path;
032import org.apache.hadoop.hbase.HBaseClassTestRule;
033import org.apache.hadoop.hbase.HBaseTestingUtility;
034import org.apache.hadoop.hbase.HConstants;
035import org.apache.hadoop.hbase.HTableDescriptor;
036import org.apache.hadoop.hbase.TableName;
037import org.apache.hadoop.hbase.TableNameTestRule;
038import org.apache.hadoop.hbase.TableNotFoundException;
039import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
040import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
041import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
042import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
043import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException;
044import org.apache.hadoop.hbase.snapshot.SnapshotManifestV1;
045import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
046import org.apache.hadoop.hbase.testclassification.ClientTests;
047import org.apache.hadoop.hbase.testclassification.LargeTests;
048import org.apache.hadoop.hbase.util.Bytes;
049import org.apache.hadoop.hbase.util.CommonFSUtils;
050import org.junit.After;
051import org.junit.AfterClass;
052import org.junit.Before;
053import org.junit.BeforeClass;
054import org.junit.ClassRule;
055import org.junit.Rule;
056import org.junit.Test;
057import org.junit.experimental.categories.Category;
058import org.junit.runner.RunWith;
059import org.junit.runners.Parameterized;
060import org.junit.runners.Parameterized.Parameter;
061import org.junit.runners.Parameterized.Parameters;
062import org.slf4j.Logger;
063import org.slf4j.LoggerFactory;
064
065import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
066
067import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
068
069/**
070 * Test create/using/deleting snapshots from the client
071 * <p>
072 * This is an end-to-end test for the snapshot utility
073 */
074@RunWith(Parameterized.class)
075@Category({ LargeTests.class, ClientTests.class })
076public class TestSnapshotFromClient {
077
078  @ClassRule
079  public static final HBaseClassTestRule CLASS_RULE =
080    HBaseClassTestRule.forClass(TestSnapshotFromClient.class);
081
082  private static final Logger LOG = LoggerFactory.getLogger(TestSnapshotFromClient.class);
083
084  protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
085  protected static final int NUM_RS = 2;
086  protected static final String STRING_TABLE_NAME = "test";
087  protected static final byte[] TEST_FAM = Bytes.toBytes("fam");
088  protected static final TableName TABLE_NAME = TableName.valueOf(STRING_TABLE_NAME);
089  private static final Pattern MATCH_ALL = Pattern.compile(".*");
090
091  @Rule
092  public TableNameTestRule name = new TableNameTestRule();
093
094  @Parameter
095  public StoreFileTrackerFactory.Trackers trackerImpl;
096
097  @Parameters(name = "{index}: tracker={0}")
098  public static List<Object[]> params() {
099    return Arrays.asList(new Object[] { StoreFileTrackerFactory.Trackers.DEFAULT },
100      new Object[] { StoreFileTrackerFactory.Trackers.FILE });
101  }
102
103  /**
104   * Setup the config for the cluster
105   * @throws Exception on failure
106   */
107  @BeforeClass
108  public static void setupCluster() throws Exception {
109    setupConf(UTIL.getConfiguration());
110    UTIL.startMiniCluster(NUM_RS);
111  }
112
113  protected static void setupConf(Configuration conf) {
114    // disable the ui
115    conf.setInt("hbase.regionsever.info.port", -1);
116    // change the flush size to a small amount, regulating number of store files
117    conf.setInt("hbase.hregion.memstore.flush.size", 25000);
118    // so make sure we get a compaction when doing a load, but keep around some
119    // files in the store
120    conf.setInt("hbase.hstore.compaction.min", 10);
121    conf.setInt("hbase.hstore.compactionThreshold", 10);
122    // block writes if we get to 12 store files
123    conf.setInt("hbase.hstore.blockingStoreFiles", 12);
124    // Enable snapshot
125    conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
126    conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
127      ConstantSizeRegionSplitPolicy.class.getName());
128  }
129
130  @Before
131  public void setup() throws Exception {
132    createTable();
133  }
134
135  protected void createTable() throws Exception {
136    TableDescriptor htd =
137      TableDescriptorBuilder.newBuilder(TABLE_NAME).setRegionReplication(getNumReplicas())
138        .setValue(StoreFileTrackerFactory.TRACKER_IMPL, trackerImpl.name()).build();
139    UTIL.createTable(htd, new byte[][] { TEST_FAM }, null);
140  }
141
142  protected int getNumReplicas() {
143    return 1;
144  }
145
146  @After
147  public void tearDown() throws Exception {
148    UTIL.deleteTable(TABLE_NAME);
149    SnapshotTestingUtils.deleteAllSnapshots(UTIL.getAdmin());
150    SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
151  }
152
153  @AfterClass
154  public static void cleanupTest() throws Exception {
155    try {
156      UTIL.shutdownMiniCluster();
157    } catch (Exception e) {
158      LOG.warn("failure shutting down cluster", e);
159    }
160  }
161
162  /**
163   * Test snapshotting not allowed hbase:meta and -ROOT-
164   */
165  @Test
166  public void testMetaTablesSnapshot() throws Exception {
167    Admin admin = UTIL.getAdmin();
168    byte[] snapshotName = Bytes.toBytes("metaSnapshot");
169
170    try {
171      admin.snapshot(snapshotName, TableName.META_TABLE_NAME);
172      fail("taking a snapshot of hbase:meta should not be allowed");
173    } catch (IllegalArgumentException e) {
174      // expected
175    }
176  }
177
178  /**
179   * Test HBaseAdmin#deleteSnapshots(String) which deletes snapshots whose names match the parameter
180   */
181  @Test
182  public void testSnapshotDeletionWithRegex() throws Exception {
183    Admin admin = UTIL.getAdmin();
184    // make sure we don't fail on listing snapshots
185    SnapshotTestingUtils.assertNoSnapshots(admin);
186
187    // put some stuff in the table
188    Table table = UTIL.getConnection().getTable(TABLE_NAME);
189    UTIL.loadTable(table, TEST_FAM);
190    table.close();
191
192    byte[] snapshot1 = Bytes.toBytes("TableSnapshot1");
193    admin.snapshot(snapshot1, TABLE_NAME);
194    LOG.debug("Snapshot1 completed.");
195
196    byte[] snapshot2 = Bytes.toBytes("TableSnapshot2");
197    admin.snapshot(snapshot2, TABLE_NAME);
198    LOG.debug("Snapshot2 completed.");
199
200    String snapshot3 = "3rdTableSnapshot";
201    admin.snapshot(Bytes.toBytes(snapshot3), TABLE_NAME);
202    LOG.debug(snapshot3 + " completed.");
203
204    // delete the first two snapshots
205    admin.deleteSnapshots(Pattern.compile("TableSnapshot.*"));
206    List<SnapshotDescription> snapshots = admin.listSnapshots();
207    assertEquals(1, snapshots.size());
208    assertEquals(snapshot3, snapshots.get(0).getName());
209
210    admin.deleteSnapshot(snapshot3);
211    admin.close();
212  }
213
214  /**
215   * Test snapshotting a table that is offline
216   */
217  @Test
218  public void testOfflineTableSnapshot() throws Exception {
219    Admin admin = UTIL.getAdmin();
220    // make sure we don't fail on listing snapshots
221    SnapshotTestingUtils.assertNoSnapshots(admin);
222
223    // put some stuff in the table
224    Table table = UTIL.getConnection().getTable(TABLE_NAME);
225    UTIL.loadTable(table, TEST_FAM, false);
226
227    LOG.debug("FS state before disable:");
228    CommonFSUtils.logFileSystemState(UTIL.getTestFileSystem(),
229      CommonFSUtils.getRootDir(UTIL.getConfiguration()), LOG);
230    // XXX if this is flakey, might want to consider using the async version and looping as
231    // disableTable can succeed and still timeout.
232    admin.disableTable(TABLE_NAME);
233
234    LOG.debug("FS state before snapshot:");
235    CommonFSUtils.logFileSystemState(UTIL.getTestFileSystem(),
236      CommonFSUtils.getRootDir(UTIL.getConfiguration()), LOG);
237
238    // take a snapshot of the disabled table
239    final String SNAPSHOT_NAME = "offlineTableSnapshot";
240    byte[] snapshot = Bytes.toBytes(SNAPSHOT_NAME);
241
242    admin.snapshot(new SnapshotDescription(SNAPSHOT_NAME, TABLE_NAME, SnapshotType.DISABLED, null,
243      -1, SnapshotManifestV1.DESCRIPTOR_VERSION, null));
244    LOG.debug("Snapshot completed.");
245
246    // make sure we have the snapshot
247    List<SnapshotDescription> snapshots =
248      SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot, TABLE_NAME);
249
250    // make sure its a valid snapshot
251    FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
252    Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
253    LOG.debug("FS state after snapshot:");
254    CommonFSUtils.logFileSystemState(UTIL.getTestFileSystem(),
255      CommonFSUtils.getRootDir(UTIL.getConfiguration()), LOG);
256    SnapshotTestingUtils.confirmSnapshotValid(
257      ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, TEST_FAM, rootDir,
258      admin, fs);
259
260    admin.deleteSnapshot(snapshot);
261    snapshots = admin.listSnapshots();
262    SnapshotTestingUtils.assertNoSnapshots(admin);
263  }
264
265  @Test
266  public void testSnapshotFailsOnNonExistantTable() throws Exception {
267    Admin admin = UTIL.getAdmin();
268    // make sure we don't fail on listing snapshots
269    SnapshotTestingUtils.assertNoSnapshots(admin);
270    String tableName = "_not_a_table";
271
272    // make sure the table doesn't exist
273    boolean fail = false;
274    do {
275      try {
276        admin.getTableDescriptor(TableName.valueOf(tableName));
277        fail = true;
278        LOG.error("Table:" + tableName + " already exists, checking a new name");
279        tableName = tableName + "!";
280      } catch (TableNotFoundException e) {
281        fail = false;
282      }
283    } while (fail);
284
285    // snapshot the non-existant table
286    try {
287      admin.snapshot("fail", TableName.valueOf(tableName));
288      fail("Snapshot succeeded even though there is not table.");
289    } catch (SnapshotCreationException e) {
290      LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage());
291    }
292  }
293
294  @Test
295  public void testOfflineTableSnapshotWithEmptyRegions() throws Exception {
296    // test with an empty table with one region
297
298    Admin admin = UTIL.getAdmin();
299    // make sure we don't fail on listing snapshots
300    SnapshotTestingUtils.assertNoSnapshots(admin);
301
302    LOG.debug("FS state before disable:");
303    CommonFSUtils.logFileSystemState(UTIL.getTestFileSystem(),
304      CommonFSUtils.getRootDir(UTIL.getConfiguration()), LOG);
305    admin.disableTable(TABLE_NAME);
306
307    LOG.debug("FS state before snapshot:");
308    CommonFSUtils.logFileSystemState(UTIL.getTestFileSystem(),
309      CommonFSUtils.getRootDir(UTIL.getConfiguration()), LOG);
310
311    // take a snapshot of the disabled table
312    byte[] snapshot = Bytes.toBytes("testOfflineTableSnapshotWithEmptyRegions");
313    admin.snapshot(snapshot, TABLE_NAME);
314    LOG.debug("Snapshot completed.");
315
316    // make sure we have the snapshot
317    List<SnapshotDescription> snapshots =
318      SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot, TABLE_NAME);
319
320    // make sure its a valid snapshot
321    FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
322    Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
323    LOG.debug("FS state after snapshot:");
324    CommonFSUtils.logFileSystemState(UTIL.getTestFileSystem(),
325      CommonFSUtils.getRootDir(UTIL.getConfiguration()), LOG);
326
327    List<byte[]> emptyCfs = Lists.newArrayList(TEST_FAM); // no file in the region
328    List<byte[]> nonEmptyCfs = Lists.newArrayList();
329    SnapshotTestingUtils.confirmSnapshotValid(
330      ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, nonEmptyCfs,
331      emptyCfs, rootDir, admin, fs);
332
333    admin.deleteSnapshot(snapshot);
334    snapshots = admin.listSnapshots();
335    SnapshotTestingUtils.assertNoSnapshots(admin);
336  }
337
338  @Test
339  public void testListTableSnapshots() throws Exception {
340    Admin admin = null;
341    final TableName tableName = name.getTableName();
342    try {
343      admin = UTIL.getAdmin();
344
345      HTableDescriptor htd = new HTableDescriptor(tableName);
346      UTIL.createTable(htd, new byte[][] { TEST_FAM }, UTIL.getConfiguration());
347
348      String table1Snapshot1 = "Table1Snapshot1";
349      admin.snapshot(table1Snapshot1, TABLE_NAME);
350      LOG.debug("Snapshot1 completed.");
351
352      String table1Snapshot2 = "Table1Snapshot2";
353      admin.snapshot(table1Snapshot2, TABLE_NAME);
354      LOG.debug("Snapshot2 completed.");
355
356      String table2Snapshot1 = "Table2Snapshot1";
357      admin.snapshot(Bytes.toBytes(table2Snapshot1), tableName);
358      LOG.debug(table2Snapshot1 + " completed.");
359
360      List<SnapshotDescription> listTableSnapshots =
361        admin.listTableSnapshots(Pattern.compile("test.*"), MATCH_ALL);
362      List<String> listTableSnapshotNames = new ArrayList<>();
363      assertEquals(3, listTableSnapshots.size());
364      for (SnapshotDescription s : listTableSnapshots) {
365        listTableSnapshotNames.add(s.getName());
366      }
367      assertTrue(listTableSnapshotNames.contains(table1Snapshot1));
368      assertTrue(listTableSnapshotNames.contains(table1Snapshot2));
369      assertTrue(listTableSnapshotNames.contains(table2Snapshot1));
370    } finally {
371      if (admin != null) {
372        try {
373          admin.deleteSnapshots(Pattern.compile("Table.*"));
374        } catch (SnapshotDoesNotExistException ignore) {
375        }
376        if (admin.tableExists(tableName)) {
377          UTIL.deleteTable(tableName);
378        }
379        admin.close();
380      }
381    }
382  }
383
384  @Test
385  public void testListTableSnapshotsWithRegex() throws Exception {
386    Admin admin = null;
387    try {
388      admin = UTIL.getAdmin();
389
390      String table1Snapshot1 = "Table1Snapshot1";
391      admin.snapshot(table1Snapshot1, TABLE_NAME);
392      LOG.debug("Snapshot1 completed.");
393
394      String table1Snapshot2 = "Table1Snapshot2";
395      admin.snapshot(table1Snapshot2, TABLE_NAME);
396      LOG.debug("Snapshot2 completed.");
397
398      String table2Snapshot1 = "Table2Snapshot1";
399      admin.snapshot(Bytes.toBytes(table2Snapshot1), TABLE_NAME);
400      LOG.debug(table2Snapshot1 + " completed.");
401
402      List<SnapshotDescription> listTableSnapshots =
403        admin.listTableSnapshots(Pattern.compile("test.*"), Pattern.compile("Table1.*"));
404      List<String> listTableSnapshotNames = new ArrayList<>();
405      assertEquals(2, listTableSnapshots.size());
406      for (SnapshotDescription s : listTableSnapshots) {
407        listTableSnapshotNames.add(s.getName());
408      }
409      assertTrue(listTableSnapshotNames.contains(table1Snapshot1));
410      assertTrue(listTableSnapshotNames.contains(table1Snapshot2));
411      assertFalse(listTableSnapshotNames.contains(table2Snapshot1));
412    } finally {
413      if (admin != null) {
414        try {
415          admin.deleteSnapshots(Pattern.compile("Table.*"));
416        } catch (SnapshotDoesNotExistException ignore) {
417        }
418        admin.close();
419      }
420    }
421  }
422
423  @Test
424  public void testDeleteTableSnapshots() throws Exception {
425    Admin admin = null;
426    final TableName tableName = name.getTableName();
427    try {
428      admin = UTIL.getAdmin();
429
430      HTableDescriptor htd = new HTableDescriptor(tableName);
431      UTIL.createTable(htd, new byte[][] { TEST_FAM }, UTIL.getConfiguration());
432
433      String table1Snapshot1 = "Table1Snapshot1";
434      admin.snapshot(table1Snapshot1, TABLE_NAME);
435      LOG.debug("Snapshot1 completed.");
436
437      String table1Snapshot2 = "Table1Snapshot2";
438      admin.snapshot(table1Snapshot2, TABLE_NAME);
439      LOG.debug("Snapshot2 completed.");
440
441      String table2Snapshot1 = "Table2Snapshot1";
442      admin.snapshot(Bytes.toBytes(table2Snapshot1), tableName);
443      LOG.debug(table2Snapshot1 + " completed.");
444
445      Pattern tableNamePattern = Pattern.compile("test.*");
446      admin.deleteTableSnapshots(tableNamePattern, MATCH_ALL);
447      assertEquals(0, admin.listTableSnapshots(tableNamePattern, MATCH_ALL).size());
448    } finally {
449      if (admin != null) {
450        if (admin.tableExists(tableName)) {
451          UTIL.deleteTable(tableName);
452        }
453        admin.close();
454      }
455    }
456  }
457
458  @Test
459  public void testDeleteTableSnapshotsWithRegex() throws Exception {
460    Admin admin = null;
461    Pattern tableNamePattern = Pattern.compile("test.*");
462    try {
463      admin = UTIL.getAdmin();
464
465      String table1Snapshot1 = "Table1Snapshot1";
466      admin.snapshot(table1Snapshot1, TABLE_NAME);
467      LOG.debug("Snapshot1 completed.");
468
469      String table1Snapshot2 = "Table1Snapshot2";
470      admin.snapshot(table1Snapshot2, TABLE_NAME);
471      LOG.debug("Snapshot2 completed.");
472
473      String table2Snapshot1 = "Table2Snapshot1";
474      admin.snapshot(Bytes.toBytes(table2Snapshot1), TABLE_NAME);
475      LOG.debug(table2Snapshot1 + " completed.");
476
477      admin.deleteTableSnapshots(tableNamePattern, Pattern.compile("Table1.*"));
478      assertEquals(1, admin.listTableSnapshots(tableNamePattern, MATCH_ALL).size());
479    } finally {
480      if (admin != null) {
481        try {
482          admin.deleteTableSnapshots(tableNamePattern, MATCH_ALL);
483        } catch (SnapshotDoesNotExistException ignore) {
484        }
485        admin.close();
486      }
487    }
488  }
489}