Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.snapshot;
019
020import static org.apache.hadoop.util.ToolRunner.run;
021import static org.junit.Assert.assertEquals;
022import static org.junit.Assert.assertFalse;
023import static org.junit.Assert.assertTrue;
024
025import java.io.IOException;
026import java.util.ArrayList;
027import java.util.HashMap;
028import java.util.HashSet;
029import java.util.List;
030import java.util.Map;
031import java.util.Objects;
032import java.util.Optional;
033import java.util.Set;
034import java.util.stream.Collectors;
035import org.apache.hadoop.conf.Configuration;
036import org.apache.hadoop.fs.FileStatus;
037import org.apache.hadoop.fs.FileSystem;
038import org.apache.hadoop.fs.Path;
039import org.apache.hadoop.hbase.HBaseClassTestRule;
040import org.apache.hadoop.hbase.HBaseTestingUtil;
041import org.apache.hadoop.hbase.HConstants;
042import org.apache.hadoop.hbase.TableName;
043import org.apache.hadoop.hbase.client.Admin;
044import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
045import org.apache.hadoop.hbase.client.Put;
046import org.apache.hadoop.hbase.client.RegionInfo;
047import org.apache.hadoop.hbase.client.SnapshotType;
048import org.apache.hadoop.hbase.client.Table;
049import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
050import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
051import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
052import org.apache.hadoop.hbase.testclassification.LargeTests;
053import org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests;
054import org.apache.hadoop.hbase.util.AbstractHBaseTool;
055import org.apache.hadoop.hbase.util.Bytes;
056import org.apache.hadoop.hbase.util.CommonFSUtils;
057import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
058import org.apache.hadoop.hbase.util.Pair;
059import org.junit.After;
060import org.junit.AfterClass;
061import org.junit.Before;
062import org.junit.BeforeClass;
063import org.junit.ClassRule;
064import org.junit.Ignore;
065import org.junit.Rule;
066import org.junit.Test;
067import org.junit.experimental.categories.Category;
068import org.junit.rules.TestName;
069import org.slf4j.Logger;
070import org.slf4j.LoggerFactory;
071
072import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
073
074import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription;
075import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
076
077/**
078 * Test Export Snapshot Tool
079 */
080@Ignore // HBASE-24493
081@Category({ VerySlowMapReduceTests.class, LargeTests.class })
082public class TestExportSnapshot {
083
084  @ClassRule
085  public static final HBaseClassTestRule CLASS_RULE =
086    HBaseClassTestRule.forClass(TestExportSnapshot.class);
087
088  private static final Logger LOG = LoggerFactory.getLogger(TestExportSnapshot.class);
089
090  protected final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
091
092  protected final static byte[] FAMILY = Bytes.toBytes("cf");
093
094  @Rule
095  public final TestName testName = new TestName();
096
097  protected TableName tableName;
098  private String emptySnapshotName;
099  private String snapshotName;
100  private int tableNumFiles;
101  private Admin admin;
102
103  public static void setUpBaseConf(Configuration conf) {
104    conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
105    conf.setInt("hbase.regionserver.msginterval", 100);
106    // If a single node has enough failures (default 3), resource manager will blacklist it.
107    // With only 2 nodes and tests injecting faults, we don't want that.
108    conf.setInt("mapreduce.job.maxtaskfailures.per.tracker", 100);
109  }
110
111  @BeforeClass
112  public static void setUpBeforeClass() throws Exception {
113    setUpBaseConf(TEST_UTIL.getConfiguration());
114    TEST_UTIL.startMiniCluster(1);
115    TEST_UTIL.startMiniMapReduceCluster();
116  }
117
118  @AfterClass
119  public static void tearDownAfterClass() throws Exception {
120    TEST_UTIL.shutdownMiniMapReduceCluster();
121    TEST_UTIL.shutdownMiniCluster();
122  }
123
124  /**
125   * Create a table and take a snapshot of the table used by the export test.
126   */
127  @Before
128  public void setUp() throws Exception {
129    this.admin = TEST_UTIL.getAdmin();
130
131    tableName = TableName.valueOf("testtb-" + testName.getMethodName());
132    snapshotName = "snaptb0-" + testName.getMethodName();
133    emptySnapshotName = "emptySnaptb0-" + testName.getMethodName();
134
135    // create Table
136    createTable(this.tableName);
137
138    // Take an empty snapshot
139    admin.snapshot(emptySnapshotName, tableName);
140
141    // Add some rows
142    SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 50, FAMILY);
143    tableNumFiles = admin.getRegions(tableName).size();
144
145    // take a snapshot
146    admin.snapshot(snapshotName, tableName);
147  }
148
149  protected void createTable(TableName tableName) throws Exception {
150    SnapshotTestingUtils.createPreSplitTable(TEST_UTIL, tableName, 2, FAMILY);
151  }
152
153  protected interface RegionPredicate {
154    boolean evaluate(final RegionInfo regionInfo);
155  }
156
157  protected RegionPredicate getBypassRegionPredicate() {
158    return null;
159  }
160
161  @After
162  public void tearDown() throws Exception {
163    TEST_UTIL.deleteTable(tableName);
164    SnapshotTestingUtils.deleteAllSnapshots(TEST_UTIL.getAdmin());
165    SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL);
166  }
167
168  /**
169   * Verify if exported snapshot and copied files matches the original one.
170   */
171  @Test
172  public void testExportFileSystemState() throws Exception {
173    testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles);
174  }
175
176  @Test
177  public void testExportFileSystemStateWithMergeRegion() throws Exception {
178    // disable compaction
179    admin.compactionSwitch(false,
180      admin.getRegionServers().stream().map(a -> a.getServerName()).collect(Collectors.toList()));
181    // create Table
182    TableName tableName0 = TableName.valueOf("testtb-" + testName.getMethodName() + "-1");
183    String snapshotName0 = "snaptb0-" + testName.getMethodName() + "-1";
184    admin.createTable(
185      TableDescriptorBuilder.newBuilder(tableName0)
186        .setColumnFamilies(
187          Lists.newArrayList(ColumnFamilyDescriptorBuilder.newBuilder(FAMILY).build()))
188        .build(),
189      new byte[][] { Bytes.toBytes("2") });
190    // put some data
191    try (Table table = admin.getConnection().getTable(tableName0)) {
192      table.put(new Put(Bytes.toBytes("1")).addColumn(FAMILY, null, Bytes.toBytes("1")));
193      table.put(new Put(Bytes.toBytes("2")).addColumn(FAMILY, null, Bytes.toBytes("2")));
194    }
195    List<RegionInfo> regions = admin.getRegions(tableName0);
196    assertEquals(2, regions.size());
197    tableNumFiles = regions.size();
198    // merge region
199    admin.mergeRegionsAsync(new byte[][] { regions.get(0).getEncodedNameAsBytes(),
200      regions.get(1).getEncodedNameAsBytes() }, true).get();
201    // take a snapshot
202    admin.snapshot(snapshotName0, tableName0);
203    // export snapshot and verify
204    testExportFileSystemState(tableName0, snapshotName0, snapshotName0, tableNumFiles);
205    // delete table
206    TEST_UTIL.deleteTable(tableName0);
207  }
208
209  @Test
210  public void testExportFileSystemStateWithSkipTmp() throws Exception {
211    TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, true);
212    try {
213      testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles);
214    } finally {
215      TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, false);
216    }
217  }
218
219  @Test
220  public void testEmptyExportFileSystemState() throws Exception {
221    testExportFileSystemState(tableName, emptySnapshotName, emptySnapshotName, 0);
222  }
223
224  @Test
225  public void testConsecutiveExports() throws Exception {
226    Path copyDir = getLocalDestinationDir(TEST_UTIL);
227    testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles, copyDir, false);
228    testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles, copyDir, true);
229    removeExportDir(copyDir);
230  }
231
232  @Test
233  public void testExportWithChecksum() throws Exception {
234    // Test different schemes: input scheme is hdfs:// and output scheme is file://
235    // The checksum verification will fail
236    Path copyLocalDir = getLocalDestinationDir(TEST_UTIL);
237    testExportFileSystemState(TEST_UTIL.getConfiguration(), tableName, snapshotName, snapshotName,
238      tableNumFiles, TEST_UTIL.getDefaultRootDirPath(), copyLocalDir, false, false,
239      getBypassRegionPredicate(), false, true);
240
241    // Test same schemes: input scheme is hdfs:// and output scheme is hdfs://
242    // The checksum verification will success
243    Path copyHdfsDir = getHdfsDestinationDir();
244    testExportFileSystemState(TEST_UTIL.getConfiguration(), tableName, snapshotName, snapshotName,
245      tableNumFiles, TEST_UTIL.getDefaultRootDirPath(), copyHdfsDir, false, false,
246      getBypassRegionPredicate(), true, true);
247  }
248
249  @Test
250  public void testExportWithTargetName() throws Exception {
251    final String targetName = "testExportWithTargetName";
252    testExportFileSystemState(tableName, snapshotName, targetName, tableNumFiles);
253  }
254
255  @Test
256  public void testExportWithResetTtl() throws Exception {
257    String name = "testExportWithResetTtl";
258    TableName tableName = TableName.valueOf(name);
259    String snapshotName = "snaptb-" + name;
260    Long ttl = 100000L;
261
262    try {
263      // create Table
264      createTable(tableName);
265      SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 50, FAMILY);
266      int tableNumFiles = admin.getRegions(tableName).size();
267      // take a snapshot with TTL
268      Map<String, Object> props = new HashMap<>();
269      props.put("TTL", ttl);
270      admin.snapshot(snapshotName, tableName, props);
271      Optional<Long> ttlOpt =
272        admin.listSnapshots().stream().filter(s -> s.getName().equals(snapshotName))
273          .map(org.apache.hadoop.hbase.client.SnapshotDescription::getTtl).findAny();
274      assertTrue(ttlOpt.isPresent());
275      assertEquals(ttl, ttlOpt.get());
276
277      testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles,
278        getHdfsDestinationDir(), false, true);
279    } catch (Exception e) {
280      throw e;
281    } finally {
282      TEST_UTIL.deleteTable(tableName);
283    }
284  }
285
286  @Test
287  public void testExportExpiredSnapshot() throws Exception {
288    String name = "testExportExpiredSnapshot";
289    TableName tableName = TableName.valueOf(name);
290    String snapshotName = "snapshot-" + name;
291    createTable(tableName);
292    SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 50, FAMILY);
293    Map<String, Object> properties = new HashMap<>();
294    properties.put("TTL", 10);
295    org.apache.hadoop.hbase.client.SnapshotDescription snapshotDescription =
296      new org.apache.hadoop.hbase.client.SnapshotDescription(snapshotName, tableName,
297        SnapshotType.FLUSH, null, EnvironmentEdgeManager.currentTime(), -1, properties);
298    admin.snapshot(snapshotDescription);
299    boolean isExist =
300      admin.listSnapshots().stream().anyMatch(ele -> snapshotName.equals(ele.getName()));
301    assertTrue(isExist);
302    int retry = 6;
303    while (
304      !SnapshotDescriptionUtils.isExpiredSnapshot(snapshotDescription.getTtl(),
305        snapshotDescription.getCreationTime(), EnvironmentEdgeManager.currentTime()) && retry > 0
306    ) {
307      retry--;
308      Thread.sleep(10 * 1000);
309    }
310    boolean isExpiredSnapshot =
311      SnapshotDescriptionUtils.isExpiredSnapshot(snapshotDescription.getTtl(),
312        snapshotDescription.getCreationTime(), EnvironmentEdgeManager.currentTime());
313    assertTrue(isExpiredSnapshot);
314    int res = runExportSnapshot(TEST_UTIL.getConfiguration(), snapshotName, snapshotName,
315      TEST_UTIL.getDefaultRootDirPath(), getHdfsDestinationDir(), false, false, false, true, true);
316    assertTrue(res == AbstractHBaseTool.EXIT_FAILURE);
317  }
318
319  private void testExportFileSystemState(final TableName tableName, final String snapshotName,
320    final String targetName, int filesExpected) throws Exception {
321    testExportFileSystemState(tableName, snapshotName, targetName, filesExpected,
322      getHdfsDestinationDir(), false);
323  }
324
325  protected void testExportFileSystemState(final TableName tableName, final String snapshotName,
326    final String targetName, int filesExpected, Path copyDir, boolean overwrite) throws Exception {
327    testExportFileSystemState(tableName, snapshotName, targetName, filesExpected, copyDir,
328      overwrite, false);
329  }
330
331  protected void testExportFileSystemState(final TableName tableName, final String snapshotName,
332    final String targetName, int filesExpected, Path copyDir, boolean overwrite, boolean resetTtl)
333    throws Exception {
334    testExportFileSystemState(TEST_UTIL.getConfiguration(), tableName, snapshotName, targetName,
335      filesExpected, TEST_UTIL.getDefaultRootDirPath(), copyDir, overwrite, resetTtl,
336      getBypassRegionPredicate(), true, false);
337  }
338
339  /**
340   * Creates destination directory, runs ExportSnapshot() tool, and runs some verifications.
341   */
342  protected static void testExportFileSystemState(final Configuration conf,
343    final TableName tableName, final String snapshotName, final String targetName,
344    final int filesExpected, final Path srcDir, Path rawTgtDir, final boolean overwrite,
345    final boolean resetTtl, final RegionPredicate bypassregionPredicate, final boolean success,
346    final boolean checksumVerify) throws Exception {
347    FileSystem tgtFs = rawTgtDir.getFileSystem(conf);
348    FileSystem srcFs = srcDir.getFileSystem(conf);
349    Path tgtDir = rawTgtDir.makeQualified(tgtFs.getUri(), tgtFs.getWorkingDirectory());
350
351    // Export Snapshot
352    int res = runExportSnapshot(conf, snapshotName, targetName, srcDir, rawTgtDir, overwrite,
353      resetTtl, checksumVerify, true, true);
354    assertEquals("success " + success + ", res=" + res, success ? 0 : 1, res);
355    if (!success) {
356      final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, targetName);
357      assertFalse(tgtDir.toString() + " " + targetDir.toString(),
358        tgtFs.exists(new Path(tgtDir, targetDir)));
359      return;
360    }
361    LOG.info("Exported snapshot");
362
363    // Verify File-System state
364    FileStatus[] rootFiles = tgtFs.listStatus(tgtDir);
365    assertEquals(filesExpected > 0 ? 2 : 1, rootFiles.length);
366    for (FileStatus fileStatus : rootFiles) {
367      String name = fileStatus.getPath().getName();
368      assertTrue(fileStatus.toString(), fileStatus.isDirectory());
369      assertTrue(name.toString(), name.equals(HConstants.SNAPSHOT_DIR_NAME)
370        || name.equals(HConstants.HFILE_ARCHIVE_DIRECTORY));
371    }
372    LOG.info("Verified filesystem state");
373
374    // Compare the snapshot metadata and verify the hfiles
375    final Path snapshotDir = new Path(HConstants.SNAPSHOT_DIR_NAME, snapshotName);
376    final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, targetName);
377    verifySnapshotDir(srcFs, new Path(srcDir, snapshotDir), tgtFs, new Path(tgtDir, targetDir));
378    Set<String> snapshotFiles =
379      verifySnapshot(conf, tgtFs, tgtDir, tableName, targetName, resetTtl, bypassregionPredicate);
380    assertEquals(filesExpected, snapshotFiles.size());
381  }
382
383  /*
384   * verify if the snapshot folder on file-system 1 match the one on file-system 2
385   */
386  protected static void verifySnapshotDir(final FileSystem fs1, final Path root1,
387    final FileSystem fs2, final Path root2) throws IOException {
388    assertEquals(listFiles(fs1, root1, root1), listFiles(fs2, root2, root2));
389  }
390
391  /*
392   * Verify if the files exists
393   */
394  protected static Set<String> verifySnapshot(final Configuration conf, final FileSystem fs,
395    final Path rootDir, final TableName tableName, final String snapshotName,
396    final boolean resetTtl, final RegionPredicate bypassregionPredicate) throws IOException {
397    final Path exportedSnapshot =
398      new Path(rootDir, new Path(HConstants.SNAPSHOT_DIR_NAME, snapshotName));
399    final Set<String> snapshotFiles = new HashSet<>();
400    final Path exportedArchive = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
401    SnapshotReferenceUtil.visitReferencedFiles(conf, fs, exportedSnapshot,
402      new SnapshotReferenceUtil.SnapshotVisitor() {
403        @Override
404        public void storeFile(final RegionInfo regionInfo, final String family,
405          final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
406          if (bypassregionPredicate != null && bypassregionPredicate.evaluate(regionInfo)) {
407            return;
408          }
409
410          if (!storeFile.hasReference() && !StoreFileInfo.isReference(storeFile.getName())) {
411            String hfile = storeFile.getName();
412            snapshotFiles.add(hfile);
413            verifyNonEmptyFile(new Path(exportedArchive,
414              new Path(CommonFSUtils.getTableDir(new Path("./"), tableName),
415                new Path(regionInfo.getEncodedName(), new Path(family, hfile)))));
416          } else {
417            Pair<String, String> referredToRegionAndFile =
418              StoreFileInfo.getReferredToRegionAndFile(storeFile.getName());
419            String region = referredToRegionAndFile.getFirst();
420            String hfile = referredToRegionAndFile.getSecond();
421            snapshotFiles.add(hfile);
422            verifyNonEmptyFile(new Path(exportedArchive,
423              new Path(CommonFSUtils.getTableDir(new Path("./"), tableName),
424                new Path(region, new Path(family, hfile)))));
425          }
426        }
427
428        private void verifyNonEmptyFile(final Path path) throws IOException {
429          assertTrue(path + " should exists", fs.exists(path));
430          assertTrue(path + " should not be empty", fs.getFileStatus(path).getLen() > 0);
431        }
432      });
433
434    // Verify Snapshot description
435    SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, exportedSnapshot);
436    assertTrue(desc.getName().equals(snapshotName));
437    assertTrue(desc.getTable().equals(tableName.getNameAsString()));
438    if (resetTtl) {
439      assertEquals(HConstants.DEFAULT_SNAPSHOT_TTL, desc.getTtl());
440    }
441    return snapshotFiles;
442  }
443
444  private static Set<String> listFiles(final FileSystem fs, final Path root, final Path dir)
445    throws IOException {
446    Set<String> files = new HashSet<>();
447    LOG.debug("List files in {} in root {} at {}", fs, root, dir);
448    int rootPrefix = root.makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString().length();
449    FileStatus[] list = CommonFSUtils.listStatus(fs, dir);
450    if (list != null) {
451      for (FileStatus fstat : list) {
452        LOG.debug(Objects.toString(fstat.getPath()));
453        if (fstat.isDirectory()) {
454          files.addAll(listFiles(fs, root, fstat.getPath()));
455        } else {
456          files.add(fstat.getPath().makeQualified(fs).toString().substring(rootPrefix));
457        }
458      }
459    }
460    return files;
461  }
462
463  private Path getHdfsDestinationDir() {
464    Path rootDir = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
465    Path path =
466      new Path(new Path(rootDir, "export-test"), "export-" + EnvironmentEdgeManager.currentTime());
467    LOG.info("HDFS export destination path: " + path);
468    return path;
469  }
470
471  static Path getLocalDestinationDir(HBaseTestingUtil htu) {
472    Path path = htu.getDataTestDir("local-export-" + EnvironmentEdgeManager.currentTime());
473    try {
474      FileSystem fs = FileSystem.getLocal(htu.getConfiguration());
475      LOG.info("Local export destination path: " + path);
476      return path.makeQualified(fs.getUri(), fs.getWorkingDirectory());
477    } catch (IOException ioe) {
478      throw new RuntimeException(ioe);
479    }
480  }
481
482  private static void removeExportDir(final Path path) throws IOException {
483    FileSystem fs = FileSystem.get(path.toUri(), new Configuration());
484    fs.delete(path, true);
485  }
486
487  private static int runExportSnapshot(final Configuration conf, final String sourceSnapshotName,
488    final String targetSnapshotName, final Path srcDir, Path rawTgtDir, final boolean overwrite,
489    final boolean resetTtl, final boolean checksumVerify, final boolean noSourceVerify,
490    final boolean noTargetVerify) throws Exception {
491    FileSystem tgtFs = rawTgtDir.getFileSystem(conf);
492    FileSystem srcFs = srcDir.getFileSystem(conf);
493    Path tgtDir = rawTgtDir.makeQualified(tgtFs.getUri(), tgtFs.getWorkingDirectory());
494    LOG.info("tgtFsUri={}, tgtDir={}, rawTgtDir={}, srcFsUri={}, srcDir={}", tgtFs.getUri(), tgtDir,
495      rawTgtDir, srcFs.getUri(), srcDir);
496    List<String> opts = new ArrayList<>();
497    opts.add("--snapshot");
498    opts.add(sourceSnapshotName);
499    opts.add("--copy-to");
500    opts.add(tgtDir.toString());
501    if (!targetSnapshotName.equals(sourceSnapshotName)) {
502      opts.add("--target");
503      opts.add(targetSnapshotName);
504    }
505    if (overwrite) {
506      opts.add("--overwrite");
507    }
508    if (resetTtl) {
509      opts.add("--reset-ttl");
510    }
511    if (!checksumVerify) {
512      opts.add("--no-checksum-verify");
513    }
514    if (!noSourceVerify) {
515      opts.add("--no-source-verify");
516    }
517    if (!noTargetVerify) {
518      opts.add("--no-target-verify");
519    }
520
521    // Export Snapshot
522    return run(conf, new ExportSnapshot(), opts.toArray(new String[opts.size()]));
523  }
524}