Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.snapshot;
019
020import static org.apache.hadoop.util.ToolRunner.run;
021import static org.junit.Assert.assertEquals;
022import static org.junit.Assert.assertFalse;
023import static org.junit.Assert.assertTrue;
024
025import java.io.IOException;
026import java.util.ArrayList;
027import java.util.HashMap;
028import java.util.HashSet;
029import java.util.List;
030import java.util.Map;
031import java.util.Objects;
032import java.util.Optional;
033import java.util.Set;
034import java.util.stream.Collectors;
035import org.apache.hadoop.conf.Configuration;
036import org.apache.hadoop.fs.FileStatus;
037import org.apache.hadoop.fs.FileSystem;
038import org.apache.hadoop.fs.Path;
039import org.apache.hadoop.hbase.HBaseClassTestRule;
040import org.apache.hadoop.hbase.HBaseTestingUtility;
041import org.apache.hadoop.hbase.HConstants;
042import org.apache.hadoop.hbase.TableName;
043import org.apache.hadoop.hbase.client.Admin;
044import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
045import org.apache.hadoop.hbase.client.Put;
046import org.apache.hadoop.hbase.client.RegionInfo;
047import org.apache.hadoop.hbase.client.Table;
048import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
049import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
050import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
051import org.apache.hadoop.hbase.testclassification.LargeTests;
052import org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests;
053import org.apache.hadoop.hbase.util.Bytes;
054import org.apache.hadoop.hbase.util.CommonFSUtils;
055import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
056import org.apache.hadoop.hbase.util.Pair;
057import org.junit.After;
058import org.junit.AfterClass;
059import org.junit.Before;
060import org.junit.BeforeClass;
061import org.junit.ClassRule;
062import org.junit.Ignore;
063import org.junit.Rule;
064import org.junit.Test;
065import org.junit.experimental.categories.Category;
066import org.junit.rules.TestName;
067import org.slf4j.Logger;
068import org.slf4j.LoggerFactory;
069
070import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
071
072import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription;
073import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
074
075/**
076 * Test Export Snapshot Tool
077 */
078@Ignore // HBASE-24493
079@Category({ VerySlowMapReduceTests.class, LargeTests.class })
080public class TestExportSnapshot {
081
082  @ClassRule
083  public static final HBaseClassTestRule CLASS_RULE =
084    HBaseClassTestRule.forClass(TestExportSnapshot.class);
085
086  private static final Logger LOG = LoggerFactory.getLogger(TestExportSnapshot.class);
087
088  protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
089
090  protected final static byte[] FAMILY = Bytes.toBytes("cf");
091
092  @Rule
093  public final TestName testName = new TestName();
094
095  protected TableName tableName;
096  private byte[] emptySnapshotName;
097  private byte[] snapshotName;
098  private int tableNumFiles;
099  private Admin admin;
100
101  public static void setUpBaseConf(Configuration conf) {
102    conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
103    conf.setInt("hbase.regionserver.msginterval", 100);
104    // If a single node has enough failures (default 3), resource manager will blacklist it.
105    // With only 2 nodes and tests injecting faults, we don't want that.
106    conf.setInt("mapreduce.job.maxtaskfailures.per.tracker", 100);
107  }
108
109  @BeforeClass
110  public static void setUpBeforeClass() throws Exception {
111    setUpBaseConf(TEST_UTIL.getConfiguration());
112    TEST_UTIL.startMiniCluster(1);
113    TEST_UTIL.startMiniMapReduceCluster();
114  }
115
116  @AfterClass
117  public static void tearDownAfterClass() throws Exception {
118    TEST_UTIL.shutdownMiniMapReduceCluster();
119    TEST_UTIL.shutdownMiniCluster();
120  }
121
122  /**
123   * Create a table and take a snapshot of the table used by the export test.
124   */
125  @Before
126  public void setUp() throws Exception {
127    this.admin = TEST_UTIL.getAdmin();
128
129    tableName = TableName.valueOf("testtb-" + testName.getMethodName());
130    snapshotName = Bytes.toBytes("snaptb0-" + testName.getMethodName());
131    emptySnapshotName = Bytes.toBytes("emptySnaptb0-" + testName.getMethodName());
132
133    // create Table
134    createTable(this.tableName);
135
136    // Take an empty snapshot
137    admin.snapshot(emptySnapshotName, tableName);
138
139    // Add some rows
140    SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 50, FAMILY);
141    tableNumFiles = admin.getTableRegions(tableName).size();
142
143    // take a snapshot
144    admin.snapshot(snapshotName, tableName);
145  }
146
147  protected void createTable(TableName tableName) throws Exception {
148    SnapshotTestingUtils.createPreSplitTable(TEST_UTIL, tableName, 2, FAMILY);
149  }
150
151  protected interface RegionPredicate {
152    boolean evaluate(final RegionInfo regionInfo);
153  }
154
155  protected RegionPredicate getBypassRegionPredicate() {
156    return null;
157  }
158
159  @After
160  public void tearDown() throws Exception {
161    TEST_UTIL.deleteTable(tableName);
162    SnapshotTestingUtils.deleteAllSnapshots(TEST_UTIL.getAdmin());
163    SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL);
164  }
165
166  /**
167   * Verify if exported snapshot and copied files matches the original one.
168   */
169  @Test
170  public void testExportFileSystemState() throws Exception {
171    testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles);
172  }
173
174  @Test
175  public void testExportFileSystemStateWithMergeRegion() throws Exception {
176    // disable compaction
177    admin.compactionSwitch(false,
178      admin.getRegionServers().stream().map(a -> a.getServerName()).collect(Collectors.toList()));
179    // create Table
180    TableName tableName0 = TableName.valueOf("testtb-" + testName.getMethodName() + "-1");
181    byte[] snapshotName0 = Bytes.toBytes("snaptb0-" + testName.getMethodName() + "-1");
182    admin.createTable(
183      TableDescriptorBuilder.newBuilder(tableName0)
184        .setColumnFamilies(
185          Lists.newArrayList(ColumnFamilyDescriptorBuilder.newBuilder(FAMILY).build()))
186        .build(),
187      new byte[][] { Bytes.toBytes("2") });
188    // put some data
189    try (Table table = admin.getConnection().getTable(tableName0)) {
190      table.put(new Put(Bytes.toBytes("1")).addColumn(FAMILY, null, Bytes.toBytes("1")));
191      table.put(new Put(Bytes.toBytes("2")).addColumn(FAMILY, null, Bytes.toBytes("2")));
192    }
193    List<RegionInfo> regions = admin.getRegions(tableName0);
194    assertEquals(2, regions.size());
195    tableNumFiles = regions.size();
196    // merge region
197    admin.mergeRegionsAsync(new byte[][] { regions.get(0).getEncodedNameAsBytes(),
198      regions.get(1).getEncodedNameAsBytes() }, true).get();
199    // take a snapshot
200    admin.snapshot(snapshotName0, tableName0);
201    // export snapshot and verify
202    testExportFileSystemState(tableName0, snapshotName0, snapshotName0, tableNumFiles);
203    // delete table
204    TEST_UTIL.deleteTable(tableName0);
205  }
206
207  @Test
208  public void testExportFileSystemStateWithSkipTmp() throws Exception {
209    TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, true);
210    try {
211      testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles);
212    } finally {
213      TEST_UTIL.getConfiguration().setBoolean(ExportSnapshot.CONF_SKIP_TMP, false);
214    }
215  }
216
217  @Test
218  public void testEmptyExportFileSystemState() throws Exception {
219    testExportFileSystemState(tableName, emptySnapshotName, emptySnapshotName, 0);
220  }
221
222  @Test
223  public void testConsecutiveExports() throws Exception {
224    Path copyDir = getLocalDestinationDir(TEST_UTIL);
225    testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles, copyDir, false);
226    testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles, copyDir, true);
227    removeExportDir(copyDir);
228  }
229
230  @Test
231  public void testExportWithTargetName() throws Exception {
232    final byte[] targetName = Bytes.toBytes("testExportWithTargetName");
233    testExportFileSystemState(tableName, snapshotName, targetName, tableNumFiles);
234  }
235
236  @Test
237  public void testExportWithResetTtl() throws Exception {
238    String name = "testExportWithResetTtl";
239    TableName tableName = TableName.valueOf(name);
240    String snapshotNameStr = "snaptb-" + name;
241    byte[] snapshotName = Bytes.toBytes(snapshotNameStr);
242    Long ttl = 100000L;
243
244    try {
245      // create Table
246      createTable(tableName);
247      SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 50, FAMILY);
248      int tableNumFiles = admin.getRegions(tableName).size();
249      // take a snapshot with TTL
250      Map<String, Object> props = new HashMap<>();
251      props.put("TTL", ttl);
252      admin.snapshot(snapshotNameStr, tableName, props);
253      Optional<Long> ttlOpt =
254        admin.listSnapshots().stream().filter(s -> s.getName().equals(snapshotNameStr))
255          .map(org.apache.hadoop.hbase.client.SnapshotDescription::getTtl).findAny();
256      assertTrue(ttlOpt.isPresent());
257      assertEquals(ttl, ttlOpt.get());
258
259      testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles,
260        getHdfsDestinationDir(), false, true);
261    } catch (Exception e) {
262      throw e;
263    } finally {
264      TEST_UTIL.deleteTable(tableName);
265    }
266  }
267
268  private void testExportFileSystemState(final TableName tableName, final byte[] snapshotName,
269    final byte[] targetName, int filesExpected) throws Exception {
270    testExportFileSystemState(tableName, snapshotName, targetName, filesExpected,
271      getHdfsDestinationDir(), false);
272  }
273
274  protected void testExportFileSystemState(final TableName tableName, final byte[] snapshotName,
275    final byte[] targetName, int filesExpected, Path copyDir, boolean overwrite) throws Exception {
276    testExportFileSystemState(tableName, snapshotName, targetName, filesExpected, copyDir,
277      overwrite, false);
278  }
279
280  protected void testExportFileSystemState(final TableName tableName, final byte[] snapshotName,
281    final byte[] targetName, int filesExpected, Path copyDir, boolean overwrite, boolean resetTtl)
282    throws Exception {
283    testExportFileSystemState(TEST_UTIL.getConfiguration(), tableName, snapshotName, targetName,
284      filesExpected, TEST_UTIL.getDefaultRootDirPath(), copyDir, overwrite, resetTtl,
285      getBypassRegionPredicate(), true);
286  }
287
288  /**
289   * Creates destination directory, runs ExportSnapshot() tool, and runs some verifications.
290   */
291  protected static void testExportFileSystemState(final Configuration conf,
292    final TableName tableName, final byte[] snapshotName, final byte[] targetName,
293    final int filesExpected, final Path srcDir, Path rawTgtDir, final boolean overwrite,
294    final boolean resetTtl, final RegionPredicate bypassregionPredicate, boolean success)
295    throws Exception {
296    FileSystem tgtFs = rawTgtDir.getFileSystem(conf);
297    FileSystem srcFs = srcDir.getFileSystem(conf);
298    Path tgtDir = rawTgtDir.makeQualified(tgtFs.getUri(), tgtFs.getWorkingDirectory());
299    LOG.info("tgtFsUri={}, tgtDir={}, rawTgtDir={}, srcFsUri={}, srcDir={}", tgtFs.getUri(), tgtDir,
300      rawTgtDir, srcFs.getUri(), srcDir);
301    List<String> opts = new ArrayList<>();
302    opts.add("--snapshot");
303    opts.add(Bytes.toString(snapshotName));
304    opts.add("--copy-to");
305    opts.add(tgtDir.toString());
306    if (targetName != snapshotName) {
307      opts.add("--target");
308      opts.add(Bytes.toString(targetName));
309    }
310    if (overwrite) {
311      opts.add("--overwrite");
312    }
313    if (resetTtl) {
314      opts.add("--reset-ttl");
315    }
316
317    // Export Snapshot
318    int res = run(conf, new ExportSnapshot(), opts.toArray(new String[opts.size()]));
319    assertEquals("success " + success + ", res=" + res, success ? 0 : 1, res);
320    if (!success) {
321      final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(targetName));
322      assertFalse(tgtDir.toString() + " " + targetDir.toString(),
323        tgtFs.exists(new Path(tgtDir, targetDir)));
324      return;
325    }
326    LOG.info("Exported snapshot");
327
328    // Verify File-System state
329    FileStatus[] rootFiles = tgtFs.listStatus(tgtDir);
330    assertEquals(filesExpected > 0 ? 2 : 1, rootFiles.length);
331    for (FileStatus fileStatus : rootFiles) {
332      String name = fileStatus.getPath().getName();
333      assertTrue(fileStatus.toString(), fileStatus.isDirectory());
334      assertTrue(name.toString(), name.equals(HConstants.SNAPSHOT_DIR_NAME)
335        || name.equals(HConstants.HFILE_ARCHIVE_DIRECTORY));
336    }
337    LOG.info("Verified filesystem state");
338
339    // Compare the snapshot metadata and verify the hfiles
340    final Path snapshotDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(snapshotName));
341    final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(targetName));
342    verifySnapshotDir(srcFs, new Path(srcDir, snapshotDir), tgtFs, new Path(tgtDir, targetDir));
343    Set<String> snapshotFiles = verifySnapshot(conf, tgtFs, tgtDir, tableName,
344      Bytes.toString(targetName), resetTtl, bypassregionPredicate);
345    assertEquals(filesExpected, snapshotFiles.size());
346  }
347
348  /*
349   * verify if the snapshot folder on file-system 1 match the one on file-system 2
350   */
351  protected static void verifySnapshotDir(final FileSystem fs1, final Path root1,
352    final FileSystem fs2, final Path root2) throws IOException {
353    assertEquals(listFiles(fs1, root1, root1), listFiles(fs2, root2, root2));
354  }
355
356  /*
357   * Verify if the files exists
358   */
359  protected static Set<String> verifySnapshot(final Configuration conf, final FileSystem fs,
360    final Path rootDir, final TableName tableName, final String snapshotName,
361    final boolean resetTtl, final RegionPredicate bypassregionPredicate) throws IOException {
362    final Path exportedSnapshot =
363      new Path(rootDir, new Path(HConstants.SNAPSHOT_DIR_NAME, snapshotName));
364    final Set<String> snapshotFiles = new HashSet<>();
365    final Path exportedArchive = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
366    SnapshotReferenceUtil.visitReferencedFiles(conf, fs, exportedSnapshot,
367      new SnapshotReferenceUtil.SnapshotVisitor() {
368        @Override
369        public void storeFile(final RegionInfo regionInfo, final String family,
370          final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
371          if (bypassregionPredicate != null && bypassregionPredicate.evaluate(regionInfo)) {
372            return;
373          }
374
375          if (!storeFile.hasReference() && !StoreFileInfo.isReference(storeFile.getName())) {
376            String hfile = storeFile.getName();
377            snapshotFiles.add(hfile);
378            verifyNonEmptyFile(new Path(exportedArchive,
379              new Path(CommonFSUtils.getTableDir(new Path("./"), tableName),
380                new Path(regionInfo.getEncodedName(), new Path(family, hfile)))));
381          } else {
382            Pair<String, String> referredToRegionAndFile =
383              StoreFileInfo.getReferredToRegionAndFile(storeFile.getName());
384            String region = referredToRegionAndFile.getFirst();
385            String hfile = referredToRegionAndFile.getSecond();
386            snapshotFiles.add(hfile);
387            verifyNonEmptyFile(new Path(exportedArchive,
388              new Path(CommonFSUtils.getTableDir(new Path("./"), tableName),
389                new Path(region, new Path(family, hfile)))));
390          }
391        }
392
393        private void verifyNonEmptyFile(final Path path) throws IOException {
394          assertTrue(path + " should exists", fs.exists(path));
395          assertTrue(path + " should not be empty", fs.getFileStatus(path).getLen() > 0);
396        }
397      });
398
399    // Verify Snapshot description
400    SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, exportedSnapshot);
401    assertTrue(desc.getName().equals(snapshotName));
402    assertTrue(desc.getTable().equals(tableName.getNameAsString()));
403    if (resetTtl) {
404      assertEquals(HConstants.DEFAULT_SNAPSHOT_TTL, desc.getTtl());
405    }
406    return snapshotFiles;
407  }
408
409  private static Set<String> listFiles(final FileSystem fs, final Path root, final Path dir)
410    throws IOException {
411    Set<String> files = new HashSet<>();
412    LOG.debug("List files in {} in root {} at {}", fs, root, dir);
413    int rootPrefix = root.makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString().length();
414    FileStatus[] list = CommonFSUtils.listStatus(fs, dir);
415    if (list != null) {
416      for (FileStatus fstat : list) {
417        LOG.debug(Objects.toString(fstat.getPath()));
418        if (fstat.isDirectory()) {
419          files.addAll(listFiles(fs, root, fstat.getPath()));
420        } else {
421          files.add(fstat.getPath().makeQualified(fs).toString().substring(rootPrefix));
422        }
423      }
424    }
425    return files;
426  }
427
428  private Path getHdfsDestinationDir() {
429    Path rootDir = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
430    Path path =
431      new Path(new Path(rootDir, "export-test"), "export-" + EnvironmentEdgeManager.currentTime());
432    LOG.info("HDFS export destination path: " + path);
433    return path;
434  }
435
436  static Path getLocalDestinationDir(HBaseTestingUtility htu) {
437    Path path = htu.getDataTestDir("local-export-" + EnvironmentEdgeManager.currentTime());
438    try {
439      FileSystem fs = FileSystem.getLocal(htu.getConfiguration());
440      LOG.info("Local export destination path: " + path);
441      return path.makeQualified(fs.getUri(), fs.getWorkingDirectory());
442    } catch (IOException ioe) {
443      throw new RuntimeException(ioe);
444    }
445  }
446
447  private static void removeExportDir(final Path path) throws IOException {
448    FileSystem fs = FileSystem.get(path.toUri(), new Configuration());
449    fs.delete(path, true);
450  }
451}