001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.snapshot;
019
020import static org.junit.Assert.assertTrue;
021
022import java.io.IOException;
023import java.util.HashSet;
024import java.util.Set;
025import org.apache.hadoop.fs.FileSystem;
026import org.apache.hadoop.fs.LocalFileSystem;
027import org.apache.hadoop.fs.Path;
028import org.apache.hadoop.hbase.HBaseClassTestRule;
029import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
030import org.apache.hadoop.hbase.HConstants;
031import org.apache.hadoop.hbase.TableName;
032import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
033import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
034import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils.SnapshotMock;
035import org.apache.hadoop.hbase.testclassification.MapReduceTests;
036import org.apache.hadoop.hbase.testclassification.MediumTests;
037import org.apache.hadoop.hbase.util.Bytes;
038import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
039import org.apache.hadoop.hbase.util.Pair;
040import org.junit.Before;
041import org.junit.ClassRule;
042import org.junit.Test;
043import org.junit.experimental.categories.Category;
044import org.slf4j.Logger;
045import org.slf4j.LoggerFactory;
046
047/**
048 * Test Export Snapshot Tool Tests V1 snapshots only. Used to ALSO test v2 but strange failure so
049 * separate the tests. See companion file for test of v2 snapshot.
050 * @see TestExportSnapshotV2NoCluster
051 */
052@Category({ MapReduceTests.class, MediumTests.class })
053public class TestExportSnapshotV1NoCluster {
054  @ClassRule
055  public static final HBaseClassTestRule CLASS_RULE =
056    HBaseClassTestRule.forClass(TestExportSnapshotV1NoCluster.class);
057  private static final Logger LOG = LoggerFactory.getLogger(TestExportSnapshotV1NoCluster.class);
058
059  private HBaseCommonTestingUtility testUtil = new HBaseCommonTestingUtility();
060  private Path testDir;
061  private FileSystem fs;
062
063  @Before
064  public void setUpBefore() throws Exception {
065    // Make sure testDir is on LocalFileSystem
066    this.fs = FileSystem.getLocal(this.testUtil.getConfiguration());
067    this.testDir = setup(fs, this.testUtil);
068    LOG.info("fs={}, fsuri={}, fswd={}, testDir={}", this.fs, this.fs.getUri(),
069      this.fs.getWorkingDirectory(), this.testDir);
070    assertTrue("FileSystem '" + fs + "' is not local", fs instanceof LocalFileSystem);
071  }
072
073  /**
074   * Setup for test. Returns path to test data dir. Sets configuration into the passed
075   * hctu.getConfiguration.
076   */
077  static Path setup(FileSystem fs, HBaseCommonTestingUtility hctu) throws IOException {
078    Path testDir = hctu.getDataTestDir().makeQualified(fs.getUri(), fs.getWorkingDirectory());
079    hctu.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
080    hctu.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
081    hctu.getConfiguration().setInt("hbase.client.pause", 250);
082    hctu.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
083    hctu.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true);
084    hctu.getConfiguration().setInt("mapreduce.map.maxattempts", 10);
085    hctu.getConfiguration().set(HConstants.HBASE_DIR, testDir.toString());
086    return testDir.makeQualified(fs.getUri(), fs.getWorkingDirectory());
087  }
088
089  /**
090   * V1 snapshot test
091   */
092  @Test
093  public void testSnapshotWithRefsExportFileSystemState() throws Exception {
094    final SnapshotMock snapshotMock =
095      new SnapshotMock(testUtil.getConfiguration(), this.fs, testDir);
096    final SnapshotMock.SnapshotBuilder builder =
097      snapshotMock.createSnapshotV1("tableWithRefsV1", "tableWithRefsV1");
098    testSnapshotWithRefsExportFileSystemState(this.fs, builder, testUtil, testDir);
099  }
100
101  /**
102   * Generates a couple of regions for the specified SnapshotMock, and then it will run the export
103   * and verification.
104   */
105  static void testSnapshotWithRefsExportFileSystemState(FileSystem fs,
106    SnapshotMock.SnapshotBuilder builder, HBaseCommonTestingUtility testUtil, Path testDir)
107    throws Exception {
108    Path[] r1Files = builder.addRegion();
109    Path[] r2Files = builder.addRegion();
110    builder.commit();
111    // remove references, only keep data files
112    Set<String> dataFiles = new HashSet<>();
113    for (Path[] files : new Path[][] { r1Files, r2Files }) {
114      for (Path file : files) {
115        if (StoreFileInfo.isReference(file.getName())) {
116          Pair<String, String> referredToRegionAndFile =
117            StoreFileInfo.getReferredToRegionAndFile(file.getName());
118          dataFiles.add(referredToRegionAndFile.getSecond());
119        } else {
120          dataFiles.add(file.getName());
121        }
122      }
123    }
124    int snapshotFilesCount = dataFiles.size();
125    byte[] snapshotName = Bytes.toBytes(builder.getSnapshotDescription().getName());
126    TableName tableName = builder.getTableDescriptor().getTableName();
127    TestExportSnapshot.testExportFileSystemState(testUtil.getConfiguration(), tableName,
128      snapshotName, snapshotName, snapshotFilesCount, testDir,
129      getDestinationDir(fs, testUtil, testDir), false, false, null, true);
130  }
131
132  static Path getDestinationDir(FileSystem fs, HBaseCommonTestingUtility hctu, Path testDir)
133    throws IOException {
134    Path path =
135      new Path(new Path(testDir, "export-test"), "export-" + EnvironmentEdgeManager.currentTime())
136        .makeQualified(fs.getUri(), fs.getWorkingDirectory());
137    LOG.info("Export destination={}, fs={}, fsurl={}, fswd={}, testDir={}", path, fs, fs.getUri(),
138      fs.getWorkingDirectory(), testDir);
139    return path;
140  }
141}