001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.snapshot;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertThrows;
022import static org.junit.Assert.assertTrue;
023
024import java.io.IOException;
025import java.util.HashMap;
026import java.util.List;
027import java.util.Map;
028import org.apache.hadoop.conf.Configuration;
029import org.apache.hadoop.fs.FileSystem;
030import org.apache.hadoop.fs.LocatedFileStatus;
031import org.apache.hadoop.fs.Path;
032import org.apache.hadoop.fs.RemoteIterator;
033import org.apache.hadoop.hbase.HBaseClassTestRule;
034import org.apache.hadoop.hbase.HBaseTestingUtil;
035import org.apache.hadoop.hbase.HConstants;
036import org.apache.hadoop.hbase.TableName;
037import org.apache.hadoop.hbase.client.RegionInfo;
038import org.apache.hadoop.hbase.client.SnapshotType;
039import org.apache.hadoop.hbase.client.Table;
040import org.apache.hadoop.hbase.client.TableDescriptor;
041import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
042import org.apache.hadoop.hbase.io.HFileLink;
043import org.apache.hadoop.hbase.mob.MobUtils;
044import org.apache.hadoop.hbase.monitoring.MonitoredTask;
045import org.apache.hadoop.hbase.regionserver.HRegion;
046import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
047import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils.SnapshotMock;
048import org.apache.hadoop.hbase.testclassification.MediumTests;
049import org.apache.hadoop.hbase.testclassification.RegionServerTests;
050import org.apache.hadoop.hbase.util.Bytes;
051import org.apache.hadoop.hbase.util.CommonFSUtils;
052import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
053import org.apache.hadoop.hbase.util.FSTableDescriptors;
054import org.apache.hadoop.hbase.wal.WALSplitUtil;
055import org.junit.After;
056import org.junit.AfterClass;
057import org.junit.Assert;
058import org.junit.Before;
059import org.junit.BeforeClass;
060import org.junit.ClassRule;
061import org.junit.Test;
062import org.junit.experimental.categories.Category;
063import org.mockito.Mockito;
064import org.slf4j.Logger;
065import org.slf4j.LoggerFactory;
066
067import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription;
068
069/**
070 * Test the restore/clone operation from a file-system point of view.
071 */
072@Category({ RegionServerTests.class, MediumTests.class })
073public class TestRestoreSnapshotHelper {
074
075  @ClassRule
076  public static final HBaseClassTestRule CLASS_RULE =
077    HBaseClassTestRule.forClass(TestRestoreSnapshotHelper.class);
078
079  private static final Logger LOG = LoggerFactory.getLogger(TestRestoreSnapshotHelper.class);
080
081  protected final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
082  protected final static String TEST_HFILE = "abc";
083
084  protected Configuration conf;
085  protected Path archiveDir;
086  protected FileSystem fs;
087  protected Path rootDir;
088
089  protected void setupConf(Configuration conf) {
090  }
091
092  @BeforeClass
093  public static void setupCluster() throws Exception {
094    TEST_UTIL.startMiniCluster();
095  }
096
097  @AfterClass
098  public static void tearDownCluster() throws Exception {
099    TEST_UTIL.shutdownMiniCluster();
100  }
101
102  @Before
103  public void setup() throws Exception {
104    rootDir = TEST_UTIL.getDataTestDir("testRestore");
105    archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
106    fs = TEST_UTIL.getTestFileSystem();
107    conf = TEST_UTIL.getConfiguration();
108    setupConf(conf);
109    CommonFSUtils.setRootDir(conf, rootDir);
110  }
111
112  @After
113  public void tearDown() throws Exception {
114    fs.delete(TEST_UTIL.getDataTestDir(), true);
115  }
116
117  protected SnapshotMock createSnapshotMock() throws IOException {
118    return new SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir);
119  }
120
121  @Test
122  public void testRestore() throws IOException {
123    restoreAndVerify("snapshot", "testRestore");
124  }
125
126  @Test
127  public void testRestoreWithNamespace() throws IOException {
128    restoreAndVerify("snapshot", "namespace1:testRestoreWithNamespace");
129  }
130
131  @Test
132  public void testNoHFileLinkInRootDir() throws IOException {
133    rootDir = TEST_UTIL.getDefaultRootDirPath();
134    CommonFSUtils.setRootDir(conf, rootDir);
135    fs = rootDir.getFileSystem(conf);
136
137    TableName tableName = TableName.valueOf("testNoHFileLinkInRootDir");
138    String snapshotName = tableName.getNameAsString() + "-snapshot";
139    createTableAndSnapshot(tableName, snapshotName);
140
141    Path restoreDir = new Path("/hbase/.tmp-restore");
142    RestoreSnapshotHelper.copySnapshotForScanner(conf, fs, rootDir, restoreDir, snapshotName);
143    checkNoHFileLinkInTableDir(tableName);
144  }
145
146  @Test
147  public void testSkipReplayAndUpdateSeqId() throws Exception {
148    rootDir = TEST_UTIL.getDefaultRootDirPath();
149    CommonFSUtils.setRootDir(conf, rootDir);
150    TableName tableName = TableName.valueOf("testSkipReplayAndUpdateSeqId");
151    String snapshotName = "testSkipReplayAndUpdateSeqId";
152    createTableAndSnapshot(tableName, snapshotName);
153    // put some data in the table
154    Table table = TEST_UTIL.getConnection().getTable(tableName);
155    TEST_UTIL.loadTable(table, Bytes.toBytes("A"));
156
157    Configuration conf = TEST_UTIL.getConfiguration();
158    Path rootDir = CommonFSUtils.getRootDir(conf);
159    Path restoreDir = new Path("/hbase/.tmp-restore/testScannerWithRestoreScanner2");
160    // restore snapshot.
161    final RestoreSnapshotHelper.RestoreMetaChanges meta =
162      RestoreSnapshotHelper.copySnapshotForScanner(conf, fs, rootDir, restoreDir, snapshotName);
163    TableDescriptor htd = meta.getTableDescriptor();
164    final List<RegionInfo> restoredRegions = meta.getRegionsToAdd();
165    for (RegionInfo restoredRegion : restoredRegions) {
166      // open restored region
167      HRegion region = HRegion.newHRegion(CommonFSUtils.getTableDir(restoreDir, tableName), null,
168        fs, conf, restoredRegion, htd, null);
169      // set restore flag
170      region.setRestoredRegion(true);
171      region.initialize();
172      Path recoveredEdit =
173        CommonFSUtils.getWALRegionDir(conf, tableName, region.getRegionInfo().getEncodedName());
174      long maxSeqId = WALSplitUtil.getMaxRegionSequenceId(fs, recoveredEdit);
175
176      // open restored region without set restored flag
177      HRegion region2 = HRegion.newHRegion(CommonFSUtils.getTableDir(restoreDir, tableName), null,
178        fs, conf, restoredRegion, htd, null);
179      region2.initialize();
180      long maxSeqId2 = WALSplitUtil.getMaxRegionSequenceId(fs, recoveredEdit);
181      Assert.assertTrue(maxSeqId2 > maxSeqId);
182    }
183  }
184
185  @Test
186  public void testCopyExpiredSnapshotForScanner() throws IOException, InterruptedException {
187    rootDir = TEST_UTIL.getDefaultRootDirPath();
188    CommonFSUtils.setRootDir(conf, rootDir);
189    TableName tableName = TableName.valueOf("testCopyExpiredSnapshotForScanner");
190    String snapshotName = tableName.getNameAsString() + "-snapshot";
191    Path restoreDir = new Path("/hbase/.tmp-expired-snapshot/copySnapshotDest");
192    // create table and put some data into the table
193    byte[] columnFamily = Bytes.toBytes("A");
194    Table table = TEST_UTIL.createTable(tableName, columnFamily);
195    TEST_UTIL.loadTable(table, columnFamily);
196    // create snapshot with ttl = 10 sec
197    Map<String, Object> properties = new HashMap<>();
198    properties.put("TTL", 10);
199    org.apache.hadoop.hbase.client.SnapshotDescription snapshotDesc =
200      new org.apache.hadoop.hbase.client.SnapshotDescription(snapshotName, tableName,
201        SnapshotType.FLUSH, null, EnvironmentEdgeManager.currentTime(), -1, properties);
202    TEST_UTIL.getAdmin().snapshot(snapshotDesc);
203    boolean isExist = TEST_UTIL.getAdmin().listSnapshots().stream()
204      .anyMatch(ele -> snapshotName.equals(ele.getName()));
205    assertTrue(isExist);
206    int retry = 6;
207    while (
208      !SnapshotDescriptionUtils.isExpiredSnapshot(snapshotDesc.getTtl(),
209        snapshotDesc.getCreationTime(), EnvironmentEdgeManager.currentTime()) && retry > 0
210    ) {
211      retry--;
212      Thread.sleep(10 * 1000);
213    }
214    boolean isExpiredSnapshot = SnapshotDescriptionUtils.isExpiredSnapshot(snapshotDesc.getTtl(),
215      snapshotDesc.getCreationTime(), EnvironmentEdgeManager.currentTime());
216    assertTrue(isExpiredSnapshot);
217    assertThrows(SnapshotTTLExpiredException.class, () -> RestoreSnapshotHelper
218      .copySnapshotForScanner(conf, fs, rootDir, restoreDir, snapshotName));
219  }
220
221  protected void createTableAndSnapshot(TableName tableName, String snapshotName)
222    throws IOException {
223    byte[] column = Bytes.toBytes("A");
224    Table table = TEST_UTIL.createTable(tableName, column, 2);
225    TEST_UTIL.loadTable(table, column);
226    TEST_UTIL.getAdmin().snapshot(snapshotName, tableName);
227  }
228
229  private void checkNoHFileLinkInTableDir(TableName tableName) throws IOException {
230    Path[] tableDirs = new Path[] { CommonFSUtils.getTableDir(rootDir, tableName),
231      CommonFSUtils.getTableDir(new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY), tableName),
232      CommonFSUtils.getTableDir(MobUtils.getMobHome(rootDir), tableName) };
233    for (Path tableDir : tableDirs) {
234      Assert.assertFalse(hasHFileLink(tableDir));
235    }
236  }
237
238  private boolean hasHFileLink(Path tableDir) throws IOException {
239    if (fs.exists(tableDir)) {
240      RemoteIterator<LocatedFileStatus> iterator = fs.listFiles(tableDir, true);
241      while (iterator.hasNext()) {
242        LocatedFileStatus fileStatus = iterator.next();
243        if (fileStatus.isFile() && HFileLink.isHFileLink(fileStatus.getPath())) {
244          return true;
245        }
246      }
247    }
248    return false;
249  }
250
251  private void restoreAndVerify(final String snapshotName, final String tableName)
252    throws IOException {
253    // Test Rolling-Upgrade like Snapshot.
254    // half machines writing using v1 and the others using v2 format.
255    SnapshotMock snapshotMock = createSnapshotMock();
256    SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2("snapshot", tableName);
257    builder.addRegionV1();
258    builder.addRegionV2();
259    builder.addRegionV2();
260    builder.addRegionV1();
261    Path snapshotDir = builder.commit();
262    TableDescriptor htd = builder.getTableDescriptor();
263    SnapshotDescription desc = builder.getSnapshotDescription();
264
265    // Test clone a snapshot
266    TableDescriptor htdClone = snapshotMock.createHtd("testtb-clone");
267    testRestore(snapshotDir, desc, htdClone);
268    verifyRestore(rootDir, htd, htdClone);
269
270    // Test clone a clone ("link to link")
271    SnapshotDescription cloneDesc =
272      SnapshotDescription.newBuilder().setName("cloneSnapshot").setTable("testtb-clone").build();
273    Path cloneDir = CommonFSUtils.getTableDir(rootDir, htdClone.getTableName());
274    TableDescriptor htdClone2 = snapshotMock.createHtd("testtb-clone2");
275    testRestore(cloneDir, cloneDesc, htdClone2);
276    verifyRestore(rootDir, htd, htdClone2);
277  }
278
279  private void verifyRestore(final Path rootDir, final TableDescriptor sourceHtd,
280    final TableDescriptor htdClone) throws IOException {
281    List<String> files = SnapshotTestingUtils.listHFileNames(fs,
282      CommonFSUtils.getTableDir(rootDir, htdClone.getTableName()));
283    assertEquals(12, files.size());
284    for (int i = 0; i < files.size(); i += 2) {
285      String linkFile = files.get(i);
286      String refFile = files.get(i + 1);
287      assertTrue(linkFile + " should be a HFileLink", HFileLink.isHFileLink(linkFile));
288      assertTrue(refFile + " should be a Referene", StoreFileInfo.isReference(refFile));
289      assertEquals(sourceHtd.getTableName(), HFileLink.getReferencedTableName(linkFile));
290      Path refPath = getReferredToFile(refFile);
291      LOG.debug("get reference name for file " + refFile + " = " + refPath);
292      assertTrue(refPath.getName() + " should be a HFileLink",
293        HFileLink.isHFileLink(refPath.getName()));
294      assertEquals(linkFile, refPath.getName());
295    }
296  }
297
298  /**
299   * Execute the restore operation
300   * @param snapshotDir The snapshot directory to use as "restore source"
301   * @param sd          The snapshot descriptor
302   * @param htdClone    The HTableDescriptor of the table to restore/clone.
303   */
304  private void testRestore(final Path snapshotDir, final SnapshotDescription sd,
305    final TableDescriptor htdClone) throws IOException {
306    LOG.debug("pre-restore table=" + htdClone.getTableName() + " snapshot=" + snapshotDir);
307    CommonFSUtils.logFileSystemState(fs, rootDir, LOG);
308
309    new FSTableDescriptors(conf).createTableDescriptor(htdClone);
310    RestoreSnapshotHelper helper = getRestoreHelper(rootDir, snapshotDir, sd, htdClone);
311    helper.restoreHdfsRegions();
312
313    LOG.debug("post-restore table=" + htdClone.getTableName() + " snapshot=" + snapshotDir);
314    CommonFSUtils.logFileSystemState(fs, rootDir, LOG);
315  }
316
317  /**
318   * Initialize the restore helper, based on the snapshot and table information provided.
319   */
320  private RestoreSnapshotHelper getRestoreHelper(final Path rootDir, final Path snapshotDir,
321    final SnapshotDescription sd, final TableDescriptor htdClone) throws IOException {
322    ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class);
323    MonitoredTask status = Mockito.mock(MonitoredTask.class);
324
325    SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd);
326    return new RestoreSnapshotHelper(conf, fs, manifest, htdClone, rootDir, monitor, status);
327  }
328
329  private Path getReferredToFile(final String referenceName) {
330    Path fakeBasePath = new Path(new Path("table", "region"), "cf");
331    return StoreFileInfo.getReferredToFile(new Path(fakeBasePath, referenceName));
332  }
333}