001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.master.procedure;
019
020import java.io.IOException;
021import java.util.Optional;
022import org.apache.hadoop.conf.Configuration;
023import org.apache.hadoop.fs.FileSystem;
024import org.apache.hadoop.fs.Path;
025import org.apache.hadoop.hbase.HBaseClassTestRule;
026import org.apache.hadoop.hbase.HBaseTestingUtil;
027import org.apache.hadoop.hbase.ServerName;
028import org.apache.hadoop.hbase.TableName;
029import org.apache.hadoop.hbase.client.RegionInfo;
030import org.apache.hadoop.hbase.client.SnapshotDescription;
031import org.apache.hadoop.hbase.client.SnapshotType;
032import org.apache.hadoop.hbase.client.Table;
033import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
034import org.apache.hadoop.hbase.master.HMaster;
035import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
036import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
037import org.apache.hadoop.hbase.procedure2.RemoteProcedureDispatcher;
038import org.apache.hadoop.hbase.regionserver.HRegion;
039import org.apache.hadoop.hbase.regionserver.HStoreFile;
040import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
041import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
042import org.apache.hadoop.hbase.testclassification.MediumTests;
043import org.apache.hadoop.hbase.testclassification.RegionServerTests;
044import org.apache.hadoop.hbase.util.Bytes;
045import org.apache.hadoop.hbase.util.CommonFSUtils;
046import org.apache.hadoop.hbase.util.RegionSplitter;
047import org.junit.After;
048import org.junit.Assert;
049import org.junit.Before;
050import org.junit.ClassRule;
051import org.junit.Test;
052import org.junit.experimental.categories.Category;
053import org.slf4j.Logger;
054import org.slf4j.LoggerFactory;
055
056import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
057import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos;
058
059@Category({ RegionServerTests.class, MediumTests.class })
060public class TestSnapshotVerifyProcedure {
061  private static final Logger LOG = LoggerFactory.getLogger(TestSnapshotVerifyProcedure.class);
062
063  @ClassRule
064  public static final HBaseClassTestRule CLASS_RULE =
065    HBaseClassTestRule.forClass(TestSnapshotVerifyProcedure.class);
066
067  private HBaseTestingUtil TEST_UTIL;
068  private final TableName tableName = TableName.valueOf("TestRSSnapshotVerifier");
069  private final byte[] cf = Bytes.toBytes("cf");
070  private final SnapshotDescription snapshot =
071    new SnapshotDescription("test-snapshot", tableName, SnapshotType.FLUSH);
072  private SnapshotProtos.SnapshotDescription snapshotProto =
073    ProtobufUtil.createHBaseProtosSnapshotDesc(snapshot);
074
075  @Before
076  public void setup() throws Exception {
077    TEST_UTIL = new HBaseTestingUtil();
078    Configuration conf = TEST_UTIL.getConfiguration();
079    // delay procedure dispatch
080    conf.setInt(RemoteProcedureDispatcher.DISPATCH_DELAY_CONF_KEY, 10000);
081    conf.setInt(RemoteProcedureDispatcher.DISPATCH_MAX_QUEUE_SIZE_CONF_KEY, 128);
082    TEST_UTIL.startMiniCluster(3);
083    final byte[][] splitKeys = new RegionSplitter.HexStringSplit().split(10);
084    Table table = TEST_UTIL.createTable(tableName, cf, splitKeys);
085    TEST_UTIL.loadTable(table, cf, false);
086    TEST_UTIL.getAdmin().flush(tableName);
087
088    // prepare unverified snapshot
089    snapshotProto = SnapshotDescriptionUtils.validate(snapshotProto, conf);
090    Path rootDir = CommonFSUtils.getRootDir(conf);
091    Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshotProto, rootDir, conf);
092    FileSystem workingDirFs = workingDir.getFileSystem(conf);
093    if (!workingDirFs.exists(workingDir)) {
094      workingDirFs.mkdirs(workingDir);
095    }
096    ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(snapshot.getName());
097    SnapshotManifest manifest =
098      SnapshotManifest.create(conf, workingDirFs, workingDir, snapshotProto, monitor);
099    manifest.addTableDescriptor(
100      TEST_UTIL.getHBaseCluster().getMaster().getTableDescriptors().get(tableName));
101    SnapshotDescriptionUtils.writeSnapshotInfo(snapshotProto, workingDir, workingDirFs);
102    TEST_UTIL.getHBaseCluster().getRegions(tableName).forEach(r -> {
103      try {
104        r.addRegionToSnapshot(snapshotProto, monitor);
105      } catch (IOException e) {
106        LOG.warn("Failed snapshot region {}", r.getRegionInfo());
107      }
108    });
109    manifest.consolidate();
110  }
111
112  @Test
113  public void testSimpleVerify() throws Exception {
114    Optional<HRegion> regionOpt = TEST_UTIL.getHBaseCluster().getRegions(tableName).stream()
115      .filter(r -> !r.getStore(cf).getStorefiles().isEmpty()).findFirst();
116    Assert.assertTrue(regionOpt.isPresent());
117    HRegion region = regionOpt.get();
118    SnapshotVerifyProcedure p1 = new SnapshotVerifyProcedure(snapshotProto, region.getRegionInfo());
119    ProcedureExecutor<MasterProcedureEnv> procExec =
120      TEST_UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
121    long procId = procExec.submitProcedure(p1);
122    ProcedureTestingUtility.waitProcedure(procExec, procId);
123    Assert.assertTrue(p1.isSuccess());
124
125    // delete store file to trigger a CorruptedSnapshotException
126    for (HStoreFile file : region.getStore(cf).getStorefiles()) {
127      TEST_UTIL.getDFSCluster().getFileSystem().delete(file.getPath(), true);
128      LOG.info("delete store file {}", file.getPath());
129    }
130    SnapshotVerifyProcedure p2 = new SnapshotVerifyProcedure(snapshotProto, region.getRegionInfo());
131    long newProcId = procExec.submitProcedure(p2);
132    ProcedureTestingUtility.waitProcedure(procExec, newProcId);
133    Assert.assertTrue(p2.isSuccess());
134  }
135
136  @Test
137  public void testRestartMaster() throws Exception {
138    RegionInfo region = TEST_UTIL.getHBaseCluster().getRegions(tableName).get(0).getRegionInfo();
139    SnapshotVerifyProcedure svp = new SnapshotVerifyProcedure(snapshotProto, region);
140    HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
141    long procId = master.getMasterProcedureExecutor().submitProcedure(svp);
142    TEST_UTIL.waitFor(10000, () -> svp.getServerName() != null);
143    ServerName worker = svp.getServerName();
144    int availableWorker = master.getSnapshotManager().getAvailableWorker(worker);
145
146    // restart master
147    TEST_UTIL.getHBaseCluster().killMaster(master.getServerName());
148    TEST_UTIL.getHBaseCluster().waitForMasterToStop(master.getServerName(), 30000);
149    TEST_UTIL.getHBaseCluster().startMaster();
150    TEST_UTIL.getHBaseCluster().waitForActiveAndReadyMaster();
151
152    // restore used worker
153    master = TEST_UTIL.getHBaseCluster().getMaster();
154    SnapshotVerifyProcedure svp2 =
155      master.getMasterProcedureExecutor().getProcedure(SnapshotVerifyProcedure.class, procId);
156    Assert.assertNotNull(svp2);
157    Assert.assertFalse(svp2.isFinished());
158    Assert.assertNotNull(svp2.getServerName());
159    Assert.assertEquals(worker, svp.getServerName());
160    Assert.assertEquals((int) master.getSnapshotManager().getAvailableWorker(worker),
161      availableWorker);
162
163    // release worker
164    ProcedureTestingUtility.waitProcedure(master.getMasterProcedureExecutor(), svp2);
165    Assert.assertEquals((int) master.getSnapshotManager().getAvailableWorker(worker),
166      availableWorker + 1);
167  }
168
169  @After
170  public void teardown() throws Exception {
171    TEST_UTIL.shutdownMiniCluster();
172  }
173}