001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.backup;
019
020import static org.junit.Assert.assertNull;
021import static org.junit.Assert.assertTrue;
022
023import java.io.IOException;
024import java.util.ArrayList;
025import java.util.List;
026import java.util.Optional;
027import org.apache.hadoop.fs.FileSystem;
028import org.apache.hadoop.fs.Path;
029import org.apache.hadoop.hbase.HBaseClassTestRule;
030import org.apache.hadoop.hbase.HBaseTestingUtility;
031import org.apache.hadoop.hbase.HConstants;
032import org.apache.hadoop.hbase.TableName;
033import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;
034import org.apache.hadoop.hbase.client.Admin;
035import org.apache.hadoop.hbase.client.Connection;
036import org.apache.hadoop.hbase.client.SnapshotDescription;
037import org.apache.hadoop.hbase.client.TableDescriptor;
038import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
039import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor;
040import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
041import org.apache.hadoop.hbase.coprocessor.MasterObserver;
042import org.apache.hadoop.hbase.coprocessor.ObserverContext;
043import org.apache.hadoop.hbase.testclassification.LargeTests;
044import org.apache.hadoop.util.ToolRunner;
045import org.junit.BeforeClass;
046import org.junit.ClassRule;
047import org.junit.Test;
048import org.junit.experimental.categories.Category;
049import org.slf4j.Logger;
050import org.slf4j.LoggerFactory;
051
052import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
053
054/**
055 * This class is only a base for other integration-level backup tests. Do not add tests here.
056 * TestBackupSmallTests is where tests that don't require bring machines up/down should go All other
057 * tests should have their own classes and extend this one
058 */
059@Category(LargeTests.class)
060public class TestBackupDeleteWithFailures extends TestBackupBase {
061
062  @ClassRule
063  public static final HBaseClassTestRule CLASS_RULE =
064    HBaseClassTestRule.forClass(TestBackupDeleteWithFailures.class);
065
066  private static final Logger LOG = LoggerFactory.getLogger(TestBackupDeleteWithFailures.class);
067
068  public enum Failure {
069    NO_FAILURES,
070    PRE_SNAPSHOT_FAILURE,
071    PRE_DELETE_SNAPSHOT_FAILURE,
072    POST_DELETE_SNAPSHOT_FAILURE
073  }
074
075  public static class MasterSnapshotObserver implements MasterCoprocessor, MasterObserver {
076    List<Failure> failures = new ArrayList<>();
077
078    public void setFailures(Failure... f) {
079      failures.clear();
080      for (int i = 0; i < f.length; i++) {
081        failures.add(f[i]);
082      }
083    }
084
085    @Override
086    public Optional<MasterObserver> getMasterObserver() {
087      return Optional.of(this);
088    }
089
090    @Override
091    public void preSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
092      final SnapshotDescription snapshot, final TableDescriptor hTableDescriptor)
093      throws IOException {
094      if (failures.contains(Failure.PRE_SNAPSHOT_FAILURE)) {
095        throw new IOException("preSnapshot");
096      }
097    }
098
099    @Override
100    public void preDeleteSnapshot(ObserverContext<MasterCoprocessorEnvironment> ctx,
101      SnapshotDescription snapshot) throws IOException {
102      if (failures.contains(Failure.PRE_DELETE_SNAPSHOT_FAILURE)) {
103        throw new IOException("preDeleteSnapshot");
104      }
105    }
106
107    @Override
108    public void postDeleteSnapshot(ObserverContext<MasterCoprocessorEnvironment> ctx,
109      SnapshotDescription snapshot) throws IOException {
110      if (failures.contains(Failure.POST_DELETE_SNAPSHOT_FAILURE)) {
111        throw new IOException("postDeleteSnapshot");
112      }
113    }
114  }
115
116  /**
117   * Setup Cluster with appropriate configurations before running tests.
118   * @throws Exception if starting the mini cluster or setting up the tables fails
119   */
120  @BeforeClass
121  public static void setUp() throws Exception {
122    TEST_UTIL = new HBaseTestingUtility();
123    conf1 = TEST_UTIL.getConfiguration();
124    conf1.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, MasterSnapshotObserver.class.getName());
125    conf1.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
126    setUpHelper();
127  }
128
129  private MasterSnapshotObserver getMasterSnapshotObserver() {
130    return TEST_UTIL.getHBaseCluster().getMaster().getMasterCoprocessorHost()
131      .findCoprocessor(MasterSnapshotObserver.class);
132  }
133
134  @Test
135  public void testBackupDeleteWithFailures() throws Exception {
136    testBackupDeleteWithFailuresAfter(1, Failure.PRE_DELETE_SNAPSHOT_FAILURE);
137    testBackupDeleteWithFailuresAfter(0, Failure.POST_DELETE_SNAPSHOT_FAILURE);
138    testBackupDeleteWithFailuresAfter(1, Failure.PRE_SNAPSHOT_FAILURE);
139  }
140
141  private void testBackupDeleteWithFailuresAfter(int expected, Failure... failures)
142    throws Exception {
143    LOG.info("test repair backup delete on a single table with data and failures " + failures[0]);
144    List<TableName> tableList = Lists.newArrayList(table1);
145    String backupId = fullTableBackup(tableList);
146    assertTrue(checkSucceeded(backupId));
147    LOG.info("backup complete");
148    String[] backupIds = new String[] { backupId };
149    BackupSystemTable table = new BackupSystemTable(TEST_UTIL.getConnection());
150    BackupInfo info = table.readBackupInfo(backupId);
151    Path path = new Path(info.getBackupRootDir(), backupId);
152    FileSystem fs = FileSystem.get(path.toUri(), conf1);
153    assertTrue(fs.exists(path));
154
155    Connection conn = TEST_UTIL.getConnection();
156    Admin admin = conn.getAdmin();
157    MasterSnapshotObserver observer = getMasterSnapshotObserver();
158
159    observer.setFailures(failures);
160    try {
161      getBackupAdmin().deleteBackups(backupIds);
162    } catch (IOException e) {
163      if (expected != 1) {
164        assertTrue(false);
165      }
166    }
167
168    // Verify that history length == expected after delete failure
169    assertTrue(table.getBackupHistory().size() == expected);
170
171    String[] ids = table.getListOfBackupIdsFromDeleteOperation();
172
173    // Verify that we still have delete record in backup system table
174    if (expected == 1) {
175      assertTrue(ids.length == 1);
176      assertTrue(ids[0].equals(backupId));
177    } else {
178      assertNull(ids);
179    }
180
181    // Now run repair command to repair "failed" delete operation
182    String[] args = new String[] { "repair" };
183
184    observer.setFailures(Failure.NO_FAILURES);
185
186    // Run repair
187    int ret = ToolRunner.run(conf1, new BackupDriver(), args);
188    assertTrue(ret == 0);
189    // Verify that history length == 0
190    assertTrue(table.getBackupHistory().size() == 0);
191    ids = table.getListOfBackupIdsFromDeleteOperation();
192
193    // Verify that we do not have delete record in backup system table
194    assertNull(ids);
195
196    table.close();
197    admin.close();
198  }
199}