001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.backup;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertFalse;
022import static org.junit.Assert.assertTrue;
023
024import java.io.IOException;
025import java.nio.ByteBuffer;
026import java.util.List;
027import java.util.Map;
028import org.apache.hadoop.fs.FileSystem;
029import org.apache.hadoop.fs.Path;
030import org.apache.hadoop.hbase.HBaseClassTestRule;
031import org.apache.hadoop.hbase.TableName;
032import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;
033import org.apache.hadoop.hbase.backup.impl.BulkLoad;
034import org.apache.hadoop.hbase.backup.util.BackupUtils;
035import org.apache.hadoop.hbase.client.Get;
036import org.apache.hadoop.hbase.client.Result;
037import org.apache.hadoop.hbase.client.Table;
038import org.apache.hadoop.hbase.testclassification.LargeTests;
039import org.apache.hadoop.hbase.tool.BulkLoadHFiles;
040import org.apache.hadoop.hbase.util.Bytes;
041import org.apache.hadoop.hbase.util.HFileTestUtil;
042import org.junit.ClassRule;
043import org.junit.Test;
044import org.junit.experimental.categories.Category;
045
046/**
047 * This test checks whether backups properly track & manage bulk files loads.
048 */
049@Category(LargeTests.class)
050public class TestIncrementalBackupWithBulkLoad extends TestBackupBase {
051
052  @ClassRule
053  public static final HBaseClassTestRule CLASS_RULE =
054    HBaseClassTestRule.forClass(TestIncrementalBackupWithBulkLoad.class);
055
056  private static final String TEST_NAME = TestIncrementalBackupWithBulkLoad.class.getSimpleName();
057  private static final int ROWS_IN_BULK_LOAD = 100;
058
059  // implement all test cases in 1 test since incremental backup/restore has dependencies
060  @Test
061  public void TestIncBackupDeleteTable() throws Exception {
062    try (BackupSystemTable systemTable = new BackupSystemTable(TEST_UTIL.getConnection())) {
063      // The test starts with some data, and no bulk loaded rows.
064      int expectedRowCount = NB_ROWS_IN_BATCH;
065      assertEquals(expectedRowCount, TEST_UTIL.countRows(table1));
066      assertTrue(systemTable.readBulkloadRows(List.of(table1)).isEmpty());
067
068      // Bulk loads aren't tracked if the table isn't backed up yet
069      performBulkLoad("bulk1");
070      expectedRowCount += ROWS_IN_BULK_LOAD;
071      assertEquals(expectedRowCount, TEST_UTIL.countRows(table1));
072      assertEquals(0, systemTable.readBulkloadRows(List.of(table1)).size());
073
074      // Create a backup, bulk loads are now being tracked
075      String backup1 = backupTables(BackupType.FULL, List.of(table1), BACKUP_ROOT_DIR);
076      assertTrue(checkSucceeded(backup1));
077      performBulkLoad("bulk2");
078      expectedRowCount += ROWS_IN_BULK_LOAD;
079      assertEquals(expectedRowCount, TEST_UTIL.countRows(table1));
080      assertEquals(1, systemTable.readBulkloadRows(List.of(table1)).size());
081
082      // Truncating or deleting a table clears the tracked bulk loads (and all rows)
083      TEST_UTIL.truncateTable(table1).close();
084      expectedRowCount = 0;
085      assertEquals(expectedRowCount, TEST_UTIL.countRows(table1));
086      assertEquals(0, systemTable.readBulkloadRows(List.of(table1)).size());
087
088      // Creating a full backup clears the bulk loads (since they are captured in the snapshot)
089      performBulkLoad("bulk3");
090      expectedRowCount = ROWS_IN_BULK_LOAD;
091      assertEquals(expectedRowCount, TEST_UTIL.countRows(table1));
092      assertEquals(1, systemTable.readBulkloadRows(List.of(table1)).size());
093      String backup2 = backupTables(BackupType.FULL, List.of(table1), BACKUP_ROOT_DIR);
094      assertTrue(checkSucceeded(backup2));
095      assertEquals(expectedRowCount, TEST_UTIL.countRows(table1));
096      assertEquals(0, systemTable.readBulkloadRows(List.of(table1)).size());
097
098      // Creating an incremental backup clears the bulk loads
099      performBulkLoad("bulk4");
100      performBulkLoad("bulk5");
101      performBulkLoad("bulk6");
102      expectedRowCount += 3 * ROWS_IN_BULK_LOAD;
103      assertEquals(expectedRowCount, TEST_UTIL.countRows(table1));
104      assertEquals(3, systemTable.readBulkloadRows(List.of(table1)).size());
105      String backup3 = backupTables(BackupType.INCREMENTAL, List.of(table1), BACKUP_ROOT_DIR);
106      assertTrue(checkSucceeded(backup3));
107      assertEquals(expectedRowCount, TEST_UTIL.countRows(table1));
108      assertEquals(0, systemTable.readBulkloadRows(List.of(table1)).size());
109      int rowCountAfterBackup3 = expectedRowCount;
110
111      // Doing another bulk load, to check that this data will disappear after a restore operation
112      performBulkLoad("bulk7");
113      expectedRowCount += ROWS_IN_BULK_LOAD;
114      assertEquals(expectedRowCount, TEST_UTIL.countRows(table1));
115      List<BulkLoad> bulkloadsTemp = systemTable.readBulkloadRows(List.of(table1));
116      assertEquals(1, bulkloadsTemp.size());
117      BulkLoad bulk7 = bulkloadsTemp.get(0);
118
119      // Doing a restore. Overwriting the table implies clearing the bulk loads,
120      // but the loading of restored data involves loading bulk data, we expect 2 bulk loads
121      // associated with backup 3 (loading of full backup, loading of incremental backup).
122      BackupAdmin client = getBackupAdmin();
123      client.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, backup3, false,
124        new TableName[] { table1 }, new TableName[] { table1 }, true));
125      assertEquals(rowCountAfterBackup3, TEST_UTIL.countRows(table1));
126      List<BulkLoad> bulkLoads = systemTable.readBulkloadRows(List.of(table1));
127      assertEquals(2, bulkLoads.size());
128      assertFalse(bulkLoads.contains(bulk7));
129
130      // Check that we have data of all expected bulk loads
131      try (Table restoredTable = TEST_UTIL.getConnection().getTable(table1)) {
132        assertFalse(containsRowWithKey(restoredTable, "bulk1"));
133        assertFalse(containsRowWithKey(restoredTable, "bulk2"));
134        assertTrue(containsRowWithKey(restoredTable, "bulk3"));
135        assertTrue(containsRowWithKey(restoredTable, "bulk4"));
136        assertTrue(containsRowWithKey(restoredTable, "bulk5"));
137        assertTrue(containsRowWithKey(restoredTable, "bulk6"));
138        assertFalse(containsRowWithKey(restoredTable, "bulk7"));
139      }
140    }
141  }
142
143  private boolean containsRowWithKey(Table table, String rowKey) throws IOException {
144    byte[] data = Bytes.toBytes(rowKey);
145    Get get = new Get(data);
146    Result result = table.get(get);
147    return result.containsColumn(famName, qualName);
148  }
149
150  private void performBulkLoad(String keyPrefix) throws IOException {
151    FileSystem fs = TEST_UTIL.getTestFileSystem();
152    Path baseDirectory = TEST_UTIL.getDataTestDirOnTestFS(TEST_NAME);
153    Path hfilePath =
154      new Path(baseDirectory, Bytes.toString(famName) + Path.SEPARATOR + "hfile_" + keyPrefix);
155
156    HFileTestUtil.createHFile(TEST_UTIL.getConfiguration(), fs, hfilePath, famName, qualName,
157      Bytes.toBytes(keyPrefix), Bytes.toBytes(keyPrefix + "z"), ROWS_IN_BULK_LOAD);
158
159    Map<BulkLoadHFiles.LoadQueueItem, ByteBuffer> result =
160      BulkLoadHFiles.create(TEST_UTIL.getConfiguration()).bulkLoad(table1, baseDirectory);
161    assertFalse(result.isEmpty());
162  }
163}