001/*
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hbase.client;
019
020import static org.junit.Assert.assertEquals;
021import static org.junit.Assert.assertTrue;
022
023import java.util.ArrayList;
024import java.util.Collections;
025import java.util.List;
026import org.apache.hadoop.conf.Configuration;
027import org.apache.hadoop.fs.FileSystem;
028import org.apache.hadoop.fs.Path;
029import org.apache.hadoop.hbase.HBaseClassTestRule;
030import org.apache.hadoop.hbase.HBaseTestingUtil;
031import org.apache.hadoop.hbase.HConstants;
032import org.apache.hadoop.hbase.TableName;
033import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
034import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
035import org.apache.hadoop.hbase.regionserver.BloomType;
036import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
037import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
038import org.apache.hadoop.hbase.testclassification.ClientTests;
039import org.apache.hadoop.hbase.testclassification.MediumTests;
040import org.apache.hadoop.hbase.util.Bytes;
041import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
042import org.junit.After;
043import org.junit.AfterClass;
044import org.junit.Before;
045import org.junit.BeforeClass;
046import org.junit.ClassRule;
047import org.junit.Test;
048import org.junit.experimental.categories.Category;
049import org.slf4j.Logger;
050import org.slf4j.LoggerFactory;
051
052/**
053 * Test class to verify that metadata is consistent before and after a snapshot attempt.
054 */
055@Category({ MediumTests.class, ClientTests.class })
056public class TestSnapshotMetadata {
057
058  @ClassRule
059  public static final HBaseClassTestRule CLASS_RULE =
060    HBaseClassTestRule.forClass(TestSnapshotMetadata.class);
061
062  private static final Logger LOG = LoggerFactory.getLogger(TestSnapshotMetadata.class);
063
064  private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
065  private static final int NUM_RS = 2;
066  private static final String STRING_TABLE_NAME = "TestSnapshotMetadata";
067
068  private static final String MAX_VERSIONS_FAM_STR = "fam_max_columns";
069  private static final byte[] MAX_VERSIONS_FAM = Bytes.toBytes(MAX_VERSIONS_FAM_STR);
070
071  private static final String COMPRESSED_FAM_STR = "fam_compressed";
072  private static final byte[] COMPRESSED_FAM = Bytes.toBytes(COMPRESSED_FAM_STR);
073
074  private static final String BLOCKSIZE_FAM_STR = "fam_blocksize";
075  private static final byte[] BLOCKSIZE_FAM = Bytes.toBytes(BLOCKSIZE_FAM_STR);
076
077  private static final String BLOOMFILTER_FAM_STR = "fam_bloomfilter";
078  private static final byte[] BLOOMFILTER_FAM = Bytes.toBytes(BLOOMFILTER_FAM_STR);
079
080  private static final String TEST_CONF_CUSTOM_VALUE = "TestCustomConf";
081  private static final String TEST_CUSTOM_VALUE = "TestCustomValue";
082
083  private static final byte[][] families =
084    { MAX_VERSIONS_FAM, BLOOMFILTER_FAM, COMPRESSED_FAM, BLOCKSIZE_FAM };
085
086  private static final DataBlockEncoding DATA_BLOCK_ENCODING_TYPE = DataBlockEncoding.FAST_DIFF;
087  private static final BloomType BLOOM_TYPE = BloomType.ROW;
088  private static final int BLOCK_SIZE = 98;
089  private static final int MAX_VERSIONS = 8;
090
091  private Admin admin;
092  private String originalTableDescription;
093  private TableDescriptor originalTableDescriptor;
094  TableName originalTableName;
095
096  private static FileSystem fs;
097  private static Path rootDir;
098
099  @BeforeClass
100  public static void setupCluster() throws Exception {
101    setupConf(UTIL.getConfiguration());
102    UTIL.startMiniCluster(NUM_RS);
103
104    fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
105    rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
106  }
107
108  @AfterClass
109  public static void cleanupTest() throws Exception {
110    try {
111      UTIL.shutdownMiniCluster();
112    } catch (Exception e) {
113      LOG.warn("failure shutting down cluster", e);
114    }
115  }
116
117  private static void setupConf(Configuration conf) {
118    // enable snapshot support
119    conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
120    // disable the ui
121    conf.setInt("hbase.regionsever.info.port", -1);
122    // change the flush size to a small amount, regulating number of store files
123    conf.setInt("hbase.hregion.memstore.flush.size", 25000);
124    // so make sure we get a compaction when doing a load, but keep around
125    // some files in the store
126    conf.setInt("hbase.hstore.compaction.min", 10);
127    conf.setInt("hbase.hstore.compactionThreshold", 10);
128    // block writes if we get to 12 store files
129    conf.setInt("hbase.hstore.blockingStoreFiles", 12);
130    conf.setInt("hbase.regionserver.msginterval", 100);
131    conf.setBoolean("hbase.master.enabletable.roundrobin", true);
132    // Avoid potentially aggressive splitting which would cause snapshot to fail
133    conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
134      ConstantSizeRegionSplitPolicy.class.getName());
135  }
136
137  @Before
138  public void setup() throws Exception {
139    admin = UTIL.getAdmin();
140    createTableWithNonDefaultProperties();
141  }
142
143  @After
144  public void tearDown() throws Exception {
145    SnapshotTestingUtils.deleteAllSnapshots(admin);
146  }
147
148  /*
149   * Create a table that has non-default properties so we can see if they hold
150   */
151  private void createTableWithNonDefaultProperties() throws Exception {
152    final long startTime = EnvironmentEdgeManager.currentTime();
153    final String sourceTableNameAsString = STRING_TABLE_NAME + startTime;
154    originalTableName = TableName.valueOf(sourceTableNameAsString);
155
156    // enable replication on a column family
157    ColumnFamilyDescriptor maxVersionsColumn = ColumnFamilyDescriptorBuilder
158      .newBuilder(MAX_VERSIONS_FAM).setMaxVersions(MAX_VERSIONS).build();
159    ColumnFamilyDescriptor bloomFilterColumn = ColumnFamilyDescriptorBuilder
160      .newBuilder(BLOOMFILTER_FAM).setBloomFilterType(BLOOM_TYPE).build();
161    ColumnFamilyDescriptor dataBlockColumn = ColumnFamilyDescriptorBuilder
162      .newBuilder(COMPRESSED_FAM).setDataBlockEncoding(DATA_BLOCK_ENCODING_TYPE).build();
163    ColumnFamilyDescriptor blockSizeColumn =
164      ColumnFamilyDescriptorBuilder.newBuilder(BLOCKSIZE_FAM).setBlocksize(BLOCK_SIZE).build();
165
166    TableDescriptor tableDescriptor = TableDescriptorBuilder
167      .newBuilder(TableName.valueOf(sourceTableNameAsString)).setColumnFamily(maxVersionsColumn)
168      .setColumnFamily(bloomFilterColumn).setColumnFamily(dataBlockColumn)
169      .setColumnFamily(blockSizeColumn).setValue(TEST_CUSTOM_VALUE, TEST_CUSTOM_VALUE)
170      .setValue(TEST_CONF_CUSTOM_VALUE, TEST_CONF_CUSTOM_VALUE).build();
171    assertTrue(tableDescriptor.getValues().size() > 0);
172
173    admin.createTable(tableDescriptor);
174    Table original = UTIL.getConnection().getTable(originalTableName);
175    originalTableName = TableName.valueOf(sourceTableNameAsString);
176    originalTableDescriptor = admin.getDescriptor(originalTableName);
177    originalTableDescription = originalTableDescriptor.toStringCustomizedValues();
178
179    original.close();
180  }
181
182  /**
183   * Verify that the describe for a cloned table matches the describe from the original.
184   */
185  @Test
186  public void testDescribeMatchesAfterClone() throws Exception {
187    // Clone the original table
188    final String clonedTableNameAsString = "clone" + originalTableName;
189    final TableName clonedTableName = TableName.valueOf(clonedTableNameAsString);
190    final String snapshotNameAsString =
191      "snapshot" + originalTableName + EnvironmentEdgeManager.currentTime();
192    final String snapshotName = snapshotNameAsString;
193
194    // restore the snapshot into a cloned table and examine the output
195    List<byte[]> familiesList = new ArrayList<>();
196    Collections.addAll(familiesList, families);
197
198    // Create a snapshot in which all families are empty
199    SnapshotTestingUtils.createSnapshotAndValidate(admin, originalTableName, null, familiesList,
200      snapshotNameAsString, rootDir, fs, /* onlineSnapshot= */ false);
201
202    admin.cloneSnapshot(snapshotName, clonedTableName);
203    Table clonedTable = UTIL.getConnection().getTable(clonedTableName);
204    TableDescriptor cloneHtd = admin.getDescriptor(clonedTableName);
205    assertEquals(originalTableDescription.replace(originalTableName.getNameAsString(),
206      clonedTableNameAsString), cloneHtd.toStringCustomizedValues());
207
208    // Verify the custom fields
209    assertEquals(originalTableDescriptor.getValues().size(), cloneHtd.getValues().size());
210    assertEquals(TEST_CUSTOM_VALUE, cloneHtd.getValue(TEST_CUSTOM_VALUE));
211    assertEquals(TEST_CONF_CUSTOM_VALUE, cloneHtd.getValue(TEST_CONF_CUSTOM_VALUE));
212    assertEquals(originalTableDescriptor.getValues(), cloneHtd.getValues());
213
214    admin.enableTable(originalTableName);
215    clonedTable.close();
216  }
217
218  /**
219   * Verify that the describe for a restored table matches the describe for one the original.
220   */
221  @Test
222  public void testDescribeMatchesAfterRestore() throws Exception {
223    runRestoreWithAdditionalMetadata(false);
224  }
225
226  /**
227   * Verify that if metadata changed after a snapshot was taken, that the old metadata replaces the
228   * new metadata during a restore
229   */
230  @Test
231  public void testDescribeMatchesAfterMetadataChangeAndRestore() throws Exception {
232    runRestoreWithAdditionalMetadata(true);
233  }
234
235  /**
236   * Verify that when the table is empty, making metadata changes after the restore does not affect
237   * the restored table's original metadata
238   */
239  @Test
240  public void testDescribeOnEmptyTableMatchesAfterMetadataChangeAndRestore() throws Exception {
241    runRestoreWithAdditionalMetadata(true, false);
242  }
243
244  private void runRestoreWithAdditionalMetadata(boolean changeMetadata) throws Exception {
245    runRestoreWithAdditionalMetadata(changeMetadata, true);
246  }
247
248  private void runRestoreWithAdditionalMetadata(boolean changeMetadata, boolean addData)
249    throws Exception {
250
251    if (admin.isTableDisabled(originalTableName)) {
252      admin.enableTable(originalTableName);
253    }
254
255    // populate it with data
256    final byte[] familyForUpdate = BLOCKSIZE_FAM;
257
258    List<byte[]> familiesWithDataList = new ArrayList<>();
259    List<byte[]> emptyFamiliesList = new ArrayList<>();
260    if (addData) {
261      Table original = UTIL.getConnection().getTable(originalTableName);
262      UTIL.loadTable(original, familyForUpdate); // family arbitrarily chosen
263      original.close();
264
265      for (byte[] family : families) {
266        if (family != familyForUpdate) {
267          emptyFamiliesList.add(family);
268        }
269      }
270      familiesWithDataList.add(familyForUpdate);
271    } else {
272      Collections.addAll(emptyFamiliesList, families);
273    }
274
275    // take a "disabled" snapshot
276    final String snapshotNameAsString =
277      "snapshot" + originalTableName + EnvironmentEdgeManager.currentTime();
278
279    SnapshotTestingUtils.createSnapshotAndValidate(admin, originalTableName, familiesWithDataList,
280      emptyFamiliesList, snapshotNameAsString, rootDir, fs, /* onlineSnapshot= */ false);
281
282    admin.enableTable(originalTableName);
283
284    if (changeMetadata) {
285      final String newFamilyNameAsString = "newFamily" + EnvironmentEdgeManager.currentTime();
286      final byte[] newFamilyName = Bytes.toBytes(newFamilyNameAsString);
287
288      admin.disableTable(originalTableName);
289      ColumnFamilyDescriptor familyDescriptor = ColumnFamilyDescriptorBuilder.of(newFamilyName);
290      admin.addColumnFamily(originalTableName, familyDescriptor);
291      assertTrue("New column family was not added.",
292        admin.getDescriptor(originalTableName).toString().contains(newFamilyNameAsString));
293    }
294
295    // restore it
296    if (!admin.isTableDisabled(originalTableName)) {
297      admin.disableTable(originalTableName);
298    }
299
300    admin.restoreSnapshot(snapshotNameAsString);
301    admin.enableTable(originalTableName);
302
303    // verify that the descrption is reverted
304    try (Table original = UTIL.getConnection().getTable(originalTableName)) {
305      assertEquals(originalTableDescriptor, admin.getDescriptor(originalTableName));
306      assertEquals(originalTableDescriptor, original.getDescriptor());
307    }
308  }
309}