Package org.apache.hadoop.hbase.backup

Class TestHFileArchiving

java.lang.Object

org.apache.hadoop.hbase.backup.TestHFileArchiving

public class TestHFileArchiving
extends Object

Test that the HFileArchiver correctly removes all the parts of a region when cleaning up a region

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
private static interface	TestHFileArchiving.ArchivingFunction<Configuration,FS,Region,Dir,Family,Files>

Field Summary

Fields

Modifier and Type	Field	Description
static final HBaseClassTestRule	CLASS_RULE
private static final org.slf4j.Logger	LOG
org.junit.rules.TestName	name
private static org.apache.hadoop.hbase.master.cleaner.DirScanPool	POOL
private static final byte[]	TEST_FAM
private static final HBaseTestingUtil	UTIL

Constructor Summary

Constructors

Constructor	Description
TestHFileArchiving()

Method Summary

Modifier and Type	Method	Description
private void	assertArchiveFiles(org.apache.hadoop.fs.FileSystem fs, List<String> storeFiles, long timeout)
static void	cleanupTest()
private void	clearArchiveDirectory()
private List<String>	getAllFileNames(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path archiveDir)	Get the names of all the files below the given directory
private org.apache.hadoop.hbase.master.cleaner.HFileCleaner	getHFileCleaner(org.apache.hadoop.hbase.Stoppable stoppable, org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path archiveDir)
private List<org.apache.hadoop.hbase.regionserver.HRegion>	initTableForArchivingRegions(org.apache.hadoop.hbase.TableName tableName)
private List<String>	recurseOnFiles(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.FileStatus[] files, List<String> fileNames)	Recursively lookup all the file names under the file[] array
static void	setupCluster()	Setup the config for the cluster
private static void	setupConf(org.apache.hadoop.conf.Configuration conf)
void	tearDown()
void	testArchiveOnTableDelete()
void	testArchiveOnTableFamilyDelete()	Test that the store files are archived when a column family is removed.
void	testArchiveRecoveredEditsWalDirDifferentFS()
void	testArchiveRecoveredEditsWalDirNull()
private void	testArchiveRecoveredEditsWalDirNullOrSame(String walDir)
void	testArchiveRecoveredEditsWalDirSameFsStoreFiles()
void	testArchiveRecoveredEditsWrongFS()
void	testArchiveRegions()
void	testArchiveRegionsWhenPermissionDenied()
void	testArchiveRegionTableAndRegionDirsNull()
void	testArchiveRegionWithRegionDirNull()
void	testArchiveRegionWithTableDirNull()
private void	testArchiveStoreFilesDifferentFileSystems(String walDir, String expectedBase, boolean archiveFileExists, boolean sourceFileExists, boolean archiveFileDifferentLength, TestHFileArchiving.ArchivingFunction<org.apache.hadoop.conf.Configuration,org.apache.hadoop.fs.FileSystem,org.apache.hadoop.hbase.client.RegionInfo,org.apache.hadoop.fs.Path,byte[],Collection<org.apache.hadoop.hbase.regionserver.HStoreFile>> archivingFunction)
private void	testArchiveStoreFilesDifferentFileSystems(String walDir, String expectedBase, TestHFileArchiving.ArchivingFunction<org.apache.hadoop.conf.Configuration,org.apache.hadoop.fs.FileSystem,org.apache.hadoop.hbase.client.RegionInfo,org.apache.hadoop.fs.Path,byte[],Collection<org.apache.hadoop.hbase.regionserver.HStoreFile>> archivingFunction)
void	testArchiveStoreFilesDifferentFileSystemsArchiveFileMatchCurrent()
void	testArchiveStoreFilesDifferentFileSystemsArchiveFileMismatch()
void	testArchiveStoreFilesDifferentFileSystemsFileAlreadyArchived()
void	testArchiveStoreFilesDifferentFileSystemsWallAndRootSame()
void	testArchiveStoreFilesDifferentFileSystemsWallNullPlainRoot()
void	testArchiveStoreFilesDifferentFileSystemsWallWithSchemaPlainRoot()
void	testCleaningRace()	Test HFileArchiver.resolveAndArchive() race condition HBASE-7643
void	testDeleteRegionWithNoStoreFiles()	Test that the region directory is removed when we archive a region without store files, but still has hidden files.
void	testRemoveRegionDirOnArchive()

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Details

CLASS_RULE

public static final HBaseClassTestRule CLASS_RULE

LOG

private static final org.slf4j.Logger LOG

UTIL

private static final HBaseTestingUtil UTIL

TEST_FAM

private static final byte[] TEST_FAM

POOL

private static org.apache.hadoop.hbase.master.cleaner.DirScanPool POOL

name

public org.junit.rules.TestName name

Constructor Details

TestHFileArchiving

public TestHFileArchiving()

Method Details

setupCluster

public static void setupCluster()
                         throws Exception

Setup the config for the cluster

Throws:

Exception

setupConf

private static void setupConf(org.apache.hadoop.conf.Configuration conf)

tearDown

public void tearDown()
              throws Exception

Throws:

Exception

cleanupTest

public static void cleanupTest()
                        throws Exception

Throws:

Exception

testArchiveStoreFilesDifferentFileSystemsWallWithSchemaPlainRoot

public void testArchiveStoreFilesDifferentFileSystemsWallWithSchemaPlainRoot()
                                                                      throws Exception

Throws:

Exception

testArchiveStoreFilesDifferentFileSystemsWallNullPlainRoot

public void testArchiveStoreFilesDifferentFileSystemsWallNullPlainRoot()
                                                                throws Exception

Throws:

Exception

testArchiveStoreFilesDifferentFileSystemsWallAndRootSame

public void testArchiveStoreFilesDifferentFileSystemsWallAndRootSame()
                                                              throws Exception

Throws:

Exception

testArchiveStoreFilesDifferentFileSystemsFileAlreadyArchived

public void testArchiveStoreFilesDifferentFileSystemsFileAlreadyArchived()
                                                                  throws Exception

Throws:

Exception

testArchiveStoreFilesDifferentFileSystemsArchiveFileMatchCurrent

public void testArchiveStoreFilesDifferentFileSystemsArchiveFileMatchCurrent()
                                                                      throws Exception

Throws:

Exception

testArchiveStoreFilesDifferentFileSystemsArchiveFileMismatch

public void testArchiveStoreFilesDifferentFileSystemsArchiveFileMismatch()
                                                                  throws Exception

Throws:

Exception

testArchiveStoreFilesDifferentFileSystems

private void testArchiveStoreFilesDifferentFileSystems(String walDir,
 String expectedBase,
 TestHFileArchiving.ArchivingFunction<org.apache.hadoop.conf.Configuration,org.apache.hadoop.fs.FileSystem,org.apache.hadoop.hbase.client.RegionInfo,org.apache.hadoop.fs.Path,byte[],Collection<org.apache.hadoop.hbase.regionserver.HStoreFile>> archivingFunction)
                                                throws IOException

Throws:

IOException

testArchiveStoreFilesDifferentFileSystems

private void testArchiveStoreFilesDifferentFileSystems(String walDir,
 String expectedBase,
 boolean archiveFileExists,
 boolean sourceFileExists,
 boolean archiveFileDifferentLength,
 TestHFileArchiving.ArchivingFunction<org.apache.hadoop.conf.Configuration,org.apache.hadoop.fs.FileSystem,org.apache.hadoop.hbase.client.RegionInfo,org.apache.hadoop.fs.Path,byte[],Collection<org.apache.hadoop.hbase.regionserver.HStoreFile>> archivingFunction)
                                                throws IOException

Throws:

IOException

testArchiveRecoveredEditsWalDirNull

public void testArchiveRecoveredEditsWalDirNull()
                                         throws Exception

Throws:

Exception

testArchiveRecoveredEditsWalDirSameFsStoreFiles

public void testArchiveRecoveredEditsWalDirSameFsStoreFiles()
                                                     throws Exception

Throws:

Exception

testArchiveRecoveredEditsWalDirNullOrSame

private void testArchiveRecoveredEditsWalDirNullOrSame(String walDir)
                                                throws Exception

Throws:

Exception

testArchiveRecoveredEditsWrongFS

public void testArchiveRecoveredEditsWrongFS()
                                      throws Exception

Throws:

Exception

testArchiveRecoveredEditsWalDirDifferentFS

public void testArchiveRecoveredEditsWalDirDifferentFS()
                                                throws Exception

Throws:

Exception

testRemoveRegionDirOnArchive

public void testRemoveRegionDirOnArchive()
                                  throws Exception

Throws:

Exception

testDeleteRegionWithNoStoreFiles

public void testDeleteRegionWithNoStoreFiles()
                                      throws IOException

Test that the region directory is removed when we archive a region without store files, but still has hidden files.

Throws:

IOException - throws an IOException if there's problem creating a table or if there's an issue with accessing FileSystem.

initTableForArchivingRegions

private List<org.apache.hadoop.hbase.regionserver.HRegion> initTableForArchivingRegions(org.apache.hadoop.hbase.TableName tableName)
                                                                                 throws IOException

Throws:

IOException

testArchiveRegions

public void testArchiveRegions()
                        throws Exception

Throws:

Exception

testArchiveRegionsWhenPermissionDenied

public void testArchiveRegionsWhenPermissionDenied()
                                            throws Exception

Throws:

Exception

testArchiveOnTableDelete

public void testArchiveOnTableDelete()
                              throws Exception

Throws:

Exception

assertArchiveFiles

private void assertArchiveFiles(org.apache.hadoop.fs.FileSystem fs,
 List<String> storeFiles,
 long timeout)
                         throws IOException

Throws:

IOException

testArchiveOnTableFamilyDelete

public void testArchiveOnTableFamilyDelete()
                                    throws IOException,
InterruptedException

Test that the store files are archived when a column family is removed.

Throws:

IOException - if there's a problem creating a table.

InterruptedException - problem getting a RegionServer.

testCleaningRace

public void testCleaningRace()
                      throws Exception

Test HFileArchiver.resolveAndArchive() race condition HBASE-7643

Throws:

Exception

testArchiveRegionTableAndRegionDirsNull

public void testArchiveRegionTableAndRegionDirsNull()
                                             throws IOException

Throws:

IOException

testArchiveRegionWithTableDirNull

public void testArchiveRegionWithTableDirNull()
                                       throws IOException

Throws:

IOException

testArchiveRegionWithRegionDirNull

public void testArchiveRegionWithRegionDirNull()
                                        throws IOException

Throws:

IOException

getHFileCleaner

private org.apache.hadoop.hbase.master.cleaner.HFileCleaner getHFileCleaner(org.apache.hadoop.hbase.Stoppable stoppable,
 org.apache.hadoop.conf.Configuration conf,
 org.apache.hadoop.fs.FileSystem fs,
 org.apache.hadoop.fs.Path archiveDir)
                                                                     throws IOException

Throws:

IOException

clearArchiveDirectory

private void clearArchiveDirectory()
                            throws IOException

Throws:

IOException

getAllFileNames

private List<String> getAllFileNames(org.apache.hadoop.fs.FileSystem fs,
 org.apache.hadoop.fs.Path archiveDir)
                              throws IOException

Get the names of all the files below the given directory

Parameters:

fs - the file system to inspect

archiveDir - the directory in which to look

Returns:

a list of all files in the directory and sub-directories

Throws:

IOException - throws IOException in case FS is unavailable

recurseOnFiles

private List<String> recurseOnFiles(org.apache.hadoop.fs.FileSystem fs,
 org.apache.hadoop.fs.FileStatus[] files,
 List<String> fileNames)
                             throws IOException

Recursively lookup all the file names under the file[] array

Throws:

IOException