Package org.apache.hadoop.hbase.tool

Class TestBulkLoadHFiles

java.lang.Object

org.apache.hadoop.hbase.tool.TestBulkLoadHFiles

Direct Known Subclasses:

TestBulkLoadHFilesSFT, TestSecureBulkLoadHFiles

public class TestBulkLoadHFiles
extends Object

Test cases for the "load" half of the HFileOutputFormat bulk load functionality. These tests run faster than the full MR cluster tests in TestHFileOutputFormat

Field Summary

Fields

Modifier and Type	Field	Description
static final HBaseClassTestRule	CLASS_RULE
(package private) static final String	EXPECTED_MSG_FOR_NON_EXISTING_FAMILY
private static final byte[]	FAMILY
(package private) static final int	MAX_FILES_PER_REGION_PER_FAMILY
private static final String	NAMESPACE
private static final byte[]	QUALIFIER
private static final byte[][]	SPLIT_KEYS
org.junit.rules.TestName	tn
(package private) static HBaseTestingUtil	util

Constructor Summary

Constructors

Constructor	Description
TestBulkLoadHFiles()

Method Summary

Modifier and Type	Method	Description
private void	addStartEndKeysForTest(TreeMap<byte[],Integer> map, byte[] first, byte[] last)
private org.apache.hadoop.hbase.client.TableDescriptor	buildHTD(org.apache.hadoop.hbase.TableName tableName, org.apache.hadoop.hbase.regionserver.BloomType bloomType)
private static void	createRandomDataFile(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path path, int size)
static int	loadHFiles(String testName, org.apache.hadoop.hbase.client.TableDescriptor htd, HBaseTestingUtil util, byte[] fam, byte[] qual, boolean preCreateTable, byte[][] tableSplitKeys, byte[][][] hfileRanges, boolean useMap, boolean deleteFile, boolean copyFiles, int initRowCount, int factor)
static int	loadHFiles(String testName, org.apache.hadoop.hbase.client.TableDescriptor htd, HBaseTestingUtil util, byte[] fam, byte[] qual, boolean preCreateTable, byte[][] tableSplitKeys, byte[][][] hfileRanges, boolean useMap, boolean deleteFile, boolean copyFiles, int initRowCount, int factor, int depth)
private void	runTest(String testName, org.apache.hadoop.hbase.client.TableDescriptor htd, boolean preCreateTable, byte[][] tableSplitKeys, byte[][][] hfileRanges, boolean useMap, boolean copyFiles, int depth)
private void	runTest(String testName, org.apache.hadoop.hbase.regionserver.BloomType bloomType, byte[][][] hfileRanges)
private void	runTest(String testName, org.apache.hadoop.hbase.regionserver.BloomType bloomType, byte[][][] hfileRanges, boolean useMap)
private void	runTest(String testName, org.apache.hadoop.hbase.regionserver.BloomType bloomType, byte[][] tableSplitKeys, byte[][][] hfileRanges)
private void	runTest(String testName, org.apache.hadoop.hbase.regionserver.BloomType bloomType, byte[][] tableSplitKeys, byte[][][] hfileRanges, boolean useMap)
private void	runTest(String testName, org.apache.hadoop.hbase.TableName tableName, org.apache.hadoop.hbase.regionserver.BloomType bloomType, boolean preCreateTable, byte[][] tableSplitKeys, byte[][][] hfileRanges, boolean useMap, int depth)
static void	setUpBeforeClass()
protected static void	setupNamespace()
static void	tearDownAfterClass()
void	testBulkLoadByFamily()
void	testFailIfNeedSplitHFile()
void	testInferBoundaries()
void	testLoadTooMayHFiles()
void	testNonexistentColumnFamilyLoad()	Test loading into a column family that does not exist.
void	testNonHfileFolder()
private void	testNonHfileFolder(String tableName, boolean preCreateTable)	Write a random data file and a non-file in a dir with a valid family name but not part of the table families.
void	testNonHfileFolderWithUnmatchedFamilyName()
void	testRegionCrossingHFileSplit()	Test case that creates some regions and loads HFiles that cross the boundaries and have different region boundaries than the table pre-split.
private void	testRegionCrossingHFileSplit(org.apache.hadoop.hbase.regionserver.BloomType bloomType)
void	testRegionCrossingHFileSplitRowBloom()	Test case that creates some regions and loads HFiles that cross the boundaries have a ROW bloom filter and a different region boundaries than the table pre-split.
void	testRegionCrossingHFileSplitRowColBloom()	Test case that creates some regions and loads HFiles that cross the boundaries have a ROWCOL bloom filter and a different region boundaries than the table pre-split.
void	testRegionCrossingLoad()	Test case that creates some regions and loads HFiles that cross the boundaries of those regions
void	testRegionCrossingRowBloom()	Test loading into a column family that has a ROW bloom filter.
void	testRegionCrossingRowColBloom()	Test loading into a column family that has a ROWCOL bloom filter.
void	testSimpleHFileSplit()	Test case that creates some regions and loads HFiles that have different region boundaries than the table pre-split.
void	testSimpleLoad()	Test case that creates some regions and loads HFiles that fit snugly inside those regions
void	testSimpleLoadWithFileCopy()
void	testSimpleLoadWithMap()
void	testSplitALot()
void	testSplitStoreFile()
void	testSplitStoreFileWithCreateTimeTS()
private void	testSplitStoreFileWithDifferentEncoding(org.apache.hadoop.hbase.io.encoding.DataBlockEncoding bulkloadEncoding, org.apache.hadoop.hbase.io.encoding.DataBlockEncoding cfEncoding)
void	testSplitStoreFileWithEncodedToEncoded()
void	testSplitStoreFileWithEncodedToNone()
void	testSplitStoreFileWithFavoriteNodes()	Test hfile splits with the favored nodes
void	testSplitStoreFileWithNoneToEncoded()
void	testSplitStoreFileWithNoneToNone()
void	testTableWithCFNameStartWithUnderScore()
void	testTagsSurviveBulkLoadSplit()	Test that tags survive through a bulk load that needs to split hfiles.
void	testWithoutAnExistingTableAndCreateTableSetToNo()
private int	verifyHFile(org.apache.hadoop.fs.Path p)
private void	verifyHFileCreateTimeTS(org.apache.hadoop.fs.Path p)
private void	verifyHFileFavoriteNode(org.apache.hadoop.fs.Path p, org.apache.hadoop.hbase.client.AsyncTableRegionLocator regionLocator, org.apache.hadoop.fs.FileSystem fs)	test split storefile with favorite node information

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Details

CLASS_RULE

public static final HBaseClassTestRule CLASS_RULE

tn

public org.junit.rules.TestName tn

QUALIFIER

private static final byte[] QUALIFIER

FAMILY

private static final byte[] FAMILY

NAMESPACE

private static final String NAMESPACE

See Also:

• Constant Field Values

EXPECTED_MSG_FOR_NON_EXISTING_FAMILY

static final String EXPECTED_MSG_FOR_NON_EXISTING_FAMILY

See Also:

• Constant Field Values

MAX_FILES_PER_REGION_PER_FAMILY

static final int MAX_FILES_PER_REGION_PER_FAMILY

See Also:

• Constant Field Values

SPLIT_KEYS

private static final byte[][] SPLIT_KEYS

util

static HBaseTestingUtil util

Constructor Details

TestBulkLoadHFiles

public TestBulkLoadHFiles()

Method Details

setUpBeforeClass

public static void setUpBeforeClass()
                             throws Exception

Throws:

Exception

setupNamespace

protected static void setupNamespace()
                              throws Exception

Throws:

Exception

tearDownAfterClass

public static void tearDownAfterClass()
                               throws Exception

Throws:

Exception

testSimpleLoadWithMap

public void testSimpleLoadWithMap()
                           throws Exception

Throws:

Exception

testSimpleLoad

public void testSimpleLoad()
                    throws Exception

Test case that creates some regions and loads HFiles that fit snugly inside those regions

Throws:

Exception

testSimpleLoadWithFileCopy

public void testSimpleLoadWithFileCopy()
                                throws Exception

Throws:

Exception

testRegionCrossingLoad

public void testRegionCrossingLoad()
                            throws Exception

Test case that creates some regions and loads HFiles that cross the boundaries of those regions

Throws:

Exception

testRegionCrossingRowBloom

public void testRegionCrossingRowBloom()
                                throws Exception

Test loading into a column family that has a ROW bloom filter.

Throws:

Exception

testRegionCrossingRowColBloom

public void testRegionCrossingRowColBloom()
                                   throws Exception

Test loading into a column family that has a ROWCOL bloom filter.

Throws:

Exception

testSimpleHFileSplit

public void testSimpleHFileSplit()
                          throws Exception

Test case that creates some regions and loads HFiles that have different region boundaries than the table pre-split.

Throws:

Exception

testRegionCrossingHFileSplit

public void testRegionCrossingHFileSplit()
                                  throws Exception

Test case that creates some regions and loads HFiles that cross the boundaries and have different region boundaries than the table pre-split.

Throws:

Exception

testRegionCrossingHFileSplitRowBloom

public void testRegionCrossingHFileSplitRowBloom()
                                          throws Exception

Test case that creates some regions and loads HFiles that cross the boundaries have a ROW bloom filter and a different region boundaries than the table pre-split.

Throws:

Exception

testRegionCrossingHFileSplitRowColBloom

public void testRegionCrossingHFileSplitRowColBloom()
                                             throws Exception

Test case that creates some regions and loads HFiles that cross the boundaries have a ROWCOL bloom filter and a different region boundaries than the table pre-split.

Throws:

Exception

testSplitALot

public void testSplitALot()
                   throws Exception

Throws:

Exception

testRegionCrossingHFileSplit

private void testRegionCrossingHFileSplit(org.apache.hadoop.hbase.regionserver.BloomType bloomType)
                                   throws Exception

Throws:

Exception

buildHTD

private org.apache.hadoop.hbase.client.TableDescriptor buildHTD(org.apache.hadoop.hbase.TableName tableName,
 org.apache.hadoop.hbase.regionserver.BloomType bloomType)

runTest

private void runTest(String testName,
 org.apache.hadoop.hbase.regionserver.BloomType bloomType,
 byte[][][] hfileRanges)
              throws Exception

Throws:

Exception

runTest

private void runTest(String testName,
 org.apache.hadoop.hbase.regionserver.BloomType bloomType,
 byte[][][] hfileRanges,
 boolean useMap)
              throws Exception

Throws:

Exception

runTest

private void runTest(String testName,
 org.apache.hadoop.hbase.regionserver.BloomType bloomType,
 byte[][] tableSplitKeys,
 byte[][][] hfileRanges)
              throws Exception

Throws:

Exception

runTest

private void runTest(String testName,
 org.apache.hadoop.hbase.regionserver.BloomType bloomType,
 byte[][] tableSplitKeys,
 byte[][][] hfileRanges,
 boolean useMap)
              throws Exception

Throws:

Exception

runTest

private void runTest(String testName,
 org.apache.hadoop.hbase.TableName tableName,
 org.apache.hadoop.hbase.regionserver.BloomType bloomType,
 boolean preCreateTable,
 byte[][] tableSplitKeys,
 byte[][][] hfileRanges,
 boolean useMap,
 int depth)
              throws Exception

Throws:

Exception

loadHFiles

public static int loadHFiles(String testName,
 org.apache.hadoop.hbase.client.TableDescriptor htd,
 HBaseTestingUtil util,
 byte[] fam,
 byte[] qual,
 boolean preCreateTable,
 byte[][] tableSplitKeys,
 byte[][][] hfileRanges,
 boolean useMap,
 boolean deleteFile,
 boolean copyFiles,
 int initRowCount,
 int factor)
                      throws Exception

Throws:

Exception

loadHFiles

public static int loadHFiles(String testName,
 org.apache.hadoop.hbase.client.TableDescriptor htd,
 HBaseTestingUtil util,
 byte[] fam,
 byte[] qual,
 boolean preCreateTable,
 byte[][] tableSplitKeys,
 byte[][][] hfileRanges,
 boolean useMap,
 boolean deleteFile,
 boolean copyFiles,
 int initRowCount,
 int factor,
 int depth)
                      throws Exception

Throws:

Exception

runTest

private void runTest(String testName,
 org.apache.hadoop.hbase.client.TableDescriptor htd,
 boolean preCreateTable,
 byte[][] tableSplitKeys,
 byte[][][] hfileRanges,
 boolean useMap,
 boolean copyFiles,
 int depth)
              throws Exception

Throws:

Exception

testTagsSurviveBulkLoadSplit

public void testTagsSurviveBulkLoadSplit()
                                  throws Exception

Test that tags survive through a bulk load that needs to split hfiles. This test depends on the "hbase.client.rpc.codec" = KeyValueCodecWithTags so that the client can get tags in the responses.

Throws:

Exception

testNonexistentColumnFamilyLoad

public void testNonexistentColumnFamilyLoad()
                                     throws Exception

Test loading into a column family that does not exist.

Throws:

Exception

testNonHfileFolderWithUnmatchedFamilyName

public void testNonHfileFolderWithUnmatchedFamilyName()
                                               throws Exception

Throws:

Exception

testNonHfileFolder

public void testNonHfileFolder()
                        throws Exception

Throws:

Exception

testNonHfileFolder

private void testNonHfileFolder(String tableName,
 boolean preCreateTable)
                         throws Exception

Write a random data file and a non-file in a dir with a valid family name but not part of the table families. we should we able to bulkload without getting the unmatched family exception. HBASE-13037/HBASE-13227

Throws:

Exception

createRandomDataFile

private static void createRandomDataFile(org.apache.hadoop.fs.FileSystem fs,
 org.apache.hadoop.fs.Path path,
 int size)
                                  throws IOException

Throws:

IOException

testSplitStoreFile

public void testSplitStoreFile()
                        throws IOException

Throws:

IOException

testSplitStoreFileWithFavoriteNodes

public void testSplitStoreFileWithFavoriteNodes()
                                         throws IOException

Test hfile splits with the favored nodes

Throws:

IOException

testSplitStoreFileWithCreateTimeTS

public void testSplitStoreFileWithCreateTimeTS()
                                        throws IOException

Throws:

IOException

testSplitStoreFileWithNoneToNone

public void testSplitStoreFileWithNoneToNone()
                                      throws IOException

Throws:

IOException

testSplitStoreFileWithEncodedToEncoded

public void testSplitStoreFileWithEncodedToEncoded()
                                            throws IOException

Throws:

IOException

testSplitStoreFileWithEncodedToNone

public void testSplitStoreFileWithEncodedToNone()
                                         throws IOException

Throws:

IOException

testSplitStoreFileWithNoneToEncoded

public void testSplitStoreFileWithNoneToEncoded()
                                         throws IOException

Throws:

IOException

testSplitStoreFileWithDifferentEncoding

private void testSplitStoreFileWithDifferentEncoding(org.apache.hadoop.hbase.io.encoding.DataBlockEncoding bulkloadEncoding,
 org.apache.hadoop.hbase.io.encoding.DataBlockEncoding cfEncoding)
                                              throws IOException

Throws:

IOException

verifyHFile

private int verifyHFile(org.apache.hadoop.fs.Path p)
                 throws IOException

Throws:

IOException

verifyHFileCreateTimeTS

private void verifyHFileCreateTimeTS(org.apache.hadoop.fs.Path p)
                              throws IOException

Throws:

IOException

verifyHFileFavoriteNode

private void verifyHFileFavoriteNode(org.apache.hadoop.fs.Path p,
 org.apache.hadoop.hbase.client.AsyncTableRegionLocator regionLocator,
 org.apache.hadoop.fs.FileSystem fs)
                              throws IOException

test split storefile with favorite node information

Throws:

IOException

addStartEndKeysForTest

private void addStartEndKeysForTest(TreeMap<byte[],Integer> map,
 byte[] first,
 byte[] last)

testInferBoundaries

public void testInferBoundaries()

testLoadTooMayHFiles

public void testLoadTooMayHFiles()
                          throws Exception

Throws:

Exception

testWithoutAnExistingTableAndCreateTableSetToNo

public void testWithoutAnExistingTableAndCreateTableSetToNo()
                                                     throws Exception

Throws:

Exception

testTableWithCFNameStartWithUnderScore

public void testTableWithCFNameStartWithUnderScore()
                                            throws Exception

Throws:

Exception

testBulkLoadByFamily

public void testBulkLoadByFamily()
                          throws Exception

Throws:

Exception

testFailIfNeedSplitHFile

public void testFailIfNeedSplitHFile()
                              throws IOException

Throws:

IOException