Package org.apache.hadoop.hbase.mapreduce

Class TestHFileOutputFormat2

java.lang.Object

org.apache.hadoop.hbase.mapreduce.TestHFileOutputFormat2

public class TestHFileOutputFormat2
extends Object

Simple test for HFileOutputFormat2. Sets up and runs a mapreduce job that writes hfile output. Creates a few inner classes to implement splits and an inputformat that emits keys and values.

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
private static class	TestHFileOutputFormat2.ConfigurationCaptorConnection
(package private) static class	TestHFileOutputFormat2.RandomKVGeneratingMapper	Simple mapper that makes KeyValue output.
(package private) static class	TestHFileOutputFormat2.RandomPutGeneratingMapper	Simple mapper that makes Put output.

Field Summary

Fields

Modifier and Type	Field	Description
static final HBaseClassTestRule	CLASS_RULE
private static final int	DEFAULT_VALUE_LENGTH
private static final byte[][]	FAMILIES
static final byte[]	FAMILY_NAME
private static final org.slf4j.Logger	LOG
private static final int	ROWSPERSPLIT
private static final org.apache.hadoop.hbase.TableName[]	TABLE_NAMES
private HBaseTestingUtil	util

Constructor Summary

Constructors

Constructor	Description
TestHFileOutputFormat2()

Method Summary

Modifier and Type	Method	Description
private org.apache.hadoop.mapreduce.TaskAttemptContext	createTestTaskAttemptContext(org.apache.hadoop.mapreduce.Job job)
private void	doIncrementalLoadTest(boolean shouldChangeRegions, boolean shouldKeepLocality, boolean putSortReducer, String tableStr)
private void	doIncrementalLoadTest(boolean shouldChangeRegions, boolean shouldKeepLocality, boolean putSortReducer, List<String> tableStr)
static byte[]	generateData(Random r, int length)
private byte[][]	generateRandomSplitKeys(int numKeys)
private byte[][]	generateRandomStartKeys(int numKeys)
private Map<String,Integer>	getMockColumnFamiliesForBlockSize(int numCfs)
private Map<String,org.apache.hadoop.hbase.regionserver.BloomType>	getMockColumnFamiliesForBloomType(int numCfs)
private Map<String,org.apache.hadoop.hbase.io.compress.Compression.Algorithm>	getMockColumnFamiliesForCompression(int numCfs)
private Map<String,org.apache.hadoop.hbase.io.encoding.DataBlockEncoding>	getMockColumnFamiliesForDataBlockEncoding(int numCfs)
private String	getStoragePolicyName(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path path)
private String	getStoragePolicyNameForOldHDFSVersion(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path path)
static void	main(String[] args)
void	manualTest(String[] args)
private void	quickPoll(Callable<Boolean> c, int waitMs)
private void	runIncrementalPELoad(org.apache.hadoop.conf.Configuration conf, List<org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2.TableInfo> tableInfo, org.apache.hadoop.fs.Path outDir, boolean putSortReducer)
private void	setupMockColumnFamiliesForBlockSize(org.apache.hadoop.hbase.client.Table table, Map<String,Integer> familyToDataBlockEncoding)
private void	setupMockColumnFamiliesForBloomType(org.apache.hadoop.hbase.client.Table table, Map<String,org.apache.hadoop.hbase.regionserver.BloomType> familyToDataBlockEncoding)
private void	setupMockColumnFamiliesForCompression(org.apache.hadoop.hbase.client.Table table, Map<String,org.apache.hadoop.hbase.io.compress.Compression.Algorithm> familyToCompression)
private void	setupMockColumnFamiliesForDataBlockEncoding(org.apache.hadoop.hbase.client.Table table, Map<String,org.apache.hadoop.hbase.io.encoding.DataBlockEncoding> familyToDataBlockEncoding)
private void	setupMockStartKeys(org.apache.hadoop.hbase.client.RegionLocator table)
private void	setupMockTableName(org.apache.hadoop.hbase.client.RegionLocator table)
private void	setupRandomGeneratorMapper(org.apache.hadoop.mapreduce.Job job, boolean putSortReducer)
void	test_LATEST_TIMESTAMP_isReplaced()	Test that HFileOutputFormat2 RecordWriter amends timestamps if passed a keyvalue whose timestamp is HConstants.LATEST_TIMESTAMP.
void	test_TIMERANGE()
void	test_WritingTagData()	Test that HFileOutputFormat2 RecordWriter writes tags such as ttl into hfile.
void	testBlockStoragePolicy()
void	testColumnFamilySettings()	Test that HFileOutputFormat2 RecordWriter uses compression and bloom filter settings from the column family descriptor
void	TestConfigureCompression()
void	TestConfigurePartitioner()
void	testExcludeAllFromMinorCompaction()	This test is to test the scenario happened in HBASE-6901.
void	testExcludeMinorCompaction()
void	testJobConfiguration()
void	testMRIncrementalLoad()
void	testMRIncrementalLoadWithLocality()	Test for HFileOutputFormat2.LOCALITY_SENSITIVE_CONF_KEY = true This test could only check the correctness of original logic if LOCALITY_SENSITIVE_CONF_KEY is set to true.
void	testMRIncrementalLoadWithLocalityMultiCluster()
void	testMRIncrementalLoadWithPutSortReducer()
void	testMRIncrementalLoadWithSplit()
void	testMultiMRIncrementalLoadWithPutSortReducer()
void	testSerializeDeserializeFamilyBlockSizeMap()	Test for HFileOutputFormat2.createFamilyBlockSizeMap(Configuration).
void	testSerializeDeserializeFamilyBloomTypeMap()	Test for HFileOutputFormat2.createFamilyBloomTypeMap(Configuration).
void	testSerializeDeserializeFamilyCompressionMap()	Test for HFileOutputFormat2.createFamilyCompressionMap(Configuration).
void	testSerializeDeserializeFamilyDataBlockEncodingMap()	Test for HFileOutputFormat2.createFamilyDataBlockEncodingMap(Configuration).
void	testWritingPEData()	Run small MR job.
private void	writeRandomKeyValues(org.apache.hadoop.mapreduce.RecordWriter<org.apache.hadoop.hbase.io.ImmutableBytesWritable,org.apache.hadoop.hbase.Cell> writer, org.apache.hadoop.mapreduce.TaskAttemptContext context, Set<byte[]> families, int numRows)	Write random values to the writer assuming a table created using FAMILIES as column family descriptors

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Details

CLASS_RULE

public static final HBaseClassTestRule CLASS_RULE

ROWSPERSPLIT

private static final int ROWSPERSPLIT

See Also:

• Constant Field Values

DEFAULT_VALUE_LENGTH

private static final int DEFAULT_VALUE_LENGTH

See Also:

• Constant Field Values

FAMILY_NAME

public static final byte[] FAMILY_NAME

FAMILIES

private static final byte[][] FAMILIES

TABLE_NAMES

private static final org.apache.hadoop.hbase.TableName[] TABLE_NAMES

util

private HBaseTestingUtil util

LOG

private static final org.slf4j.Logger LOG

Constructor Details

TestHFileOutputFormat2

public TestHFileOutputFormat2()

Method Details

setupRandomGeneratorMapper

private void setupRandomGeneratorMapper(org.apache.hadoop.mapreduce.Job job,
 boolean putSortReducer)

test_LATEST_TIMESTAMP_isReplaced

public void test_LATEST_TIMESTAMP_isReplaced()
                                      throws Exception

Test that HFileOutputFormat2 RecordWriter amends timestamps if passed a keyvalue whose timestamp is HConstants.LATEST_TIMESTAMP.

Throws:

Exception

See Also:

• HBASE-2615

createTestTaskAttemptContext

private org.apache.hadoop.mapreduce.TaskAttemptContext createTestTaskAttemptContext(org.apache.hadoop.mapreduce.Job job)
                                                                             throws Exception

Throws:

Exception

test_TIMERANGE

public void test_TIMERANGE()
                    throws Exception

Throws:

Exception

testWritingPEData

public void testWritingPEData()
                       throws Exception

Run small MR job.

Throws:

Exception

test_WritingTagData

public void test_WritingTagData()
                         throws Exception

Test that HFileOutputFormat2 RecordWriter writes tags such as ttl into hfile.

Throws:

Exception

testJobConfiguration

public void testJobConfiguration()
                          throws Exception

Throws:

Exception

generateRandomStartKeys

private byte[][] generateRandomStartKeys(int numKeys)

generateData

public static byte[] generateData(Random r,
 int length)

generateRandomSplitKeys

private byte[][] generateRandomSplitKeys(int numKeys)

testMRIncrementalLoad

public void testMRIncrementalLoad()
                           throws Exception

Throws:

Exception

testMRIncrementalLoadWithSplit

public void testMRIncrementalLoadWithSplit()
                                    throws Exception

Throws:

Exception

testMRIncrementalLoadWithLocality

public void testMRIncrementalLoadWithLocality()
                                       throws Exception

Test for HFileOutputFormat2.LOCALITY_SENSITIVE_CONF_KEY = true This test could only check the correctness of original logic if LOCALITY_SENSITIVE_CONF_KEY is set to true. Because MiniHBaseCluster always run with single hostname (and different ports), it's not possible to check the region locality by comparing region locations and DN hostnames. When MiniHBaseCluster supports explicit hostnames parameter (just like MiniDFSCluster does), we could test region locality features more easily.

Throws:

Exception

testMRIncrementalLoadWithPutSortReducer

public void testMRIncrementalLoadWithPutSortReducer()
                                             throws Exception

Throws:

Exception

doIncrementalLoadTest

private void doIncrementalLoadTest(boolean shouldChangeRegions,
 boolean shouldKeepLocality,
 boolean putSortReducer,
 String tableStr)
                            throws Exception

Throws:

Exception

testMultiMRIncrementalLoadWithPutSortReducer

public void testMultiMRIncrementalLoadWithPutSortReducer()
                                                  throws Exception

Throws:

Exception

doIncrementalLoadTest

private void doIncrementalLoadTest(boolean shouldChangeRegions,
 boolean shouldKeepLocality,
 boolean putSortReducer,
 List<String> tableStr)
                            throws Exception

Throws:

Exception

runIncrementalPELoad

private void runIncrementalPELoad(org.apache.hadoop.conf.Configuration conf,
 List<org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2.TableInfo> tableInfo,
 org.apache.hadoop.fs.Path outDir,
 boolean putSortReducer)
                           throws IOException,
InterruptedException,
ClassNotFoundException

Throws:

IOException

InterruptedException

ClassNotFoundException

testSerializeDeserializeFamilyCompressionMap

public void testSerializeDeserializeFamilyCompressionMap()
                                                  throws IOException

Test for HFileOutputFormat2.createFamilyCompressionMap(Configuration). Tests that the family compression map is correctly serialized into and deserialized from configuration

Throws:

IOException

setupMockColumnFamiliesForCompression

private void setupMockColumnFamiliesForCompression(org.apache.hadoop.hbase.client.Table table,
 Map<String,org.apache.hadoop.hbase.io.compress.Compression.Algorithm> familyToCompression)
                                            throws IOException

Throws:

IOException

getMockColumnFamiliesForCompression

private Map<String,org.apache.hadoop.hbase.io.compress.Compression.Algorithm> getMockColumnFamiliesForCompression(int numCfs)

Returns:

a map from column family names to compression algorithms for testing column family compression. Column family names have special characters

testSerializeDeserializeFamilyBloomTypeMap

public void testSerializeDeserializeFamilyBloomTypeMap()
                                                throws IOException

Test for HFileOutputFormat2.createFamilyBloomTypeMap(Configuration). Tests that the family bloom type map is correctly serialized into and deserialized from configuration

Throws:

IOException

setupMockColumnFamiliesForBloomType

private void setupMockColumnFamiliesForBloomType(org.apache.hadoop.hbase.client.Table table,
 Map<String,org.apache.hadoop.hbase.regionserver.BloomType> familyToDataBlockEncoding)
                                          throws IOException

Throws:

IOException

getMockColumnFamiliesForBloomType

private Map<String,org.apache.hadoop.hbase.regionserver.BloomType> getMockColumnFamiliesForBloomType(int numCfs)

Returns:

a map from column family names to compression algorithms for testing column family compression. Column family names have special characters

testSerializeDeserializeFamilyBlockSizeMap

public void testSerializeDeserializeFamilyBlockSizeMap()
                                                throws IOException

Test for HFileOutputFormat2.createFamilyBlockSizeMap(Configuration). Tests that the family block size map is correctly serialized into and deserialized from configuration

Throws:

IOException

setupMockColumnFamiliesForBlockSize

private void setupMockColumnFamiliesForBlockSize(org.apache.hadoop.hbase.client.Table table,
 Map<String,Integer> familyToDataBlockEncoding)
                                          throws IOException

Throws:

IOException

getMockColumnFamiliesForBlockSize

private Map<String,Integer> getMockColumnFamiliesForBlockSize(int numCfs)

Returns:

a map from column family names to compression algorithms for testing column family compression. Column family names have special characters

testSerializeDeserializeFamilyDataBlockEncodingMap

public void testSerializeDeserializeFamilyDataBlockEncodingMap()
                                                        throws IOException

Test for HFileOutputFormat2.createFamilyDataBlockEncodingMap(Configuration). Tests that the family data block encoding map is correctly serialized into and deserialized from configuration

Throws:

IOException

setupMockColumnFamiliesForDataBlockEncoding

private void setupMockColumnFamiliesForDataBlockEncoding(org.apache.hadoop.hbase.client.Table table,
 Map<String,org.apache.hadoop.hbase.io.encoding.DataBlockEncoding> familyToDataBlockEncoding)
                                                  throws IOException

Throws:

IOException

getMockColumnFamiliesForDataBlockEncoding

private Map<String,org.apache.hadoop.hbase.io.encoding.DataBlockEncoding> getMockColumnFamiliesForDataBlockEncoding(int numCfs)

Returns:

a map from column family names to compression algorithms for testing column family compression. Column family names have special characters

setupMockStartKeys

private void setupMockStartKeys(org.apache.hadoop.hbase.client.RegionLocator table)
                         throws IOException

Throws:

IOException

setupMockTableName

private void setupMockTableName(org.apache.hadoop.hbase.client.RegionLocator table)
                         throws IOException

Throws:

IOException

testColumnFamilySettings

public void testColumnFamilySettings()
                              throws Exception

Test that HFileOutputFormat2 RecordWriter uses compression and bloom filter settings from the column family descriptor

Throws:

Exception

writeRandomKeyValues

private void writeRandomKeyValues(org.apache.hadoop.mapreduce.RecordWriter<org.apache.hadoop.hbase.io.ImmutableBytesWritable,org.apache.hadoop.hbase.Cell> writer,
 org.apache.hadoop.mapreduce.TaskAttemptContext context,
 Set<byte[]> families,
 int numRows)
                           throws IOException,
InterruptedException

Write random values to the writer assuming a table created using FAMILIES as column family descriptors

Throws:

IOException

InterruptedException

testExcludeAllFromMinorCompaction

public void testExcludeAllFromMinorCompaction()
                                       throws Exception

This test is to test the scenario happened in HBASE-6901. All files are bulk loaded and excluded from minor compaction. Without the fix of HBASE-6901, an ArrayIndexOutOfBoundsException will be thrown.

Throws:

Exception

testExcludeMinorCompaction

public void testExcludeMinorCompaction()
                                throws Exception

Throws:

Exception

quickPoll

private void quickPoll(Callable<Boolean> c,
 int waitMs)
                throws Exception

Throws:

Exception

main

public static void main(String[] args)
                 throws Exception

Throws:

Exception

manualTest

public void manualTest(String[] args)
                throws Exception

Throws:

Exception

testBlockStoragePolicy

public void testBlockStoragePolicy()
                            throws Exception

Throws:

Exception

getStoragePolicyName

private String getStoragePolicyName(org.apache.hadoop.fs.FileSystem fs,
 org.apache.hadoop.fs.Path path)

getStoragePolicyNameForOldHDFSVersion

private String getStoragePolicyNameForOldHDFSVersion(org.apache.hadoop.fs.FileSystem fs,
 org.apache.hadoop.fs.Path path)

TestConfigurePartitioner

public void TestConfigurePartitioner()
                              throws IOException

Throws:

IOException

TestConfigureCompression

public void TestConfigureCompression()
                              throws Exception

Throws:

Exception

testMRIncrementalLoadWithLocalityMultiCluster

public void testMRIncrementalLoadWithLocalityMultiCluster()
                                                   throws Exception

Throws:

Exception