Package org.apache.hadoop.hbase.io.hfile.bucket

Class TestBucketCache

java.lang.Object

org.apache.hadoop.hbase.io.hfile.bucket.TestBucketCache

public class TestBucketCache
extends Object

Basic test of BucketCache.Puts and gets.

Tests will ensure that blocks' data correctness under several threads concurrency

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
private static class	TestBucketCache.MockedBucketCache

Field Summary

Fields

Modifier and Type	Field	Description
(package private) final int	BLOCK_SIZE
(package private) org.apache.hadoop.hbase.io.hfile.bucket.BucketCache	cache
(package private) final int	CACHE_SIZE
(package private) final long	capacitySize
static final HBaseClassTestRule	CLASS_RULE
int	constructedBlockSize
int[]	constructedBlockSizes
private static final HBaseTestingUtil	HBASE_TESTING_UTILITY
private String	ioEngineName
private static final org.slf4j.Logger	LOG
(package private) final int	NUM_BLOCKS
(package private) final int	NUM_QUERIES
(package private) final int	NUM_THREADS
(package private) final int	writerQLen
(package private) final int	writeThreads

Constructor Summary

Constructors

Constructor	Description
TestBucketCache()

Method Summary

Modifier and Type	Method	Description
private void	cacheAndWaitUntilFlushedToBucket(org.apache.hadoop.hbase.io.hfile.bucket.BucketCache cache, org.apache.hadoop.hbase.io.hfile.BlockCacheKey cacheKey, org.apache.hadoop.hbase.io.hfile.Cacheable block, boolean waitWhenCache)
private void	checkConfigValues(org.apache.hadoop.conf.Configuration conf, Map<String,float[]> configMap, boolean[] expectSuccess)
private org.apache.hadoop.fs.Path	createAndGetTestDir()	Test Utility to create test dir and return name
static Iterable<Object[]>	data()
private static <T> T	randFrom(List<T> a)	Return a random element from a.
void	setup()
void	tearDown()
void	testBlockAdditionWaitWhenCache()
void	testBucketAllocator()
void	testBucketAllocatorLargeBuckets()
void	testCacheBlockNextBlockMetadataMissing()
void	testCacheMultiThreadedSingleKey()
void	testCacheSimple()
void	testCacheSizeCapacity()
void	testEvictionCount()
private org.apache.hadoop.hbase.io.hfile.bucket.BucketCache	testEvictOrphans(long orphanEvictionGracePeriod)
void	testEvictOrphansOutOfGracePeriod()
void	testEvictOrphansWithinGracePeriod()
void	testFreeBlockWhenIOEngineWriteFailure()
void	testFreeBucketEntryRestoredFromFile()	This test is for HBASE-26295, BucketEntry which is restored from a persistence file could not be freed even if corresponding HFileBlock is evicted from BucketCache.
void	testGetPartitionSize()
void	testHeapSizeChanges()
void	testInvalidAcceptFactorConfig()
void	testInvalidCacheSplitFactorConfig()
void	testInvalidExtraFreeFactorConfig()
void	testInvalidMinFactorConfig()
void	testMemoryLeak()
private org.apache.hadoop.hbase.io.hfile.bucket.BucketCache	testNotifyFileCachingCompletedForTenBlocks(org.apache.hadoop.fs.Path filePath, int totalBlocksToCheck)
void	testNotifyFileCachingCompletedNotAllCached()
void	testNotifyFileCachingCompletedSuccess()
void	testOffsetProducesPositiveOutput()
void	testRAMCache()
private void	testRetrievalUtils(org.apache.hadoop.fs.Path testDir, String ioEngineName)
void	testRetrieveFromFile()
void	testRetrieveFromFileWithoutPersistence()
void	testRetrieveFromMMap()
void	testRetrieveFromMultipleFiles()
void	testRetrieveFromPMem()
void	testRetrieveUnsupportedIOE()
void	testValidBucketCacheConfigs()
private void	validateGetPartitionSize(org.apache.hadoop.hbase.io.hfile.bucket.BucketCache bucketCache, float partitionFactor, float minFactor)
static void	waitUntilAllFlushedToBucket(org.apache.hadoop.hbase.io.hfile.bucket.BucketCache cache)
static void	waitUntilFlushedToBucket(org.apache.hadoop.hbase.io.hfile.bucket.BucketCache cache, org.apache.hadoop.hbase.io.hfile.BlockCacheKey cacheKey)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Details

LOG

private static final org.slf4j.Logger LOG

CLASS_RULE

public static final HBaseClassTestRule CLASS_RULE

constructedBlockSize

public int constructedBlockSize

constructedBlockSizes

public int[] constructedBlockSizes

cache

org.apache.hadoop.hbase.io.hfile.bucket.BucketCache cache

CACHE_SIZE

final int CACHE_SIZE

See Also:

• Constant Field Values

NUM_BLOCKS

final int NUM_BLOCKS

See Also:

• Constant Field Values

BLOCK_SIZE

final int BLOCK_SIZE

See Also:

• Constant Field Values

NUM_THREADS

final int NUM_THREADS

See Also:

• Constant Field Values

NUM_QUERIES

final int NUM_QUERIES

See Also:

• Constant Field Values

capacitySize

final long capacitySize

See Also:

• Constant Field Values

writeThreads

final int writeThreads

See Also:

• Constant Field Values

writerQLen

final int writerQLen

See Also:

• Constant Field Values

ioEngineName

private String ioEngineName

HBASE_TESTING_UTILITY

private static final HBaseTestingUtil HBASE_TESTING_UTILITY

Constructor Details

TestBucketCache

public TestBucketCache()

Method Details

data

public static Iterable<Object[]> data()

setup

public void setup()
           throws IOException

Throws:

IOException

tearDown

public void tearDown()

createAndGetTestDir

private org.apache.hadoop.fs.Path createAndGetTestDir()
                                               throws IOException

Test Utility to create test dir and return name

Returns:

return name of created dir

Throws:

IOException - throws IOException

randFrom

private static <T> T randFrom(List<T> a)

Return a random element from a.

testBucketAllocator

public void testBucketAllocator()
                         throws org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocatorException

Throws:

org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocatorException

testCacheSimple

public void testCacheSimple()
                     throws Exception

Throws:

Exception

testCacheMultiThreadedSingleKey

public void testCacheMultiThreadedSingleKey()
                                     throws Exception

Throws:

Exception

testHeapSizeChanges

public void testHeapSizeChanges()
                         throws Exception

Throws:

Exception

waitUntilFlushedToBucket

public static void waitUntilFlushedToBucket(org.apache.hadoop.hbase.io.hfile.bucket.BucketCache cache,
 org.apache.hadoop.hbase.io.hfile.BlockCacheKey cacheKey)
                                     throws InterruptedException

Throws:

InterruptedException

waitUntilAllFlushedToBucket

public static void waitUntilAllFlushedToBucket(org.apache.hadoop.hbase.io.hfile.bucket.BucketCache cache)
                                        throws InterruptedException

Throws:

InterruptedException

cacheAndWaitUntilFlushedToBucket

private void cacheAndWaitUntilFlushedToBucket(org.apache.hadoop.hbase.io.hfile.bucket.BucketCache cache,
 org.apache.hadoop.hbase.io.hfile.BlockCacheKey cacheKey,
 org.apache.hadoop.hbase.io.hfile.Cacheable block,
 boolean waitWhenCache)
                                       throws InterruptedException

Throws:

InterruptedException

testMemoryLeak

public void testMemoryLeak()
                    throws Exception

Throws:

Exception

testRetrieveFromFile

public void testRetrieveFromFile()
                          throws Exception

Throws:

Exception

testRetrieveFromMMap

public void testRetrieveFromMMap()
                          throws Exception

Throws:

Exception

testRetrieveFromPMem

public void testRetrieveFromPMem()
                          throws Exception

Throws:

Exception

testRetrievalUtils

private void testRetrievalUtils(org.apache.hadoop.fs.Path testDir,
 String ioEngineName)
                         throws IOException,
InterruptedException

Throws:

IOException

InterruptedException

testRetrieveUnsupportedIOE

public void testRetrieveUnsupportedIOE()
                                throws Exception

Throws:

Exception

testRetrieveFromMultipleFiles

public void testRetrieveFromMultipleFiles()
                                   throws Exception

Throws:

Exception

testRetrieveFromFileWithoutPersistence

public void testRetrieveFromFileWithoutPersistence()
                                            throws Exception

Throws:

Exception

testBucketAllocatorLargeBuckets

public void testBucketAllocatorLargeBuckets()
                                     throws org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocatorException

Throws:

org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocatorException

testGetPartitionSize

public void testGetPartitionSize()
                          throws IOException

Throws:

IOException

testCacheSizeCapacity

public void testCacheSizeCapacity()
                           throws IOException

Throws:

IOException

testValidBucketCacheConfigs

public void testValidBucketCacheConfigs()
                                 throws IOException

Throws:

IOException

testInvalidAcceptFactorConfig

public void testInvalidAcceptFactorConfig()
                                   throws IOException

Throws:

IOException

testInvalidMinFactorConfig

public void testInvalidMinFactorConfig()
                                throws IOException

Throws:

IOException

testInvalidExtraFreeFactorConfig

public void testInvalidExtraFreeFactorConfig()
                                      throws IOException

Throws:

IOException

testInvalidCacheSplitFactorConfig

public void testInvalidCacheSplitFactorConfig()
                                       throws IOException

Throws:

IOException

checkConfigValues

private void checkConfigValues(org.apache.hadoop.conf.Configuration conf,
 Map<String,float[]> configMap,
 boolean[] expectSuccess)
                        throws IOException

Throws:

IOException

validateGetPartitionSize

private void validateGetPartitionSize(org.apache.hadoop.hbase.io.hfile.bucket.BucketCache bucketCache,
 float partitionFactor,
 float minFactor)

testOffsetProducesPositiveOutput

public void testOffsetProducesPositiveOutput()

testEvictionCount

public void testEvictionCount()
                       throws InterruptedException

Throws:

InterruptedException

testCacheBlockNextBlockMetadataMissing

public void testCacheBlockNextBlockMetadataMissing()
                                            throws Exception

Throws:

Exception

testRAMCache

public void testRAMCache()

testFreeBlockWhenIOEngineWriteFailure

public void testFreeBlockWhenIOEngineWriteFailure()
                                           throws IOException

Throws:

IOException

testFreeBucketEntryRestoredFromFile

public void testFreeBucketEntryRestoredFromFile()
                                         throws Exception

This test is for HBASE-26295, BucketEntry which is restored from a persistence file could not be freed even if corresponding HFileBlock is evicted from BucketCache.

Throws:

Exception

testBlockAdditionWaitWhenCache

public void testBlockAdditionWaitWhenCache()
                                    throws Exception

Throws:

Exception

testNotifyFileCachingCompletedSuccess

public void testNotifyFileCachingCompletedSuccess()
                                           throws Exception

Throws:

Exception

testNotifyFileCachingCompletedNotAllCached

public void testNotifyFileCachingCompletedNotAllCached()
                                                throws Exception

Throws:

Exception

testNotifyFileCachingCompletedForTenBlocks

private org.apache.hadoop.hbase.io.hfile.bucket.BucketCache testNotifyFileCachingCompletedForTenBlocks(org.apache.hadoop.fs.Path filePath,
 int totalBlocksToCheck)
                                                                                                throws Exception

Throws:

Exception

testEvictOrphansOutOfGracePeriod

public void testEvictOrphansOutOfGracePeriod()
                                      throws Exception

Throws:

Exception

testEvictOrphansWithinGracePeriod

public void testEvictOrphansWithinGracePeriod()
                                       throws Exception

Throws:

Exception

testEvictOrphans

private org.apache.hadoop.hbase.io.hfile.bucket.BucketCache testEvictOrphans(long orphanEvictionGracePeriod)
                                                                      throws Exception

Throws:

Exception