Package org.apache.hadoop.hbase.client

Class TestAsyncProcess

java.lang.Object

org.apache.hadoop.hbase.client.TestAsyncProcess

public class TestAsyncProcess
extends Object

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
(package private) static class	TestAsyncProcess.AsyncProcessForThrowableCheck
(package private) static class	TestAsyncProcess.AsyncProcessWithFailure
(package private) static class	TestAsyncProcess.CallerWithFailure
(package private) static class	TestAsyncProcess.CallerWithRegionException	Used to simulate the case where a RegionServer responds to a multi request, but some or all of the actions have an Exception instead of Result.
(package private) static class	TestAsyncProcess.CountingThreadFactory
(package private) static class	TestAsyncProcess.MyAsyncProcess
(package private) static class	TestAsyncProcess.MyAsyncProcessWithReplicas
(package private) static class	TestAsyncProcess.MyAsyncRequestFutureImpl<Res>
(package private) static class	TestAsyncProcess.MyClientBackoffPolicy	Make the backoff time always different on each call.
(package private) static class	TestAsyncProcess.MyConnectionImpl	Returns our async process.
(package private) static class	TestAsyncProcess.MyConnectionImpl2	Returns our async process.
(package private) static class	TestAsyncProcess.MyThreadPoolExecutor
private static interface	TestAsyncProcess.ResponseGenerator
private static enum	TestAsyncProcess.RR	After reading TheDailyWtf, I always wanted to create a MyBoolean enum like this!

Field Summary

Fields

Modifier and Type	Field	Description
static final HBaseClassTestRule	CLASS_RULE
private org.apache.hadoop.conf.Configuration	CONF
private org.apache.hadoop.hbase.client.ConnectionConfiguration	CONNECTION_CONFIG
private static final byte[]	DUMMY_BYTES_1
private static final byte[]	DUMMY_BYTES_2
private static final byte[]	DUMMY_BYTES_3
private static final org.apache.hadoop.hbase.TableName	DUMMY_TABLE
private static final byte[]	FAILS
private static Exception	failure
private static final org.apache.hadoop.hbase.HRegionInfo	hri1
private static final org.apache.hadoop.hbase.client.RegionInfo	hri1r1
private static final org.apache.hadoop.hbase.client.RegionInfo	hri1r2
private static final org.apache.hadoop.hbase.HRegionInfo	hri2
private static final org.apache.hadoop.hbase.client.RegionInfo	hri2r1
private static final org.apache.hadoop.hbase.HRegionInfo	hri3
private static final org.apache.hadoop.hbase.RegionLocations	hrls1
private static final org.apache.hadoop.hbase.RegionLocations	hrls2
private static final org.apache.hadoop.hbase.RegionLocations	hrls3
private static final org.apache.hadoop.hbase.HRegionLocation	loc1
private static final org.apache.hadoop.hbase.HRegionLocation	loc2
private static final org.apache.hadoop.hbase.HRegionLocation	loc3
private static final org.slf4j.Logger	LOG
private static final int	NB_RETRIES
private int	OPERATION_TIMEOUT
private int	RPC_TIMEOUT
private static final org.apache.hadoop.hbase.ServerName	sn
private static final org.apache.hadoop.hbase.ServerName	sn2
private static final org.apache.hadoop.hbase.ServerName	sn3
private static final String	success

Constructor Summary

Constructors

Constructor	Description
TestAsyncProcess()

Method Summary

Modifier and Type	Method	Description
void	beforeEach()
private static long	calculateRequestCount(long putSizePerServer, long maxHeapSizePerRequest)
private void	checkPeriodicFlushParameters(org.apache.hadoop.hbase.client.ClusterConnection conn, TestAsyncProcess.MyAsyncProcess ap, long setTO, long expectTO, long setTT, long expectTT)
private org.apache.hadoop.hbase.client.BufferedMutatorParams	createBufferedMutatorParams(TestAsyncProcess.MyAsyncProcess ap, org.apache.hadoop.hbase.TableName name)
private org.apache.hadoop.hbase.client.ClusterConnection	createHConnection()
private org.apache.hadoop.hbase.client.ClusterConnection	createHConnection(org.apache.hadoop.hbase.client.ConnectionConfiguration configuration)
private org.apache.hadoop.hbase.client.ClusterConnection	createHConnectionCommon(org.apache.hadoop.hbase.client.ConnectionConfiguration connectionConfiguration)
private org.apache.hadoop.hbase.client.ClusterConnection	createHConnectionWithReplicas(org.apache.hadoop.hbase.client.ConnectionConfiguration configuration)
(package private) static org.apache.hadoop.hbase.client.MultiResponse	createMultiResponse(org.apache.hadoop.hbase.client.MultiAction multi, AtomicInteger nbMultiResponse, AtomicInteger nbActions, TestAsyncProcess.ResponseGenerator gen)
private org.apache.hadoop.hbase.client.Put	createPut(int regCnt, boolean success)
private TestAsyncProcess.MyAsyncProcessWithReplicas	createReplicaAp(int replicaAfterMs, int primaryMs, int replicaMs)
private TestAsyncProcess.MyAsyncProcessWithReplicas	createReplicaAp(int replicaAfterMs, int primaryMs, int replicaMs, int retries)
private void	doSubmitRequest(long maxHeapSizePerRequest, long putsHeapSize)
private static List<org.apache.hadoop.hbase.client.Get>	makeTimelineGets(byte[]... rows)
private static void	setMockLocation(org.apache.hadoop.hbase.client.ClusterConnection hc, byte[] row, org.apache.hadoop.hbase.RegionLocations result)
void	testAction()
void	testBatch()
void	testBufferedMutatorImplWithSharedPool()
void	testCallQueueTooLarge()
void	testErrors()
void	testErrorsServers()
void	testFail()
void	testFailAndSuccess()
void	testFailedPutAndNewPut()
void	testFastFailIfBackoffGreaterThanRemaining()
void	testFlush()
void	testHTablePutSuccess()
void	testListRowAccess()
void	testMaxTask()
void	testQueueRowAccess()
void	testReadAndWriteTimeout()
void	testReplicaAllCallsFailForOneRegion()
void	testReplicaMainFailsBeforeReplicaCalls()
void	testReplicaParallelCallsSucceed()
void	testReplicaPartialReplicaCall()
void	testReplicaPrimarySuccessWoReplicaCalls()
void	testReplicaReplicaSuccess()
void	testReplicaReplicaSuccessWithParallelFailures()
private void	testRetryPause(org.apache.hadoop.conf.Configuration testConf, long expectedPause, org.apache.hadoop.hbase.HBaseIOException exception)
void	testRetryPauseForRpcThrottling()
private void	testRetryPauseWhenServerIsOverloaded(org.apache.hadoop.hbase.HBaseServerException exception)
void	testRetryPauseWhenServerOverloadedDueToCDE()
void	testRetryPauseWhenServerOverloadedDueToCQTBE()	Below tests make sure we could use a special pause setting when retry an exception where HBaseServerException.isServerOverloaded(Throwable) is true, see HBASE-17114
private void	testRetryWithExceptionClearsMetaCache(boolean useRegionException)
void	testRetryWithExceptionClearsMetaCacheUsingRegionException()	Tests that we properly recover from exceptions that DO NOT go through receiveGlobalFailure, due to updating the meta cache for the region which failed.
void	testRetryWithExceptionClearsMetaCacheUsingServerException()	Tests that we properly recover from exceptions that go through receiveGlobalFailure, due to updating the meta cache for the region which failed.
void	testSettingWriteBufferPeriodicFlushParameters()
void	testSubmit()
void	testSubmitBusyRegion()
void	testSubmitBusyRegionServer()
void	testSubmitLargeRequest()
void	testSubmitLargeRequestWithUnlimitedSize()
void	testSubmitRandomSizeRequest()
void	testSubmitSameSizeOfRequest()
void	testSubmitSmallRequest()
void	testSubmitTrue()
void	testSubmitWithCB()
private void	testTaskCount(TestAsyncProcess.MyAsyncProcess ap)
void	testTaskCountWithClientBackoffPolicy()
void	testTaskCountWithoutClientBackoffPolicy()
void	testThreadCreation()	This test simulates multiple regions on 2 servers.
void	testUncheckedException()
void	testWriteBufferPeriodicFlushTimeoutMs()
private void	verifyReplicaResult(org.apache.hadoop.hbase.client.AsyncRequestFuture ars, TestAsyncProcess.RR... expecteds)
private void	verifyResult(org.apache.hadoop.hbase.client.AsyncRequestFuture ars, boolean... expected)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Details

CLASS_RULE

public static final HBaseClassTestRule CLASS_RULE

LOG

private static final org.slf4j.Logger LOG

DUMMY_TABLE

private static final org.apache.hadoop.hbase.TableName DUMMY_TABLE

DUMMY_BYTES_1

private static final byte[] DUMMY_BYTES_1

DUMMY_BYTES_2

private static final byte[] DUMMY_BYTES_2

DUMMY_BYTES_3

private static final byte[] DUMMY_BYTES_3

FAILS

private static final byte[] FAILS

CONF

private org.apache.hadoop.conf.Configuration CONF

CONNECTION_CONFIG

private org.apache.hadoop.hbase.client.ConnectionConfiguration CONNECTION_CONFIG

sn

private static final org.apache.hadoop.hbase.ServerName sn

sn2

private static final org.apache.hadoop.hbase.ServerName sn2

sn3

private static final org.apache.hadoop.hbase.ServerName sn3

hri1

private static final org.apache.hadoop.hbase.HRegionInfo hri1

hri2

private static final org.apache.hadoop.hbase.HRegionInfo hri2

hri3

private static final org.apache.hadoop.hbase.HRegionInfo hri3

loc1

private static final org.apache.hadoop.hbase.HRegionLocation loc1

loc2

private static final org.apache.hadoop.hbase.HRegionLocation loc2

loc3

private static final org.apache.hadoop.hbase.HRegionLocation loc3

hri1r1

private static final org.apache.hadoop.hbase.client.RegionInfo hri1r1

hri1r2

private static final org.apache.hadoop.hbase.client.RegionInfo hri1r2

hri2r1

private static final org.apache.hadoop.hbase.client.RegionInfo hri2r1

hrls1

private static final org.apache.hadoop.hbase.RegionLocations hrls1

hrls2

private static final org.apache.hadoop.hbase.RegionLocations hrls2

hrls3

private static final org.apache.hadoop.hbase.RegionLocations hrls3

success

private static final String success

See Also:

• Constant Field Values

failure

private static Exception failure

NB_RETRIES

private static final int NB_RETRIES

See Also:

• Constant Field Values

RPC_TIMEOUT

private int RPC_TIMEOUT

OPERATION_TIMEOUT

private int OPERATION_TIMEOUT

Constructor Details

TestAsyncProcess

public TestAsyncProcess()

Method Details

beforeEach

public void beforeEach()

createMultiResponse

static org.apache.hadoop.hbase.client.MultiResponse createMultiResponse(org.apache.hadoop.hbase.client.MultiAction multi,
 AtomicInteger nbMultiResponse,
 AtomicInteger nbActions,
 TestAsyncProcess.ResponseGenerator gen)

testListRowAccess

public void testListRowAccess()

calculateRequestCount

private static long calculateRequestCount(long putSizePerServer,
 long maxHeapSizePerRequest)

testSubmitSameSizeOfRequest

public void testSubmitSameSizeOfRequest()
                                 throws Exception

Throws:

Exception

testSubmitLargeRequestWithUnlimitedSize

public void testSubmitLargeRequestWithUnlimitedSize()
                                             throws Exception

Throws:

Exception

testSubmitRandomSizeRequest

public void testSubmitRandomSizeRequest()
                                 throws Exception

Throws:

Exception

testSubmitSmallRequest

public void testSubmitSmallRequest()
                            throws Exception

Throws:

Exception

testSubmitLargeRequest

public void testSubmitLargeRequest()
                            throws Exception

Throws:

Exception

doSubmitRequest

private void doSubmitRequest(long maxHeapSizePerRequest,
 long putsHeapSize)
                      throws Exception

Throws:

Exception

testSubmit

public void testSubmit()
                throws Exception

Throws:

Exception

testSubmitWithCB

public void testSubmitWithCB()
                      throws Exception

Throws:

Exception

testSubmitBusyRegion

public void testSubmitBusyRegion()
                          throws Exception

Throws:

Exception

testSubmitBusyRegionServer

public void testSubmitBusyRegionServer()
                                throws Exception

Throws:

Exception

testFail

public void testFail()
              throws Exception

Throws:

Exception

testSubmitTrue

public void testSubmitTrue()
                    throws IOException

Throws:

IOException

testFailAndSuccess

public void testFailAndSuccess()
                        throws Exception

Throws:

Exception

testFlush

public void testFlush()
               throws Exception

Throws:

Exception

testTaskCountWithoutClientBackoffPolicy

public void testTaskCountWithoutClientBackoffPolicy()
                                             throws IOException,
InterruptedException

Throws:

IOException

InterruptedException

testTaskCountWithClientBackoffPolicy

public void testTaskCountWithClientBackoffPolicy()
                                          throws IOException,
InterruptedException

Throws:

IOException

InterruptedException

testTaskCount

private void testTaskCount(TestAsyncProcess.MyAsyncProcess ap)
                    throws InterruptedIOException,
InterruptedException

Throws:

InterruptedIOException

InterruptedException

testMaxTask

public void testMaxTask()
                 throws Exception

Throws:

Exception

createHConnection

private org.apache.hadoop.hbase.client.ClusterConnection createHConnection()
                                                                    throws IOException

Throws:

IOException

createHConnection

private org.apache.hadoop.hbase.client.ClusterConnection createHConnection(org.apache.hadoop.hbase.client.ConnectionConfiguration configuration)
                                                                    throws IOException

Throws:

IOException

createHConnectionWithReplicas

private org.apache.hadoop.hbase.client.ClusterConnection createHConnectionWithReplicas(org.apache.hadoop.hbase.client.ConnectionConfiguration configuration)
                                                                                throws IOException

Throws:

IOException

setMockLocation

private static void setMockLocation(org.apache.hadoop.hbase.client.ClusterConnection hc,
 byte[] row,
 org.apache.hadoop.hbase.RegionLocations result)
                             throws IOException

Throws:

IOException

createHConnectionCommon

private org.apache.hadoop.hbase.client.ClusterConnection createHConnectionCommon(org.apache.hadoop.hbase.client.ConnectionConfiguration connectionConfiguration)

testHTablePutSuccess

public void testHTablePutSuccess()
                          throws Exception

Throws:

Exception

testSettingWriteBufferPeriodicFlushParameters

public void testSettingWriteBufferPeriodicFlushParameters()
                                                   throws Exception

Throws:

Exception

checkPeriodicFlushParameters

private void checkPeriodicFlushParameters(org.apache.hadoop.hbase.client.ClusterConnection conn,
 TestAsyncProcess.MyAsyncProcess ap,
 long setTO,
 long expectTO,
 long setTT,
 long expectTT)

testWriteBufferPeriodicFlushTimeoutMs

public void testWriteBufferPeriodicFlushTimeoutMs()
                                           throws Exception

Throws:

Exception

testBufferedMutatorImplWithSharedPool

public void testBufferedMutatorImplWithSharedPool()
                                           throws Exception

Throws:

Exception

testFailedPutAndNewPut

public void testFailedPutAndNewPut()
                            throws Exception

Throws:

Exception

testAction

public void testAction()

testBatch

public void testBatch()
               throws IOException,
InterruptedException

Throws:

IOException

InterruptedException

testErrorsServers

public void testErrorsServers()
                       throws IOException

Throws:

IOException

testReadAndWriteTimeout

public void testReadAndWriteTimeout()
                             throws IOException

Throws:

IOException

testErrors

public void testErrors()
                throws IOException

Throws:

IOException

testCallQueueTooLarge

public void testCallQueueTooLarge()
                           throws IOException

Throws:

IOException

testThreadCreation

public void testThreadCreation()
                        throws Exception

This test simulates multiple regions on 2 servers. We should have 2 multi requests and 2 threads: 1 per server, this whatever the number of regions.

Throws:

Exception

testReplicaReplicaSuccess

public void testReplicaReplicaSuccess()
                               throws Exception

Throws:

Exception

testReplicaPrimarySuccessWoReplicaCalls

public void testReplicaPrimarySuccessWoReplicaCalls()
                                             throws Exception

Throws:

Exception

testReplicaParallelCallsSucceed

public void testReplicaParallelCallsSucceed()
                                     throws Exception

Throws:

Exception

testReplicaPartialReplicaCall

public void testReplicaPartialReplicaCall()
                                   throws Exception

Throws:

Exception

testReplicaMainFailsBeforeReplicaCalls

public void testReplicaMainFailsBeforeReplicaCalls()
                                            throws Exception

Throws:

Exception

testReplicaReplicaSuccessWithParallelFailures

public void testReplicaReplicaSuccessWithParallelFailures()
                                                   throws Exception

Throws:

Exception

testReplicaAllCallsFailForOneRegion

public void testReplicaAllCallsFailForOneRegion()
                                         throws Exception

Throws:

Exception

createReplicaAp

private TestAsyncProcess.MyAsyncProcessWithReplicas createReplicaAp(int replicaAfterMs,
 int primaryMs,
 int replicaMs)
                                                             throws Exception

Throws:

Exception

createReplicaAp

private TestAsyncProcess.MyAsyncProcessWithReplicas createReplicaAp(int replicaAfterMs,
 int primaryMs,
 int replicaMs,
 int retries)
                                                             throws Exception

Throws:

Exception

createBufferedMutatorParams

private org.apache.hadoop.hbase.client.BufferedMutatorParams createBufferedMutatorParams(TestAsyncProcess.MyAsyncProcess ap,
 org.apache.hadoop.hbase.TableName name)

makeTimelineGets

private static List<org.apache.hadoop.hbase.client.Get> makeTimelineGets(byte[]... rows)

verifyResult

private void verifyResult(org.apache.hadoop.hbase.client.AsyncRequestFuture ars,
 boolean... expected)
                   throws Exception

Throws:

Exception

verifyReplicaResult

private void verifyReplicaResult(org.apache.hadoop.hbase.client.AsyncRequestFuture ars,
 TestAsyncProcess.RR... expecteds)
                          throws Exception

Throws:

Exception

createPut

private org.apache.hadoop.hbase.client.Put createPut(int regCnt,
 boolean success)

Parameters:

regCnt - the region: 1 to 3.

success - if true, the put will succeed.

Returns:

a put

testUncheckedException

public void testUncheckedException()
                            throws Exception

Throws:

Exception

testRetryPauseWhenServerOverloadedDueToCQTBE

public void testRetryPauseWhenServerOverloadedDueToCQTBE()
                                                  throws Exception

Below tests make sure we could use a special pause setting when retry an exception where HBaseServerException.isServerOverloaded(Throwable) is true, see HBASE-17114

Throws:

Exception

testRetryPauseWhenServerOverloadedDueToCDE

public void testRetryPauseWhenServerOverloadedDueToCDE()
                                                throws Exception

Throws:

Exception

testRetryPauseForRpcThrottling

public void testRetryPauseForRpcThrottling()
                                    throws IOException

Throws:

IOException

testRetryPauseWhenServerIsOverloaded

private void testRetryPauseWhenServerIsOverloaded(org.apache.hadoop.hbase.HBaseServerException exception)
                                           throws IOException

Throws:

IOException

testRetryPause

private void testRetryPause(org.apache.hadoop.conf.Configuration testConf,
 long expectedPause,
 org.apache.hadoop.hbase.HBaseIOException exception)
                     throws IOException

Throws:

IOException

testFastFailIfBackoffGreaterThanRemaining

public void testFastFailIfBackoffGreaterThanRemaining()
                                               throws IOException

Throws:

IOException

testRetryWithExceptionClearsMetaCacheUsingRegionException

public void testRetryWithExceptionClearsMetaCacheUsingRegionException()
                                                               throws Exception

Tests that we properly recover from exceptions that DO NOT go through receiveGlobalFailure, due to updating the meta cache for the region which failed. Successful multigets can include region exceptions in the MultiResponse. In that case, it skips receiveGlobalFailure and instead handles in receiveMultiAction

Throws:

Exception

testRetryWithExceptionClearsMetaCacheUsingServerException

public void testRetryWithExceptionClearsMetaCacheUsingServerException()
                                                               throws Exception

Tests that we properly recover from exceptions that go through receiveGlobalFailure, due to updating the meta cache for the region which failed.

Throws:

Exception

testRetryWithExceptionClearsMetaCache

private void testRetryWithExceptionClearsMetaCache(boolean useRegionException)
                                            throws IOException

Throws:

IOException

testQueueRowAccess

public void testQueueRowAccess()
                        throws Exception

Throws:

Exception