Package org.apache.hadoop.hbase.regionserver

Class TestHRegionReplayEvents

java.lang.Object

org.apache.hadoop.hbase.regionserver.TestHRegionReplayEvents

public class TestHRegionReplayEvents
extends Object

Tests of HRegion methods for replaying flush, compaction, region open, etc events for secondary region replicas

Field Summary

Fields

Modifier and Type	Field	Description
static final HBaseClassTestRule	CLASS_RULE
static org.apache.hadoop.conf.Configuration	CONF
protected byte[]	cq
private String	dir
private byte[][]	families
private org.apache.hadoop.hbase.client.TableDescriptor	htd
private static final org.slf4j.Logger	LOG
protected String	method
org.junit.rules.TestName	name
private org.apache.hadoop.hbase.client.RegionInfo	primaryHri
private org.apache.hadoop.hbase.regionserver.HRegion	primaryRegion
private org.apache.hadoop.hbase.wal.WALStreamReader	reader
private org.apache.hadoop.fs.Path	rootDir
protected final byte[]	row
protected final byte[]	row2
private org.apache.hadoop.hbase.regionserver.RegionServerServices	rss
private org.apache.hadoop.hbase.client.RegionInfo	secondaryHri
private org.apache.hadoop.hbase.regionserver.HRegion	secondaryRegion
protected byte[]	tableName
private static HBaseTestingUtility	TEST_UTIL
private org.apache.hadoop.hbase.wal.WAL	walPrimary
private org.apache.hadoop.hbase.wal.WAL	walSecondary

Constructor Summary

Constructors

Constructor	Description
TestHRegionReplayEvents()

Method Summary

Modifier and Type	Method	Description
private void	assertPathListsEqual(List<String> list1, List<String> list2)	Paths can be qualified or not.
private org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor	clone(org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor flush, long flushSeqId)
private String	createHFileForFamilies(org.apache.hadoop.fs.Path testPath, byte[] family, byte[] valueBytes)
private org.apache.hadoop.hbase.wal.WALStreamReader	createWALReaderForPrimary()
private void	disableReads(org.apache.hadoop.hbase.regionserver.HRegion region)
(package private) String	getName()
private static org.apache.hadoop.hbase.regionserver.HRegion	initHRegion(byte[] tableName, byte[] startKey, byte[] stopKey, String callingMethod, org.apache.hadoop.conf.Configuration conf, boolean isReadOnly, org.apache.hadoop.hbase.client.Durability durability, org.apache.hadoop.hbase.wal.WAL wal, byte[]... families)
private static org.apache.hadoop.hbase.regionserver.HRegion	initHRegion(byte[] tableName, String callingMethod, byte[]... families)
private void	putDataByReplay(org.apache.hadoop.hbase.regionserver.HRegion region, int startRow, int numRows, byte[] qf, byte[]... families)
private void	putDataWithFlushes(org.apache.hadoop.hbase.regionserver.HRegion region, int flushInterval, int numRows, int numRowsAfterFlush)	Puts a total of numRows + numRowsAfterFlush records indexed with numeric row keys.
private void	replay(org.apache.hadoop.hbase.regionserver.HRegion region, org.apache.hadoop.hbase.client.Put put, long replaySeqId)
(package private) static int	replayEdit(org.apache.hadoop.hbase.regionserver.HRegion region, org.apache.hadoop.hbase.wal.WAL.Entry entry)
void	setUp()
static void	setUpBeforeClass()
void	tearDown()
static void	tearDownAfterClass()
void	testBatchReplayWithMultipleNonces()
void	testOnlyReplayingFlushStartDoesNotHoldUpRegionClose()	Tests a case where we replay only a flush start marker, then the region is closed.
void	testRefresStoreFiles()
void	testRegionReadsEnabledFlag()	Tests the reads enabled flag for the region.
void	testRegionReplicaSecondaryCannotFlush()
void	testReplayBulkLoadEvent()	Tests replaying region open markers from primary region.
void	testReplayFlushCommitMarkerLargerThanFlushStartMarker()	Tests the case where we prepare a flush with some seqId and we receive a flush commit marker larger than the previous flush start marker.
void	testReplayFlushCommitMarkerSmallerThanFlushStartMarker()	Tests the case where we prepare a flush with some seqId and we receive a flush commit marker less than the previous flush start marker.
void	testReplayFlushCommitMarkerWithoutFlushStartMarker(boolean droppableMemstore)	Tests the case where we receive a flush commit before receiving any flush prepare markers
void	testReplayFlushCommitMarkerWithoutFlushStartMarkerDroppableMemstore()	Tests the case where we receive a flush commit before receiving any flush prepare markers.
void	testReplayFlushCommitMarkerWithoutFlushStartMarkerNonDroppableMemstore()	Tests the case where we receive a flush commit before receiving any flush prepare markers.
void	testReplayFlushesAndCompactions()
void	testReplayFlushSeqIds()
void	testReplayFlushStartMarkers()	Tests cases where we prepare a flush with some seqId and we receive other flush start markers equal to, greater or less than the previous flush start marker.
void	testReplayingBulkLoadEventWithFileAlreadyDeleted()
void	testReplayingCompactionWithFileAlreadyDeleted()
void	testReplayingFlushCommitWithFileAlreadyDeleted()
void	testReplayingFlushRequestRestoresReadsEnabledState()	Test the case where the secondary region replica is not in reads enabled state because it is waiting for a flush or region open marker from primary region.
void	testReplayingFlushRestoresReadsEnabledState()	Test the case where the secondary region replica is not in reads enabled state because it is waiting for a flush or region open marker from primary region.
void	testReplayingFlushWithEmptyMemstoreRestoresReadsEnabledState()	Test the case where the secondary region replica is not in reads enabled state because it is waiting for a flush or region open marker from primary region.
void	testReplayingRegionOpenEventRestoresReadsEnabledState()	Test the case where the secondary region replica is not in reads enabled state because it is waiting for a flush or region open marker from primary region.
void	testReplayingRegionOpenEventWithFileAlreadyDeleted()
void	testReplayRegionOpenEvent()	Tests replaying region open markers from primary region.
void	testReplayRegionOpenEventAfterFlushStart()	Tests the case where we replay a region open event after a flush start but before receiving flush commit
void	testSecondaryRegionDoesNotWriteRegionEventsToWAL()	Tests that a region opened in secondary mode would not write region open / close events to its WAL.
void	testSeqIdsFromReplay()
void	testSkippingEditsWithSmallerSeqIdAfterRegionOpenEvent()	Tests whether edits coming in for replay are skipped which have smaller seq id than the seqId of the last replayed region open event.
void	testWriteFlushRequestMarker()	Tests the case where a request for flush cache is sent to the region, but region cannot flush.

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Details

CLASS_RULE

public static final HBaseClassTestRule CLASS_RULE

LOG

private static final org.slf4j.Logger LOG

name

public org.junit.rules.TestName name

TEST_UTIL

private static HBaseTestingUtility TEST_UTIL

CONF

public static org.apache.hadoop.conf.Configuration CONF

dir

private String dir

families

private byte[][] families

tableName

protected byte[] tableName

method

protected String method

row

protected final byte[] row

row2

protected final byte[] row2

cq

protected byte[] cq

rootDir

private org.apache.hadoop.fs.Path rootDir

htd

private org.apache.hadoop.hbase.client.TableDescriptor htd

rss

private org.apache.hadoop.hbase.regionserver.RegionServerServices rss

primaryHri

private org.apache.hadoop.hbase.client.RegionInfo primaryHri

secondaryHri

private org.apache.hadoop.hbase.client.RegionInfo secondaryHri

primaryRegion

private org.apache.hadoop.hbase.regionserver.HRegion primaryRegion

secondaryRegion

private org.apache.hadoop.hbase.regionserver.HRegion secondaryRegion

walPrimary

private org.apache.hadoop.hbase.wal.WAL walPrimary

walSecondary

private org.apache.hadoop.hbase.wal.WAL walSecondary

reader

private org.apache.hadoop.hbase.wal.WALStreamReader reader

Constructor Details

TestHRegionReplayEvents

public TestHRegionReplayEvents()

Method Details

setUpBeforeClass

public static void setUpBeforeClass()
                             throws Exception

Throws:

Exception

tearDownAfterClass

public static void tearDownAfterClass()
                               throws Exception

Throws:

Exception

setUp

public void setUp()
           throws Exception

Throws:

Exception

tearDown

public void tearDown()
              throws Exception

Throws:

Exception

getName

String getName()

testRegionReplicaSecondaryCannotFlush

public void testRegionReplicaSecondaryCannotFlush()
                                           throws IOException

Throws:

IOException

testOnlyReplayingFlushStartDoesNotHoldUpRegionClose

public void testOnlyReplayingFlushStartDoesNotHoldUpRegionClose()
                                                         throws IOException

Tests a case where we replay only a flush start marker, then the region is closed. This region should not block indefinitely

Throws:

IOException

replayEdit

static int replayEdit(org.apache.hadoop.hbase.regionserver.HRegion region,
 org.apache.hadoop.hbase.wal.WAL.Entry entry)
               throws IOException

Throws:

IOException

createWALReaderForPrimary

private org.apache.hadoop.hbase.wal.WALStreamReader createWALReaderForPrimary()
                                                                       throws FileNotFoundException,
IOException

Throws:

FileNotFoundException

IOException

testBatchReplayWithMultipleNonces

public void testBatchReplayWithMultipleNonces()
                                       throws IOException

Throws:

IOException

testReplayFlushesAndCompactions

public void testReplayFlushesAndCompactions()
                                     throws IOException

Throws:

IOException

testReplayFlushStartMarkers

public void testReplayFlushStartMarkers()
                                 throws IOException

Tests cases where we prepare a flush with some seqId and we receive other flush start markers equal to, greater or less than the previous flush start marker.

Throws:

IOException

testReplayFlushCommitMarkerSmallerThanFlushStartMarker

public void testReplayFlushCommitMarkerSmallerThanFlushStartMarker()
                                                            throws IOException

Tests the case where we prepare a flush with some seqId and we receive a flush commit marker less than the previous flush start marker.

Throws:

IOException

testReplayFlushCommitMarkerLargerThanFlushStartMarker

public void testReplayFlushCommitMarkerLargerThanFlushStartMarker()
                                                           throws IOException

Tests the case where we prepare a flush with some seqId and we receive a flush commit marker larger than the previous flush start marker.

Throws:

IOException

testReplayFlushCommitMarkerWithoutFlushStartMarkerDroppableMemstore

public void testReplayFlushCommitMarkerWithoutFlushStartMarkerDroppableMemstore()
                                                                         throws IOException

Tests the case where we receive a flush commit before receiving any flush prepare markers. The memstore edits should be dropped after the flush commit replay since they should be in flushed files

Throws:

IOException

testReplayFlushCommitMarkerWithoutFlushStartMarkerNonDroppableMemstore

public void testReplayFlushCommitMarkerWithoutFlushStartMarkerNonDroppableMemstore()
                                                                            throws IOException

Tests the case where we receive a flush commit before receiving any flush prepare markers. The memstore edits should be not dropped after the flush commit replay since not every edit will be in flushed files (based on seqId)

Throws:

IOException

testReplayFlushCommitMarkerWithoutFlushStartMarker

public void testReplayFlushCommitMarkerWithoutFlushStartMarker(boolean droppableMemstore)
                                                        throws IOException

Tests the case where we receive a flush commit before receiving any flush prepare markers

Throws:

IOException

clone

private org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor clone(org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor flush,
 long flushSeqId)

testReplayRegionOpenEvent

public void testReplayRegionOpenEvent()
                               throws IOException

Tests replaying region open markers from primary region. Checks whether the files are picked up

Throws:

IOException

testReplayRegionOpenEventAfterFlushStart

public void testReplayRegionOpenEventAfterFlushStart()
                                              throws IOException

Tests the case where we replay a region open event after a flush start but before receiving flush commit

Throws:

IOException

testSkippingEditsWithSmallerSeqIdAfterRegionOpenEvent

public void testSkippingEditsWithSmallerSeqIdAfterRegionOpenEvent()
                                                           throws IOException

Tests whether edits coming in for replay are skipped which have smaller seq id than the seqId of the last replayed region open event.

Throws:

IOException

testReplayFlushSeqIds

public void testReplayFlushSeqIds()
                           throws IOException

Throws:

IOException

testSeqIdsFromReplay

public void testSeqIdsFromReplay()
                          throws IOException

Throws:

IOException

testSecondaryRegionDoesNotWriteRegionEventsToWAL

public void testSecondaryRegionDoesNotWriteRegionEventsToWAL()
                                                      throws IOException

Tests that a region opened in secondary mode would not write region open / close events to its WAL.

Throws:

IOException

testRegionReadsEnabledFlag

public void testRegionReadsEnabledFlag()
                                throws IOException

Tests the reads enabled flag for the region. When unset all reads should be rejected

Throws:

IOException

testWriteFlushRequestMarker

public void testWriteFlushRequestMarker()
                                 throws IOException

Tests the case where a request for flush cache is sent to the region, but region cannot flush. It should write the flush request marker instead.

Throws:

IOException

testReplayingFlushRequestRestoresReadsEnabledState

public void testReplayingFlushRequestRestoresReadsEnabledState()
                                                        throws IOException

Test the case where the secondary region replica is not in reads enabled state because it is waiting for a flush or region open marker from primary region. Replaying CANNOT_FLUSH flush marker entry should restore the reads enabled status in the region and allow the reads to continue.

Throws:

IOException

testReplayingFlushRestoresReadsEnabledState

public void testReplayingFlushRestoresReadsEnabledState()
                                                 throws IOException

Test the case where the secondary region replica is not in reads enabled state because it is waiting for a flush or region open marker from primary region. Replaying flush start and commit entries should restore the reads enabled status in the region and allow the reads to continue.

Throws:

IOException

testReplayingFlushWithEmptyMemstoreRestoresReadsEnabledState

public void testReplayingFlushWithEmptyMemstoreRestoresReadsEnabledState()
                                                                  throws IOException

Test the case where the secondary region replica is not in reads enabled state because it is waiting for a flush or region open marker from primary region. Replaying flush start and commit entries should restore the reads enabled status in the region and allow the reads to continue.

Throws:

IOException

testReplayingRegionOpenEventRestoresReadsEnabledState

public void testReplayingRegionOpenEventRestoresReadsEnabledState()
                                                           throws IOException

Test the case where the secondary region replica is not in reads enabled state because it is waiting for a flush or region open marker from primary region. Replaying region open event entry from primary should restore the reads enabled status in the region and allow the reads to continue.

Throws:

IOException

testRefresStoreFiles

public void testRefresStoreFiles()
                          throws IOException

Throws:

IOException

assertPathListsEqual

private void assertPathListsEqual(List<String> list1,
 List<String> list2)

Paths can be qualified or not. This does the assertion using String->Path conversion.

disableReads

private void disableReads(org.apache.hadoop.hbase.regionserver.HRegion region)

replay

private void replay(org.apache.hadoop.hbase.regionserver.HRegion region,
 org.apache.hadoop.hbase.client.Put put,
 long replaySeqId)
             throws IOException

Throws:

IOException

testReplayBulkLoadEvent

public void testReplayBulkLoadEvent()
                             throws IOException

Tests replaying region open markers from primary region. Checks whether the files are picked up

Throws:

IOException

testReplayingFlushCommitWithFileAlreadyDeleted

public void testReplayingFlushCommitWithFileAlreadyDeleted()
                                                    throws IOException

Throws:

IOException

testReplayingCompactionWithFileAlreadyDeleted

public void testReplayingCompactionWithFileAlreadyDeleted()
                                                   throws IOException

Throws:

IOException

testReplayingRegionOpenEventWithFileAlreadyDeleted

public void testReplayingRegionOpenEventWithFileAlreadyDeleted()
                                                        throws IOException

Throws:

IOException

testReplayingBulkLoadEventWithFileAlreadyDeleted

public void testReplayingBulkLoadEventWithFileAlreadyDeleted()
                                                      throws IOException

Throws:

IOException

createHFileForFamilies

private String createHFileForFamilies(org.apache.hadoop.fs.Path testPath,
 byte[] family,
 byte[] valueBytes)
                               throws IOException

Throws:

IOException

putDataWithFlushes

private void putDataWithFlushes(org.apache.hadoop.hbase.regionserver.HRegion region,
 int flushInterval,
 int numRows,
 int numRowsAfterFlush)
                         throws IOException

Puts a total of numRows + numRowsAfterFlush records indexed with numeric row keys. Does a flush every flushInterval number of records. Then it puts numRowsAfterFlush number of more rows but does not execute flush after

Throws:

IOException

putDataByReplay

private void putDataByReplay(org.apache.hadoop.hbase.regionserver.HRegion region,
 int startRow,
 int numRows,
 byte[] qf,
 byte[]... families)
                      throws IOException

Throws:

IOException

initHRegion

private static org.apache.hadoop.hbase.regionserver.HRegion initHRegion(byte[] tableName,
 String callingMethod,
 byte[]... families)
                                                                 throws IOException

Throws:

IOException

initHRegion

private static org.apache.hadoop.hbase.regionserver.HRegion initHRegion(byte[] tableName,
 byte[] startKey,
 byte[] stopKey,
 String callingMethod,
 org.apache.hadoop.conf.Configuration conf,
 boolean isReadOnly,
 org.apache.hadoop.hbase.client.Durability durability,
 org.apache.hadoop.hbase.wal.WAL wal,
 byte[]... families)
                                                                 throws IOException

Throws:

IOException