Package org.apache.hadoop.hbase.regionserver

Class CompactSplit

java.lang.Object

org.apache.hadoop.hbase.regionserver.CompactSplit

All Implemented Interfaces:

ConfigurationObserver, PropagatingConfigurationObserver, CompactionRequester

@Private
public class CompactSplit
extends Object
implements CompactionRequester, PropagatingConfigurationObserver

Compact region on request and then run split if appropriate

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
private static final class	CompactSplit.AggregatingCompleteTracker
protected static interface	CompactSplit.CompactionCompleteTracker
private final class	CompactSplit.CompactionRunner

Field Summary

Fields

Modifier and Type	Field	Description
private boolean	compactionsEnabled
private ThroughputController	compactionThroughputController
private static final Comparator<Runnable>	COMPARATOR
private final org.apache.hadoop.conf.Configuration	conf
static final int	DEFAULT_REGION_SERVER_REGION_SPLIT_LIMIT
private static final CompactSplit.CompactionCompleteTracker	DUMMY_COMPLETE_TRACKER
static final String	HBASE_REGION_SERVER_ENABLE_COMPACTION
static final String	LARGE_COMPACTION_THREADS
static final int	LARGE_COMPACTION_THREADS_DEFAULT
private static final org.slf4j.Logger	LOG
private ThreadPoolExecutor	longCompactions
static final String	REGION_SERVER_REGION_SPLIT_LIMIT
private int	regionSplitLimit	Splitting should not take place if the total number of regions exceed this.
private final HRegionServer	server
private ThreadPoolExecutor	shortCompactions
static final String	SMALL_COMPACTION_THREADS
static final int	SMALL_COMPACTION_THREADS_DEFAULT
static final String	SPLIT_THREADS
static final int	SPLIT_THREADS_DEFAULT
private ThreadPoolExecutor	splits
private Set<String>	underCompactionStores

Constructor Summary

Constructors

Constructor	Description
CompactSplit(org.apache.hadoop.conf.Configuration conf)
CompactSplit(HRegionServer server)

Method Summary

Modifier and Type	Method	Description
void	clearLongCompactionsQueue()
void	clearShortCompactionsQueue()
private void	createCompactionExecutors()
private void	createSplitExcecutors()
void	deregisterChildren(ConfigurationManager manager)	Needs to be called to deregister the children from the manager.
String	dumpQueue()
int	getCompactionQueueSize()	Returns the current size of the queue containing regions that are processed.
ThroughputController	getCompactionThroughputController()
private CompactSplit.CompactionCompleteTracker	getCompleteTracker(CompactionLifeCycleTracker tracker, IntSupplier numberOfStores)
int	getLargeCompactionQueueSize()
protected int	getLargeCompactionThreadNum()
(package private) ThreadPoolExecutor	getLongCompactions()	Returns the longCompactions thread pool executor
int	getRegionSplitLimit()	Returns the regionSplitLimit
(package private) ThreadPoolExecutor	getShortCompactions()	Returns the shortCompactions thread pool executor
int	getSmallCompactionQueueSize()
protected int	getSmallCompactionThreadNum()
int	getSplitQueueSize()
protected int	getSplitThreadNum()
private String	getStoreNameForUnderCompaction(HStore store)
private void	interrupt()
(package private) void	interruptIfNecessary()	Only interrupt once it's done with a run through the work loop.
boolean	isCompactionsEnabled()
boolean	isUnderCompaction(HStore s)	Check if this store is under compaction
(package private) void	join()
void	onConfigurationChange(org.apache.hadoop.conf.Configuration newConf)	This method would be called by the ConfigurationManager object when the Configuration object is reloaded from disk.
void	registerChildren(ConfigurationManager manager)	Needs to be called to register the children to the manager.
private void	reInitializeCompactionsExecutors()
void	requestCompaction(HRegion region, String why, int priority, CompactionLifeCycleTracker tracker, User user)	Request compaction on all the stores of the given region.
void	requestCompaction(HRegion region, HStore store, String why, int priority, CompactionLifeCycleTracker tracker, User user)	Request compaction on the given store.
private void	requestCompactionInternal(HRegion region, String why, int priority, boolean selectNow, CompactionLifeCycleTracker tracker, CompactSplit.CompactionCompleteTracker completeTracker, User user)
protected void	requestCompactionInternal(HRegion region, HStore store, String why, int priority, boolean selectNow, CompactionLifeCycleTracker tracker, CompactSplit.CompactionCompleteTracker completeTracker, User user)
boolean	requestSplit(Region r)
private void	requestSplit(Region r, byte[] midKey)
private void	requestSplit(Region r, byte[] midKey, User user)
void	requestSystemCompaction(HRegion region, String why)
void	requestSystemCompaction(HRegion region, HStore store, String why)
void	requestSystemCompaction(HRegion region, HStore store, String why, boolean giveUpIfRequestedOrCompacting)	Request system compaction on the given store.
private Optional<CompactionContext>	selectCompaction(HRegion region, HStore store, int priority, CompactionLifeCycleTracker tracker, CompactSplit.CompactionCompleteTracker completeTracker, User user)
void	setCompactionsEnabled(boolean compactionsEnabled)
private boolean	shouldSplitRegion()
(package private) void	shutdownLongCompactions()	Shutdown the long compaction thread pool.
void	switchCompaction(boolean onOrOff)	on/off compaction
String	toString()
private void	waitFor(ThreadPoolExecutor t, String name)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait

Field Details

LOG

private static final org.slf4j.Logger LOG

LARGE_COMPACTION_THREADS

public static final String LARGE_COMPACTION_THREADS

See Also:

• Constant Field Values

LARGE_COMPACTION_THREADS_DEFAULT

public static final int LARGE_COMPACTION_THREADS_DEFAULT

See Also:

• Constant Field Values

SMALL_COMPACTION_THREADS

public static final String SMALL_COMPACTION_THREADS

See Also:

• Constant Field Values

SMALL_COMPACTION_THREADS_DEFAULT

public static final int SMALL_COMPACTION_THREADS_DEFAULT

See Also:

• Constant Field Values

SPLIT_THREADS

public static final String SPLIT_THREADS

See Also:

• Constant Field Values

SPLIT_THREADS_DEFAULT

public static final int SPLIT_THREADS_DEFAULT

See Also:

• Constant Field Values

REGION_SERVER_REGION_SPLIT_LIMIT

public static final String REGION_SERVER_REGION_SPLIT_LIMIT

See Also:

• Constant Field Values

DEFAULT_REGION_SERVER_REGION_SPLIT_LIMIT

public static final int DEFAULT_REGION_SERVER_REGION_SPLIT_LIMIT

See Also:

• Constant Field Values

HBASE_REGION_SERVER_ENABLE_COMPACTION

public static final String HBASE_REGION_SERVER_ENABLE_COMPACTION

See Also:

• Constant Field Values

server

private final HRegionServer server

conf

private final org.apache.hadoop.conf.Configuration conf

longCompactions

private volatile ThreadPoolExecutor longCompactions

shortCompactions

private volatile ThreadPoolExecutor shortCompactions

splits

private volatile ThreadPoolExecutor splits

compactionThroughputController

private volatile ThroughputController compactionThroughputController

underCompactionStores

private volatile Set<String> underCompactionStores

compactionsEnabled

private volatile boolean compactionsEnabled

regionSplitLimit

private int regionSplitLimit

Splitting should not take place if the total number of regions exceed this. This is not a hard limit to the number of regions but it is a guideline to stop splitting after number of online regions is greater than this.

DUMMY_COMPLETE_TRACKER

private static final CompactSplit.CompactionCompleteTracker DUMMY_COMPLETE_TRACKER

COMPARATOR

private static final Comparator<Runnable> COMPARATOR

Constructor Details

CompactSplit

CompactSplit(HRegionServer server)

CompactSplit

public CompactSplit(org.apache.hadoop.conf.Configuration conf)

Method Details

createSplitExcecutors

private void createSplitExcecutors()

createCompactionExecutors

private void createCompactionExecutors()

toString

public String toString()

Overrides:

toString in class Object

dumpQueue

public String dumpQueue()

requestSplit

public boolean requestSplit(Region r)

requestSplit

private void requestSplit(Region r,
 byte[] midKey)

requestSplit

private void requestSplit(Region r,
 byte[] midKey,
 User user)

interrupt

private void interrupt()

reInitializeCompactionsExecutors

private void reInitializeCompactionsExecutors()

getCompleteTracker

private CompactSplit.CompactionCompleteTracker getCompleteTracker(CompactionLifeCycleTracker tracker,
 IntSupplier numberOfStores)

requestCompaction

public void requestCompaction(HRegion region,
 String why,
 int priority,
 CompactionLifeCycleTracker tracker,
 User user)
                       throws IOException

Description copied from interface: CompactionRequester

Request compaction on all the stores of the given region.

Specified by:

requestCompaction in interface CompactionRequester

Throws:

IOException

requestCompaction

public void requestCompaction(HRegion region,
 HStore store,
 String why,
 int priority,
 CompactionLifeCycleTracker tracker,
 User user)
                       throws IOException

Description copied from interface: CompactionRequester

Request compaction on the given store.

Specified by:

requestCompaction in interface CompactionRequester

Throws:

IOException

switchCompaction

public void switchCompaction(boolean onOrOff)

Description copied from interface: CompactionRequester

on/off compaction

Specified by:

switchCompaction in interface CompactionRequester

requestCompactionInternal

private void requestCompactionInternal(HRegion region,
 String why,
 int priority,
 boolean selectNow,
 CompactionLifeCycleTracker tracker,
 CompactSplit.CompactionCompleteTracker completeTracker,
 User user)
                                throws IOException

Throws:

IOException

requestCompactionInternal

protected void requestCompactionInternal(HRegion region,
 HStore store,
 String why,
 int priority,
 boolean selectNow,
 CompactionLifeCycleTracker tracker,
 CompactSplit.CompactionCompleteTracker completeTracker,
 User user)
                                  throws IOException

Throws:

IOException

requestSystemCompaction

public void requestSystemCompaction(HRegion region,
 String why)
                             throws IOException

Throws:

IOException

requestSystemCompaction

public void requestSystemCompaction(HRegion region,
 HStore store,
 String why)
                             throws IOException

Throws:

IOException

requestSystemCompaction

public void requestSystemCompaction(HRegion region,
 HStore store,
 String why,
 boolean giveUpIfRequestedOrCompacting)
                             throws IOException

Description copied from interface: CompactionRequester

Request system compaction on the given store.

Specified by:

requestSystemCompaction in interface CompactionRequester

Throws:

IOException

selectCompaction

private Optional<CompactionContext> selectCompaction(HRegion region,
 HStore store,
 int priority,
 CompactionLifeCycleTracker tracker,
 CompactSplit.CompactionCompleteTracker completeTracker,
 User user)
                                              throws IOException

Throws:

IOException

interruptIfNecessary

void interruptIfNecessary()

Only interrupt once it's done with a run through the work loop.

waitFor

private void waitFor(ThreadPoolExecutor t,
 String name)

join

void join()

getCompactionQueueSize

public int getCompactionQueueSize()

Returns the current size of the queue containing regions that are processed.

Returns:

The current size of the regions queue.

getLargeCompactionQueueSize

public int getLargeCompactionQueueSize()

getSmallCompactionQueueSize

public int getSmallCompactionQueueSize()

getSplitQueueSize

public int getSplitQueueSize()

shouldSplitRegion

private boolean shouldSplitRegion()

getRegionSplitLimit

public int getRegionSplitLimit()

Returns the regionSplitLimit

isUnderCompaction

public boolean isUnderCompaction(HStore s)

Check if this store is under compaction

onConfigurationChange

public void onConfigurationChange(org.apache.hadoop.conf.Configuration newConf)

This method would be called by the ConfigurationManager object when the Configuration object is reloaded from disk.

Specified by:

onConfigurationChange in interface ConfigurationObserver

getSmallCompactionThreadNum

protected int getSmallCompactionThreadNum()

getLargeCompactionThreadNum

protected int getLargeCompactionThreadNum()

getSplitThreadNum

protected int getSplitThreadNum()

registerChildren

public void registerChildren(ConfigurationManager manager)

Needs to be called to register the children to the manager.

Specified by:

registerChildren in interface PropagatingConfigurationObserver

Parameters:

manager - : to register to

deregisterChildren

public void deregisterChildren(ConfigurationManager manager)

Needs to be called to deregister the children from the manager.

Specified by:

deregisterChildren in interface PropagatingConfigurationObserver

Parameters:

manager - : to deregister from

getCompactionThroughputController

public ThroughputController getCompactionThroughputController()

shutdownLongCompactions

void shutdownLongCompactions()

Shutdown the long compaction thread pool. Should only be used in unit test to prevent long compaction thread pool from stealing job from short compaction queue

clearLongCompactionsQueue

public void clearLongCompactionsQueue()

clearShortCompactionsQueue

public void clearShortCompactionsQueue()

isCompactionsEnabled

public boolean isCompactionsEnabled()

setCompactionsEnabled

public void setCompactionsEnabled(boolean compactionsEnabled)

getLongCompactions

ThreadPoolExecutor getLongCompactions()

Returns the longCompactions thread pool executor

getShortCompactions

ThreadPoolExecutor getShortCompactions()

Returns the shortCompactions thread pool executor

getStoreNameForUnderCompaction

private String getStoreNameForUnderCompaction(HStore store)