Package org.apache.hadoop.hbase.mapreduce

Class SyncTable.SyncMapper

java.lang.Object

org.apache.hadoop.mapreduce.Mapper<ImmutableBytesWritable,Result,KEYOUT,VALUEOUT>

org.apache.hadoop.hbase.mapreduce.TableMapper<ImmutableBytesWritable,Mutation>

org.apache.hadoop.hbase.mapreduce.SyncTable.SyncMapper

Enclosing class:

SyncTable

public static class SyncTable.SyncMapper
extends TableMapper<ImmutableBytesWritable,Mutation>

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
private static class	SyncTable.SyncMapper.CellScanner
static enum	SyncTable.SyncMapper.Counter

Nested classes/interfaces inherited from class org.apache.hadoop.mapreduce.Mapper

org.apache.hadoop.mapreduce.Mapper.Context

Field Summary

Fields

Modifier and Type	Field	Description
(package private) ImmutableBytesWritable	currentSourceHash
(package private) boolean	doDeletes
(package private) boolean	doPuts
(package private) boolean	dryRun
private static final SyncTable.SyncMapper.CellScanner	EMPTY_CELL_SCANNER
(package private) boolean	ignoreTimestamp
(package private) Throwable	mapperException
(package private) ImmutableBytesWritable	nextSourceKey
(package private) Connection	sourceConnection
(package private) org.apache.hadoop.fs.Path	sourceHashDir
(package private) HashTable.TableHash.Reader	sourceHashReader
(package private) Table	sourceTable
(package private) HashTable.TableHash	sourceTableHash
(package private) Connection	targetConnection
(package private) HashTable.ResultHasher	targetHasher
(package private) Table	targetTable

Constructor Summary

Constructors

Constructor	Description
SyncMapper()

Method Summary

Modifier and Type	Method	Description
private Cell	checkAndResetTimestamp(Cell sourceCell)
protected void	cleanup(org.apache.hadoop.mapreduce.Mapper<ImmutableBytesWritable,Result,ImmutableBytesWritable,Mutation>.org.apache.hadoop.mapreduce.Mapper.Context context)
private int	compareCellKeysWithinRow(Cell c1, Cell c2)	Compare families, qualifiers, and timestamps of the given Cells.
private static int	compareRowKeys(byte[] r1, byte[] r2)	Compare row keys of the given Result objects.
private void	findNextKeyHashPair()	Attempt to read the next source key/hash pair.
private void	finishBatchAndCompareHashes(org.apache.hadoop.mapreduce.Mapper<ImmutableBytesWritable,Result,ImmutableBytesWritable,Mutation>.org.apache.hadoop.mapreduce.Mapper.Context context)	Finish the currently open hash batch.
private void	finishRemainingHashRanges(org.apache.hadoop.mapreduce.Mapper<ImmutableBytesWritable,Result,ImmutableBytesWritable,Mutation>.org.apache.hadoop.mapreduce.Mapper.Context context)
protected void	map(ImmutableBytesWritable key, Result value, org.apache.hadoop.mapreduce.Mapper<ImmutableBytesWritable,Result,ImmutableBytesWritable,Mutation>.org.apache.hadoop.mapreduce.Mapper.Context context)
private void	moveToNextBatch(org.apache.hadoop.mapreduce.Mapper<ImmutableBytesWritable,Result,ImmutableBytesWritable,Mutation>.org.apache.hadoop.mapreduce.Mapper.Context context)	If there is an open hash batch, complete it and sync if there are diffs.
private static Connection	openConnection(org.apache.hadoop.conf.Configuration conf, String zkClusterConfKey, String configPrefix)
private static Table	openTable(Connection connection, org.apache.hadoop.conf.Configuration conf, String tableNameConfKey)
protected void	setup(org.apache.hadoop.mapreduce.Mapper<ImmutableBytesWritable,Result,ImmutableBytesWritable,Mutation>.org.apache.hadoop.mapreduce.Mapper.Context context)
private void	syncRange(org.apache.hadoop.mapreduce.Mapper<ImmutableBytesWritable,Result,ImmutableBytesWritable,Mutation>.org.apache.hadoop.mapreduce.Mapper.Context context, ImmutableBytesWritable startRow, ImmutableBytesWritable stopRow)	Rescan the given range directly from the source and target tables.
private boolean	syncRowCells(org.apache.hadoop.mapreduce.Mapper<ImmutableBytesWritable,Result,ImmutableBytesWritable,Mutation>.org.apache.hadoop.mapreduce.Mapper.Context context, byte[] rowKey, SyncTable.SyncMapper.CellScanner sourceCells, SyncTable.SyncMapper.CellScanner targetCells)	Compare the cells for the given row from the source and target tables.
private static String	toHex(ImmutableBytesWritable bytes)

Methods inherited from class org.apache.hadoop.mapreduce.Mapper

run

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Details

sourceHashDir

org.apache.hadoop.fs.Path sourceHashDir

sourceConnection

Connection sourceConnection

targetConnection

Connection targetConnection

sourceTable

Table sourceTable

targetTable

Table targetTable

dryRun

boolean dryRun

doDeletes

boolean doDeletes

doPuts

boolean doPuts

ignoreTimestamp

boolean ignoreTimestamp

sourceTableHash

HashTable.TableHash sourceTableHash

sourceHashReader

HashTable.TableHash.Reader sourceHashReader

currentSourceHash

ImmutableBytesWritable currentSourceHash

nextSourceKey

ImmutableBytesWritable nextSourceKey

targetHasher

HashTable.ResultHasher targetHasher

mapperException

Throwable mapperException

EMPTY_CELL_SCANNER

private static final SyncTable.SyncMapper.CellScanner EMPTY_CELL_SCANNER

Constructor Details

SyncMapper

public SyncMapper()

Method Details

setup

protected void setup(org.apache.hadoop.mapreduce.Mapper<ImmutableBytesWritable,Result,ImmutableBytesWritable,Mutation>.org.apache.hadoop.mapreduce.Mapper.Context context)
              throws IOException

Overrides:

setup in class org.apache.hadoop.mapreduce.Mapper<ImmutableBytesWritable,Result,ImmutableBytesWritable,Mutation>

Throws:

IOException

openConnection

private static Connection openConnection(org.apache.hadoop.conf.Configuration conf,
 String zkClusterConfKey,
 String configPrefix)
                                  throws IOException

Throws:

IOException

openTable

private static Table openTable(Connection connection,
 org.apache.hadoop.conf.Configuration conf,
 String tableNameConfKey)
                        throws IOException

Throws:

IOException

findNextKeyHashPair

private void findNextKeyHashPair()
                          throws IOException

Attempt to read the next source key/hash pair. If there are no more, set nextSourceKey to null

Throws:

IOException

map

protected void map(ImmutableBytesWritable key,
 Result value,
 org.apache.hadoop.mapreduce.Mapper<ImmutableBytesWritable,Result,ImmutableBytesWritable,Mutation>.org.apache.hadoop.mapreduce.Mapper.Context context)
            throws IOException,
InterruptedException

Overrides:

map in class org.apache.hadoop.mapreduce.Mapper<ImmutableBytesWritable,Result,ImmutableBytesWritable,Mutation>

Throws:

IOException

InterruptedException

moveToNextBatch

private void moveToNextBatch(org.apache.hadoop.mapreduce.Mapper<ImmutableBytesWritable,Result,ImmutableBytesWritable,Mutation>.org.apache.hadoop.mapreduce.Mapper.Context context)
                      throws IOException,
InterruptedException

If there is an open hash batch, complete it and sync if there are diffs. Start a new batch, and seek to read the

Throws:

IOException

InterruptedException

finishBatchAndCompareHashes

private void finishBatchAndCompareHashes(org.apache.hadoop.mapreduce.Mapper<ImmutableBytesWritable,Result,ImmutableBytesWritable,Mutation>.org.apache.hadoop.mapreduce.Mapper.Context context)
                                  throws IOException,
InterruptedException

Finish the currently open hash batch. Compare the target hash to the given source hash. If they do not match, then sync the covered key range.

Throws:

IOException

InterruptedException

toHex

private static String toHex(ImmutableBytesWritable bytes)

syncRange

private void syncRange(org.apache.hadoop.mapreduce.Mapper<ImmutableBytesWritable,Result,ImmutableBytesWritable,Mutation>.org.apache.hadoop.mapreduce.Mapper.Context context,
 ImmutableBytesWritable startRow,
 ImmutableBytesWritable stopRow)
                throws IOException,
InterruptedException

Rescan the given range directly from the source and target tables. Count and log differences, and if this is not a dry run, output Puts and Deletes to make the target table match the source table for this range

Throws:

IOException

InterruptedException

checkAndResetTimestamp

private Cell checkAndResetTimestamp(Cell sourceCell)

syncRowCells

private boolean syncRowCells(org.apache.hadoop.mapreduce.Mapper<ImmutableBytesWritable,Result,ImmutableBytesWritable,Mutation>.org.apache.hadoop.mapreduce.Mapper.Context context,
 byte[] rowKey,
 SyncTable.SyncMapper.CellScanner sourceCells,
 SyncTable.SyncMapper.CellScanner targetCells)
                      throws IOException,
InterruptedException

Compare the cells for the given row from the source and target tables. Count and log any differences. If not a dry run, output a Put and/or Delete needed to sync the target table to match the source table.

Throws:

IOException

InterruptedException

compareRowKeys

private static int compareRowKeys(byte[] r1,
 byte[] r2)

Compare row keys of the given Result objects. Nulls are after non-nulls

compareCellKeysWithinRow

private int compareCellKeysWithinRow(Cell c1,
 Cell c2)

Compare families, qualifiers, and timestamps of the given Cells. They are assumed to be of the same row. Nulls are after non-nulls.

cleanup

protected void cleanup(org.apache.hadoop.mapreduce.Mapper<ImmutableBytesWritable,Result,ImmutableBytesWritable,Mutation>.org.apache.hadoop.mapreduce.Mapper.Context context)
                throws IOException,
InterruptedException

Overrides:

cleanup in class org.apache.hadoop.mapreduce.Mapper<ImmutableBytesWritable,Result,ImmutableBytesWritable,Mutation>

Throws:

IOException

InterruptedException

finishRemainingHashRanges

private void finishRemainingHashRanges(org.apache.hadoop.mapreduce.Mapper<ImmutableBytesWritable,Result,ImmutableBytesWritable,Mutation>.org.apache.hadoop.mapreduce.Mapper.Context context)
                                throws IOException,
InterruptedException

Throws:

IOException

InterruptedException