Package org.apache.hadoop.hbase.io.asyncfs

Class FanOutOneBlockAsyncDFSOutputHelper

java.lang.Object

org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper

@Private
public final class FanOutOneBlockAsyncDFSOutputHelper
extends Object

Helper class for implementing FanOutOneBlockAsyncDFSOutput.

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
(package private) static final class	FanOutOneBlockAsyncDFSOutputHelper.CancelOnClose
private static interface	FanOutOneBlockAsyncDFSOutputHelper.FileCreator
private static interface	FanOutOneBlockAsyncDFSOutputHelper.LeaseManager
static class	FanOutOneBlockAsyncDFSOutputHelper.NameNodeException	Exception other than RemoteException thrown when calling create on namenode

Field Summary

Fields

Modifier and Type	Field	Description
private static final org.apache.hbase.thirdparty.io.netty.buffer.ByteBufAllocator	ALLOC
static final String	ASYNC_DFS_OUTPUT_CREATE_MAX_RETRIES
static final int	DEFAULT_ASYNC_DFS_OUTPUT_CREATE_MAX_RETRIES
private static final FanOutOneBlockAsyncDFSOutputHelper.FileCreator	FILE_CREATOR
static final long	HEART_BEAT_SEQNO
private static final FanOutOneBlockAsyncDFSOutputHelper.LeaseManager	LEASE_MANAGER
private static final org.slf4j.Logger	LOG
static final int	READ_TIMEOUT

Constructor Summary

Constructors

Modifier	Constructor	Description
private	FanOutOneBlockAsyncDFSOutputHelper()

Method Summary

Modifier and Type	Method	Description
private static void	beginFileLease(FanOutOneBlockAsyncDFSOutput output)
(package private) static void	completeFile(FanOutOneBlockAsyncDFSOutput output, org.apache.hadoop.hdfs.DFSClient client, org.apache.hadoop.hdfs.protocol.ClientProtocol namenode, String src, String clientName, org.apache.hadoop.hdfs.protocol.ExtendedBlock block, org.apache.hadoop.hdfs.protocol.HdfsFileStatus stat)
private static List<org.apache.hbase.thirdparty.io.netty.util.concurrent.Future<org.apache.hbase.thirdparty.io.netty.channel.Channel>>	connectToDataNodes(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.hdfs.DFSClient client, String clientName, org.apache.hadoop.hdfs.protocol.LocatedBlock locatedBlock, long maxBytesRcvd, long latestGS, org.apache.hadoop.hdfs.protocol.datatransfer.BlockConstructionStage stage, org.apache.hadoop.util.DataChecksum summer, org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup eventLoopGroup, Class<? extends org.apache.hbase.thirdparty.io.netty.channel.Channel> channelClass)
(package private) static org.apache.hadoop.util.DataChecksum	createChecksum(org.apache.hadoop.hdfs.DFSClient client)
private static FanOutOneBlockAsyncDFSOutputHelper.FileCreator	createFileCreator()
private static FanOutOneBlockAsyncDFSOutputHelper.FileCreator	createFileCreator3()
private static FanOutOneBlockAsyncDFSOutputHelper.FileCreator	createFileCreator3_3()
private static FanOutOneBlockAsyncDFSOutputHelper.LeaseManager	createLeaseManager()
private static FanOutOneBlockAsyncDFSOutputHelper.LeaseManager	createLeaseManager3()
private static FanOutOneBlockAsyncDFSOutputHelper.LeaseManager	createLeaseManager3_4()
private static FanOutOneBlockAsyncDFSOutput	createOutput(org.apache.hadoop.hdfs.DistributedFileSystem dfs, String src, boolean overwrite, boolean createParent, short replication, long blockSize, org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup eventLoopGroup, Class<? extends org.apache.hbase.thirdparty.io.netty.channel.Channel> channelClass, StreamSlowMonitor monitor, boolean noLocalWrite)
static FanOutOneBlockAsyncDFSOutput	createOutput(org.apache.hadoop.hdfs.DistributedFileSystem dfs, org.apache.hadoop.fs.Path f, boolean overwrite, boolean createParent, short replication, long blockSize, org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup eventLoopGroup, Class<? extends org.apache.hbase.thirdparty.io.netty.channel.Channel> channelClass, StreamSlowMonitor monitor, boolean noLocalWrite)	Create a FanOutOneBlockAsyncDFSOutput.
(package private) static void	endFileLease(FanOutOneBlockAsyncDFSOutput output)
private static org.apache.hadoop.io.EnumSetWritable<org.apache.hadoop.fs.CreateFlag>	getCreateFlags(boolean overwrite, boolean noLocalWrite)
static String	getDataNodeInfo(Collection<org.apache.hadoop.hdfs.protocol.DatanodeInfo> datanodeInfos)
(package private) static org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status	getStatus(org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.PipelineAckProto ack)
private static void	initialize(org.apache.hadoop.conf.Configuration conf, org.apache.hbase.thirdparty.io.netty.channel.Channel channel, org.apache.hadoop.hdfs.protocol.DatanodeInfo dnInfo, org.apache.hadoop.fs.StorageType storageType, org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.OpWriteBlockProto.Builder writeBlockProtoBuilder, int timeoutMs, org.apache.hadoop.hdfs.DFSClient client, org.apache.hadoop.security.token.Token<org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier> accessToken, org.apache.hbase.thirdparty.io.netty.util.concurrent.Promise<org.apache.hbase.thirdparty.io.netty.channel.Channel> promise)
private static void	processWriteBlockResponse(org.apache.hbase.thirdparty.io.netty.channel.Channel channel, org.apache.hadoop.hdfs.protocol.DatanodeInfo dnInfo, org.apache.hbase.thirdparty.io.netty.util.concurrent.Promise<org.apache.hbase.thirdparty.io.netty.channel.Channel> promise, int timeoutMs)
private static void	requestWriteBlock(org.apache.hbase.thirdparty.io.netty.channel.Channel channel, org.apache.hadoop.fs.StorageType storageType, org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.OpWriteBlockProto.Builder writeBlockProtoBuilder)
static boolean	shouldRetryCreate(org.apache.hadoop.ipc.RemoteException e)
(package private) static void	sleepIgnoreInterrupt(int retry)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Details

LOG

private static final org.slf4j.Logger LOG

ASYNC_DFS_OUTPUT_CREATE_MAX_RETRIES

public static final String ASYNC_DFS_OUTPUT_CREATE_MAX_RETRIES

See Also:

• Constant Field Values

DEFAULT_ASYNC_DFS_OUTPUT_CREATE_MAX_RETRIES

public static final int DEFAULT_ASYNC_DFS_OUTPUT_CREATE_MAX_RETRIES

See Also:

• Constant Field Values

ALLOC

private static final org.apache.hbase.thirdparty.io.netty.buffer.ByteBufAllocator ALLOC

HEART_BEAT_SEQNO

public static final long HEART_BEAT_SEQNO

See Also:

• Constant Field Values

READ_TIMEOUT

public static final int READ_TIMEOUT

See Also:

• Constant Field Values

LEASE_MANAGER

private static final FanOutOneBlockAsyncDFSOutputHelper.LeaseManager LEASE_MANAGER

FILE_CREATOR

private static final FanOutOneBlockAsyncDFSOutputHelper.FileCreator FILE_CREATOR

Constructor Details

FanOutOneBlockAsyncDFSOutputHelper

private FanOutOneBlockAsyncDFSOutputHelper()

Method Details

createLeaseManager3_4

private static FanOutOneBlockAsyncDFSOutputHelper.LeaseManager createLeaseManager3_4()
                                                                              throws NoSuchMethodException

Throws:

NoSuchMethodException

createLeaseManager3

private static FanOutOneBlockAsyncDFSOutputHelper.LeaseManager createLeaseManager3()
                                                                            throws NoSuchMethodException

Throws:

NoSuchMethodException

createLeaseManager

private static FanOutOneBlockAsyncDFSOutputHelper.LeaseManager createLeaseManager()
                                                                           throws NoSuchMethodException

Throws:

NoSuchMethodException

createFileCreator3_3

private static FanOutOneBlockAsyncDFSOutputHelper.FileCreator createFileCreator3_3()
                                                                            throws NoSuchMethodException

Throws:

NoSuchMethodException

createFileCreator3

private static FanOutOneBlockAsyncDFSOutputHelper.FileCreator createFileCreator3()
                                                                          throws NoSuchMethodException

Throws:

NoSuchMethodException

createFileCreator

private static FanOutOneBlockAsyncDFSOutputHelper.FileCreator createFileCreator()
                                                                         throws NoSuchMethodException

Throws:

NoSuchMethodException

beginFileLease

private static void beginFileLease(FanOutOneBlockAsyncDFSOutput output)

endFileLease

static void endFileLease(FanOutOneBlockAsyncDFSOutput output)

createChecksum

static org.apache.hadoop.util.DataChecksum createChecksum(org.apache.hadoop.hdfs.DFSClient client)

getStatus

static org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status getStatus(org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.PipelineAckProto ack)

processWriteBlockResponse

private static void processWriteBlockResponse(org.apache.hbase.thirdparty.io.netty.channel.Channel channel,
 org.apache.hadoop.hdfs.protocol.DatanodeInfo dnInfo,
 org.apache.hbase.thirdparty.io.netty.util.concurrent.Promise<org.apache.hbase.thirdparty.io.netty.channel.Channel> promise,
 int timeoutMs)

requestWriteBlock

private static void requestWriteBlock(org.apache.hbase.thirdparty.io.netty.channel.Channel channel,
 org.apache.hadoop.fs.StorageType storageType,
 org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.OpWriteBlockProto.Builder writeBlockProtoBuilder)
                               throws IOException

Throws:

IOException

initialize

private static void initialize(org.apache.hadoop.conf.Configuration conf,
 org.apache.hbase.thirdparty.io.netty.channel.Channel channel,
 org.apache.hadoop.hdfs.protocol.DatanodeInfo dnInfo,
 org.apache.hadoop.fs.StorageType storageType,
 org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.OpWriteBlockProto.Builder writeBlockProtoBuilder,
 int timeoutMs,
 org.apache.hadoop.hdfs.DFSClient client,
 org.apache.hadoop.security.token.Token<org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier> accessToken,
 org.apache.hbase.thirdparty.io.netty.util.concurrent.Promise<org.apache.hbase.thirdparty.io.netty.channel.Channel> promise)
                        throws IOException

Throws:

IOException

connectToDataNodes

private static List<org.apache.hbase.thirdparty.io.netty.util.concurrent.Future<org.apache.hbase.thirdparty.io.netty.channel.Channel>> connectToDataNodes(org.apache.hadoop.conf.Configuration conf,
 org.apache.hadoop.hdfs.DFSClient client,
 String clientName,
 org.apache.hadoop.hdfs.protocol.LocatedBlock locatedBlock,
 long maxBytesRcvd,
 long latestGS,
 org.apache.hadoop.hdfs.protocol.datatransfer.BlockConstructionStage stage,
 org.apache.hadoop.util.DataChecksum summer,
 org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup eventLoopGroup,
 Class<? extends org.apache.hbase.thirdparty.io.netty.channel.Channel> channelClass)

getCreateFlags

private static org.apache.hadoop.io.EnumSetWritable<org.apache.hadoop.fs.CreateFlag> getCreateFlags(boolean overwrite,
 boolean noLocalWrite)

createOutput

private static FanOutOneBlockAsyncDFSOutput createOutput(org.apache.hadoop.hdfs.DistributedFileSystem dfs,
 String src,
 boolean overwrite,
 boolean createParent,
 short replication,
 long blockSize,
 org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup eventLoopGroup,
 Class<? extends org.apache.hbase.thirdparty.io.netty.channel.Channel> channelClass,
 StreamSlowMonitor monitor,
 boolean noLocalWrite)
                                                  throws IOException

Throws:

IOException

createOutput

public static FanOutOneBlockAsyncDFSOutput createOutput(org.apache.hadoop.hdfs.DistributedFileSystem dfs,
 org.apache.hadoop.fs.Path f,
 boolean overwrite,
 boolean createParent,
 short replication,
 long blockSize,
 org.apache.hbase.thirdparty.io.netty.channel.EventLoopGroup eventLoopGroup,
 Class<? extends org.apache.hbase.thirdparty.io.netty.channel.Channel> channelClass,
 StreamSlowMonitor monitor,
 boolean noLocalWrite)
                                                 throws IOException

Create a FanOutOneBlockAsyncDFSOutput. The method maybe blocked so do not call it inside an EventLoop.

Throws:

IOException

shouldRetryCreate

public static boolean shouldRetryCreate(org.apache.hadoop.ipc.RemoteException e)

completeFile

static void completeFile(FanOutOneBlockAsyncDFSOutput output,
 org.apache.hadoop.hdfs.DFSClient client,
 org.apache.hadoop.hdfs.protocol.ClientProtocol namenode,
 String src,
 String clientName,
 org.apache.hadoop.hdfs.protocol.ExtendedBlock block,
 org.apache.hadoop.hdfs.protocol.HdfsFileStatus stat)
                  throws IOException

Throws:

IOException

sleepIgnoreInterrupt

static void sleepIgnoreInterrupt(int retry)

getDataNodeInfo

public static String getDataNodeInfo(Collection<org.apache.hadoop.hdfs.protocol.DatanodeInfo> datanodeInfos)