Netty: TCP Client Server File transfer: Exception TooLongFrameException: - tcp

Question

I am new to netty and I am trying to design a solution as below for transfer of file from Server to Client over TCP:
1. Zero copy based file transfer in case of non-ssl based transfer (Using default region of the file)
2. ChunkedFile transfer in case of SSL based transfer.
The Client - Server file transfer works in this way:
1. The client sends the location of the file to be transfered
2. Based on the location (sent by the client) the server transfers the file to the client
The file content could be anything (String /image /pdf etc) and any size.
Now, I get this TooLongFrameException: at the Server side, though the server is just decoding the path received from the client, for running the code mentioned below (Server/Client).
io.netty.handler.codec.TooLongFrameException: Adjusted frame length exceeds 65536: 215542494061 - discarded
at io.netty.handler.codec.LengthFieldBasedFrameDecoder.fail(LengthFieldBasedFrameDecoder.java:522)
at io.netty.handler.codec.LengthFieldBasedFrameDecoder.failIfNecessary(LengthFieldBasedFrameDecoder.java:500)
Now, My question is:
Am I wrong with the order of Encoders and Decoders and its configuration? If so, what is the correct way to configure it to receive a file from the server?
I went through few related StackOverflow posts SO Q1,SO Q2 , SO Q3, SO Q4. I got to know about the LengthFieldBasedDecoder, but I didn't get to know how to configure its corresponding LengthFieldPrepender at the Server (Encoding side). Is it even required at all?
Please point me into the right direction.
FileClient:
public final class FileClient {
static final boolean SSL = System.getProperty("ssl") != null;
static final int PORT = Integer.parseInt(System.getProperty("port", SSL ? "8992" : "8023"));
static final String HOST = System.getProperty("host", "127.0.0.1");
public static void main(String[] args) throws Exception {
// Configure SSL.
final SslContext sslCtx;
if (SSL) {
SelfSignedCertificate ssc = new SelfSignedCertificate();
sslCtx = SslContextBuilder.forServer(ssc.certificate(), ssc.privateKey()).build();
} else {
sslCtx = null;
}
// Configure the client
EventLoopGroup group = new NioEventLoopGroup();
try {
Bootstrap b = new Bootstrap();
b.group(group)
.channel(NioSocketChannel.class)
.option(ChannelOption.SO_KEEPALIVE, true)
.handler(new ChannelInitializer<SocketChannel>() {
#Override
public void initChannel(SocketChannel ch) throws Exception {
ChannelPipeline pipeline = ch.pipeline();
if (sslCtx != null) {
pipeline.addLast(sslCtx.newHandler(ch.alloc(), HOST, PORT));
}
pipeline.addLast("frameDecoder", new LengthFieldBasedFrameDecoder(64*1024, 0, 8));
pipeline.addLast("frameEncoder", new LengthFieldPrepender(4));
pipeline.addLast(new ObjectDecoder(ClassResolvers.cacheDisabled(null)));
pipeline.addLast(new ObjectEncoder());
pipeline.addLast( new FileClientHandler()); }
});
// Start the server.
ChannelFuture f = b.connect(HOST,PORT).sync();
// Wait until the server socket is closed.
f.channel().closeFuture().sync();
} finally {
// Shut down all event loops to terminate all threads.
group.shutdownGracefully();
}
}
}
FileClientHandler:
public class FileClientHandler extends ChannelInboundHandlerAdapter{
#Override
public void channelActive(ChannelHandlerContext ctx) {
String filePath = "/Users/Home/Documents/Data.pdf";
ctx.writeAndFlush(Unpooled.wrappedBuffer(filePath.getBytes()));
}
#Override
public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
System.out.println("File Client Handler Read method...");
}
#Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
cause.printStackTrace();
}
}
FileServer:
/**
* Server that accept the path of a file and echo back its content.
*/
public final class FileServer {
static final boolean SSL = System.getProperty("ssl") != null;
static final int PORT = Integer.parseInt(System.getProperty("port", SSL ? "8992" : "8023"));
public static void main(String[] args) throws Exception {
// Configure SSL.
final SslContext sslCtx;
if (SSL) {
SelfSignedCertificate ssc = new SelfSignedCertificate();
sslCtx = SslContextBuilder.forServer(ssc.certificate(), ssc.privateKey()).build();
} else {
sslCtx = null;
}
// Configure the server.
EventLoopGroup bossGroup = new NioEventLoopGroup(1);
EventLoopGroup workerGroup = new NioEventLoopGroup();
try {
ServerBootstrap b = new ServerBootstrap();
b.group(bossGroup, workerGroup).channel(NioServerSocketChannel.class)
.option(ChannelOption.SO_KEEPALIVE, true).handler(new LoggingHandler(LogLevel.INFO))
.childHandler(new ChannelInitializer<SocketChannel>() {
#Override
public void initChannel(SocketChannel ch) throws Exception {
ChannelPipeline pipeline = ch.pipeline();
if (sslCtx != null) {
pipeline.addLast(sslCtx.newHandler(ch.alloc()));
}
pipeline.addLast("frameDecoder",new LengthFieldBasedFrameDecoder(64*1024, 0, 8));
pipeline.addLast("frameEncoder", new LengthFieldPrepender(4));
pipeline.addLast(new ObjectDecoder(ClassResolvers.cacheDisabled(null)));
pipeline.addLast(new ObjectEncoder());
pipeline.addLast(new ChunkedWriteHandler());
pipeline.addLast(new FileServerHandler());
}
});
// Start the server.
ChannelFuture f = b.bind(PORT).sync();
// Wait until the server socket is closed.
f.channel().closeFuture().sync();
} finally {
bossGroup.shutdownGracefully();
workerGroup.shutdownGracefully();
}
}
}
FileServerHandler:
public class FileServerHandler extends ChannelInboundHandlerAdapter {
#Override
public void channelRead(ChannelHandlerContext ctx, Object obj) throws Exception {
RandomAccessFile raf = null;
long length = -1;
try {
ByteBuf buff = (ByteBuf)obj;
byte[] bytes = new byte[buff.readableBytes()];
buff.readBytes(bytes);
String msg = new String(bytes);
raf = new RandomAccessFile(msg, "r");
length = raf.length();
} catch (Exception e) {
ctx.writeAndFlush("ERR: " + e.getClass().getSimpleName() + ": " + e.getMessage() + '\n');
return;
} finally {
if (length < 0 && raf != null) {
raf.close();
}
}
if (ctx.pipeline().get(SslHandler.class) == null) {
// SSL not enabled - can use zero-copy file transfer.
ctx.writeAndFlush(new DefaultFileRegion(raf.getChannel(), 0, length));
} else {
// SSL enabled - cannot use zero-copy file transfer.
ctx.writeAndFlush(new ChunkedFile(raf));
}
}
#Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
cause.printStackTrace();
System.out.println("Exception server.....");
}
}
I referred Netty In Action and code samples from here

Answer 1

There are multiple things wrong with your server/client. First thing the SSL, for the client you don't need to initialize a SslContext for a server instead you would do something like this:
sslCtx = SslContextBuilder.forClient().trustManager(InsecureTrustManagerFactory.INSTANCE).build();
On the server side of things you use a SelfSignedCertificate which in itself isn't wrong but would like to remind you that it should only be used for debugging purposes and not in production. In addition you use the ChannelOption.SO_KEEPALIVE which isn't recommended since the keepalive interval is OS-dependent. Furthermore you added Object En-/Decoder to your pipeline which in your case don't do anything useful so you can remove them.
Also you configured your LengthFieldBasedFrameDecoder wrong due to an incomplete and wrong parameter list. In the netty docs you need the version of the constructor which defines the lengthFieldLength and initialBytesToStrip. Besides the not stripping the length field you also defined the wrong lengthFieldLength which should be the same as your LengthFieldPrepender's lengthFieldLength which is 4 bytes. In conlusion you could use the constructor like this:
new LengthFieldBasedFrameDecoder(64 * 1024, 0, 4, 0, 4)
In both your handler you don't specify a Charset when en-/decoding your String which could lead to problems because if no ´Charset´ is defined the systems default will be used which could vary. You could do something like this:
//to encode the String
string.getBytes(StandardCharsets.UTF_8);
//to decode the String
new String(bytes, StandardCharsets.UTF_8);
Additionally you tried to use the DefaultFileRegion if no SslHandler was added to the pipeline which would have been fine if you didn't added the LengthFieldHandler since they would need a memory copy of the byte[] to send to added the length field. Moreover I would recommend using the ChunkedNioFile instead of the ChunkedFile because it's nonblocking which is always a good thing. You would do this like that:
new ChunkedNioFile(randomAccessFile.getChannel())
One final thing on how to decode a ChunkedFile as it's split in chunks you can simply assamble them tougether with a simple OutputStream. Here's an old file handler of mine:
public class FileTransferHandler extends SimpleChannelInboundHandler<ByteBuf> {
private final Path path;
private final int size;
private final int hash;
private OutputStream outputStream;
private int writtenBytes = 0;
private byte[] buffer = new byte[0];
protected FileTransferHandler(Path path, int size, int hash) {
this.path = path;
this.size = size;
this.hash = hash;
}
#Override
protected void channelRead0(ChannelHandlerContext ctx, ByteBuf byteBuf) throws Exception {
if(this.outputStream == null) {
Files.createDirectories(this.path.getParent());
if(Files.exists(this.path))
Files.delete(this.path);
this.outputStream = Files.newOutputStream(this.path, StandardOpenOption.CREATE, StandardOpenOption.APPEND);
}
int size = byteBuf.readableBytes();
if(size > this.buffer.length)
this.buffer = new byte[size];
byteBuf.readBytes(this.buffer, 0, size);
this.outputStream.write(this.buffer, 0, size);
this.writtenBytes += size;
if(this.writtenBytes == this.size && MurMur3.hash(this.path) != this.hash) {
System.err.println("Received file has wrong hash");
return;
}
}
#Override
public void channelInactive(ChannelHandlerContext ctx) throws Exception {
if(this.outputStream != null)
this.outputStream.close();
}
}