Skip to content

Commit

Permalink
ARROW-1864: [Java] Upgrade Netty to 4.1.17
Browse files Browse the repository at this point in the history 
Upgrade Netty to 4.1.17 since the Netty community will deprecate 4.0.x soon. This PR includes the following changes:
- Bump Netty version.
- Implement new ByteBuf APIs added in Netty 4.1.x: a bunch of get/setXXXLE methods. They are the opposite of get/setXXX method regarding byte order. E.g., as ArrowBuf is little endian, `setInt` will put an `int` to the buffer in little endian byte order, while `setIntLE` will put `int` in big byte endian order. The method naming seems confusing anyway, and I opened a Netty issue: netty/netty#7465. The user can call these new methods to get or set multi-byte integers in big endian byte order.
- Make ArrowByteBufAllocator overwrite AbstractByteBufAllocator.

Author: Shixiong Zhu <[email protected]>

Closes apache#1376 from zsxwing/ARROW-1864 and squashes the following commits:

96a93e1 [Shixiong Zhu] extend AbstractByteBufAllocator; add javadoc for new methods
bb97333 [Shixiong Zhu] Add comment for calculateNewCapacity
555f88a [Shixiong Zhu] Add methods back
5e09cca [Shixiong Zhu] Upgrade Netty to 4.1.x
  • Loading branch information
@zsxwing @siddharthteotia
zsxwing authored and siddharthteotia committed Dec 29, 2017
1 parent 9d84f84 commit 707af38
Show file tree
Hide file tree
Showing 5 changed files with 320 additions and 16 deletions.
199 changes: 191 additions & 8 deletions java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.channels.FileChannel;
import java.nio.channels.GatheringByteChannel;
import java.nio.channels.ScatteringByteChannel;
import java.nio.charset.Charset;
Expand Down Expand Up @@ -493,6 +494,16 @@ public ArrowBuf retain() {
return retain(1);
}

@Override
public ByteBuf touch() {
return this;
}

@Override
public ByteBuf touch(Object hint) {
return this;
}

@Override
public long getLong(int index) {
chk(index, 8);
Expand All @@ -505,6 +516,17 @@ public float getFloat(int index) {
return Float.intBitsToFloat(getInt(index));
}

/**
* Gets a 64-bit long integer at the specified absolute {@code index} in
* this buffer in Big Endian Byte Order.
*/
@Override
public long getLongLE(int index) {
chk(index, 8);
final long v = PlatformDependent.getLong(addr(index));
return Long.reverseBytes(v);
}

@Override
public double getDouble(int index) {
return Double.longBitsToDouble(getLong(index));
Expand All @@ -527,6 +549,17 @@ public int getInt(int index) {
return v;
}

/**
* Gets a 32-bit integer at the specified absolute {@code index} in
* this buffer in Big Endian Byte Order.
*/
@Override
public int getIntLE(int index) {
chk(index, 4);
final int v = PlatformDependent.getInt(addr(index));
return Integer.reverseBytes(v);
}

@Override
public int getUnsignedShort(int index) {
return getShort(index) & 0xFFFF;
Expand All @@ -535,31 +568,125 @@ public int getUnsignedShort(int index) {
@Override
public short getShort(int index) {
chk(index, 2);
short v = PlatformDependent.getShort(addr(index));
final short v = PlatformDependent.getShort(addr(index));
return v;
}

/**
* Gets a 16-bit short integer at the specified absolute {@code index} in
* this buffer in Big Endian Byte Order.
*/
@Override
public short getShortLE(int index) {
final short v = PlatformDependent.getShort(addr(index));
return Short.reverseBytes(v);
}

/**
* Gets an unsigned 24-bit medium integer at the specified absolute
* {@code index} in this buffer.
*/
@Override
public int getUnsignedMedium(int index) {
chk(index, 3);
final long addr = addr(index);
return (PlatformDependent.getByte(addr) & 0xff) << 16 |
(PlatformDependent.getShort(addr + 1) & 0xffff);
}

/**
* Gets an unsigned 24-bit medium integer at the specified absolute {@code index} in
* this buffer in Big Endian Byte Order.
*/
@Override
public int getUnsignedMediumLE(int index) {
chk(index, 3);
final long addr = addr(index);
return (PlatformDependent.getByte(addr) & 0xff) |
(Short.reverseBytes(PlatformDependent.getShort(addr + 1)) & 0xffff) << 8;
}

@Override
public ArrowBuf setShort(int index, int value) {
chk(index, 2);
PlatformDependent.putShort(addr(index), (short) value);
return this;
}

/**
* Sets the specified 16-bit short integer at the specified absolute {@code index}
* in this buffer with Big Endian byte order.
*/
@Override
public ByteBuf setShortLE(int index, int value) {
chk(index, 2);
PlatformDependent.putShort(addr(index), Short.reverseBytes((short) value));
return this;
}

/**
* Sets the specified 24-bit medium integer at the specified absolute
* {@code index} in this buffer.
*/
@Override
public ByteBuf setMedium(int index, int value) {
chk(index, 3);
final long addr = addr(index);
PlatformDependent.putByte(addr, (byte) (value >>> 16));
PlatformDependent.putShort(addr + 1, (short) value);
return this;
}


/**
* Sets the specified 24-bit medium integer at the specified absolute {@code index}
* in this buffer with Big Endian byte order.
*/
@Override
public ByteBuf setMediumLE(int index, int value) {
chk(index, 3);
final long addr = addr(index);
PlatformDependent.putByte(addr, (byte) value);
PlatformDependent.putShort(addr + 1, Short.reverseBytes((short) (value >>> 8)));
return this;
}

@Override
public ArrowBuf setInt(int index, int value) {
chk(index, 4);
PlatformDependent.putInt(addr(index), value);
return this;
}

/**
* Sets the specified 32-bit integer at the specified absolute {@code index}
* in this buffer with Big Endian byte order.
*/
@Override
public ByteBuf setIntLE(int index, int value) {
chk(index, 4);
PlatformDependent.putInt(addr(index), Integer.reverseBytes(value));
return this;
}

@Override
public ArrowBuf setLong(int index, long value) {
chk(index, 8);
PlatformDependent.putLong(addr(index), value);
return this;
}

/**
* Sets the specified 64-bit long integer at the specified absolute {@code index}
* in this buffer with Big Endian byte order.
*/
@Override
public ByteBuf setLongLE(int index, long value) {
chk(index, 8);
PlatformDependent.putLong(addr(index), Long.reverseBytes(value));
return this;
}

@Override
public ArrowBuf setChar(int index, int value) {
chk(index, 2);
Expand Down Expand Up @@ -668,16 +795,46 @@ protected short _getShort(int index) {
return getShort(index);
}

/** @see {@link #getShortLE(int)} */
@Override
protected short _getShortLE(int index) {
return getShortLE(index);
}

@Override
protected int _getInt(int index) {
return getInt(index);
}

/** @see {@link #getIntLE(int)} */
@Override
protected int _getIntLE(int index) {
return getIntLE(index);
}

/** @see {@link #getUnsignedMedium(int)} */
@Override
protected int _getUnsignedMedium(int index) {
return getUnsignedMedium(index);
}

/** @see {@link #getUnsignedMediumLE(int)} */
@Override
protected int _getUnsignedMediumLE(int index) {
return getUnsignedMediumLE(index);
}

@Override
protected long _getLong(int index) {
return getLong(index);
}

/** @see {@link #getLongLE(int)} */
@Override
protected long _getLongLE(int index) {
return getLongLE(index);
}

@Override
protected void _setByte(int index, int value) {
setByte(index, value);
Expand All @@ -688,21 +845,45 @@ protected void _setShort(int index, int value) {
setShort(index, value);
}

/** @see {@link #setShortLE(int, int)} */
@Override
protected void _setShortLE(int index, int value) {
setShortLE(index, value);
}

@Override
protected void _setMedium(int index, int value) {
setMedium(index, value);
}

/** @see {@link #setMediumLE(int, int)} */
@Override
protected void _setMediumLE(int index, int value) {
setMediumLE(index, value);
}

@Override
protected void _setInt(int index, int value) {
setInt(index, value);
}

/** @see {@link #setIntLE(int, int)} */
@Override
protected void _setIntLE(int index, int value) {
setIntLE(index, value);
}

@Override
protected void _setLong(int index, long value) {
setLong(index, value);
}

/** @see {@link #setLongLE(int, long)} */
@Override
public void _setLongLE(int index, long value) {
setLongLE(index, value);
}

@Override
public ArrowBuf getBytes(int index, ByteBuf dst, int dstIndex, int length) {
udle.getBytes(index + offset, dst, dstIndex, length);
Expand All @@ -716,16 +897,13 @@ public ArrowBuf getBytes(int index, OutputStream out, int length) throws IOExcep
}

@Override
protected int _getUnsignedMedium(int index) {
final long addr = addr(index);
return (PlatformDependent.getByte(addr) & 0xff) << 16 |
(PlatformDependent.getByte(addr + 1) & 0xff) << 8 |
PlatformDependent.getByte(addr + 2) & 0xff;
public int getBytes(int index, GatheringByteChannel out, int length) throws IOException {
return udle.getBytes(index + offset, out, length);
}

@Override
public int getBytes(int index, GatheringByteChannel out, int length) throws IOException {
return udle.getBytes(index + offset, out, length);
public int getBytes(int index, FileChannel out, long position, int length) throws IOException {
return udle.getBytes(index + offset, out, position, length);
}

@Override
Expand Down Expand Up @@ -776,6 +954,11 @@ public int setBytes(int index, ScatteringByteChannel in, int length) throws IOEx
return udle.setBytes(index + offset, in, length);
}

@Override
public int setBytes(int index, FileChannel in, long position, int length) throws IOException {
return udle.setBytes(index + offset, in, position, length);
}

@Override
public byte getByte(int index) {
chk(index, 1);
Expand Down
Loading

0 comments on commit 707af38

Please sign in to comment.