前言
对于Java nio ByteBuffer，我们常常会拿来做缓冲数据的处理。如果我们就为了图方便，每次数据读写操作专门allocate一个比较大capacity的ByteBuffer，这样会造成不必要的JVM heap的浪费。但是如果我们转而变为多个小ByteBuffer的动态申请，又会加大ByteBuffer的管理协调 操作。那么有什么办法能结合上述二者的特点，做到既不浪费JVM heap空间的使用，又不用在业务上执行复杂的ByteBuffer逻辑。本文笔者介绍一个Ozone内部实现的增量ByteBuffer的实现。增量ByeBuffer在外部使用上和ByteBuffe原生r操作方法语义完全兼容，内部增量allocate capacity操作对调用方而言完全透明。
Ozone内部的增量ByteBuffer实现
这里简单介绍下Ozone的背景，Ozone作为对象存储系统，在存储对象文件的时候，会涉及到大量小数据对象的写入，以Chunk文件的形式进行物理存储。在读写chunk数据的过程中，Ozone使用了ByteBuffer做中间数据的存储。在初始实现中，Ozone内部初始的ByteBuffer allocate的capacity是比较大的，也不管用户写入的数据有多大。为此，后来社区实现了能够指定目标capacity，同时指定动态增量分配大小的ByteBuffer，名为IncrementalChunkBuffer。
以下为IncrementalChunkBuffer的实现代码，摘自Ozone项目：
/**
* Use a list of {@link ByteBuffer} to implement a single {@link ChunkBuffer}
* so that the buffer can be allocated incrementally.
*/
final class IncrementalChunkBuffer implements ChunkBuffer {
/**
* 全局Buffer的limit边界值
*/
private final int limit;
/** ByteBuffer的增量容量. */
private final int increment;
/** BytesBuffer下标数组边界值. */
private final int limitIndex;
/** 增量分配ByteBuffer数组 */
private final List<ByteBuffer> buffers;
/** Is this a duplicated buffer? (for debug only) */
private final boolean isDuplicated;
// 增量ByteBuffer总capacity(传入的limit值)，每次动态增量increment值大小
IncrementalChunkBuffer(int limit, int increment, boolean isDuplicated) {
Preconditions.checkArgument(limit >= 0);
Preconditions.checkArgument(increment > 0);
// 初始化边界值，增量ByteBuffer值
this.limit = limit;
this.increment = increment;
// 计算ByteBuffer数组下标最大值
this.limitIndex = limit/increment;
// 初始空ByteBuffer数组
this.buffers = new ArrayList<>(limitIndex + (limit%increment == 0? 0: 1));
this.isDuplicated = isDuplicated;
}
/** @return the capacity for the buffer at the given index. */
private int getBufferCapacityAtIndex(int i) {
Preconditions.checkArgument(i >= 0);
Preconditions.checkArgument(i <= limitIndex);
return i < limitIndex? increment: limit%increment;
}
private void assertInt(int expected, int computed, String name, int i) {
ChunkBuffer.assertInt(expected, computed,
() -> this + ": Unexpected " + name + " at index " + i);
}
/** @return the i-th buffer if it exists; otherwise, return null. */
private ByteBuffer getAtIndex(int i) {
Preconditions.checkArgument(i >= 0);
Preconditions.checkArgument(i <= limitIndex);
final ByteBuffer ith = i < buffers.size() ? buffers.get(i) : null;
if (ith != null) {
// assert limit/capacity
if (!isDuplicated) {
assertInt(getBufferCapacityAtIndex(i), ith.capacity(), "capacity", i);
} else {
if (i < limitIndex) {
assertInt(increment, ith.capacity(), "capacity", i);
} else if (i == limitIndex) {
assertInt(getBufferCapacityAtIndex(i), ith.limit(), "capacity", i);
} else {
assertInt(0, ith.limit(), "capacity", i);
}
}
}
return ith;
}
/** @return the i-th buffer. It may allocate buffers. */
private ByteBuffer getAndAllocateAtIndex(int index) {
Preconditions.checkArgument(index >= 0);
// never allocate over limit
if (limit % increment == 0) {
Preconditions.checkArgument(index < limitIndex);
} else {
Preconditions.checkArgument(index <= limitIndex);
}
int i = buffers.size();
if (index < i) {
return getAtIndex(index);
}
// allocate upto the given index
ByteBuffer b = null;
for (; i <= index; i++) {
b = ByteBuffer.allocate(getBufferCapacityAtIndex(i));
buffers.add(b);
}
return b;
}
/** @return the buffer containing the position. It may allocate buffers. */
private ByteBuffer getAndAllocateAtPosition(int position) {
Preconditions.checkArgument(position >= 0);
Preconditions.checkArgument(position < limit);
// 计算需要获取ByteBuffer的下标
final int i = position / increment;
// 得到此下标对应的ByteBuffer，如果没有创建，则进行ByteBuffer的创建操作
final ByteBuffer ith = getAndAllocateAtIndex(i);
assertInt(position%increment, ith.position(), "position", i);
return ith;
}
/** @return the index of the first non-full buffer. */
private int firstNonFullIndex() {
for (int i = 0; i < buffers.size(); i++) {
if (getAtIndex(i).position() != increment) {
return i;
}
}
return buffers.size();
}
@Override
public int position() {
// The buffers list must be in the following orders:
// full buffers, buffer containing the position, empty buffers, null buffers
// 1）寻找第一个不满的ByteBuffer下标
final int i = firstNonFullIndex();
// 2）获取此ByteBuffer项
final ByteBuffer ith = getAtIndex(i);
// 3）计算当前全局position位置，前面i个满ByteBuffer的长度+当前不满的ByteBuffer的position位置
final int position = i * increment + Optional.ofNullable(ith)
.map(ByteBuffer::position).orElse(0);
// remaining buffers must be empty
assert assertRemainingList(ith, i);
return position;
}
private boolean assertRemainingList(ByteBuffer ith, int i) {
if (ith != null) {
// buffers must be empty
for (i++; i < buffers.size(); i++) {
ith = getAtIndex(i);
if (ith == null) {
break; // found the first non-null
}
assertInt(0, ith.position(), "position", i);
}
}
final int j = i;
ChunkBuffer.assertInt(buffers.size(), i,
() -> "i = " + j + " != buffers.size() = " + buffers.size());
return true;
}
@Override
public int remaining() {
// remaining操作和原ByteBuffer语义一致
return limit - position();
}
@Override
public int limit() {
return limit;
}
@Override
public ChunkBuffer rewind() {
buffers.forEach(ByteBuffer::rewind);
return this;
}
@Override
public ChunkBuffer clear() {
buffers.forEach(ByteBuffer::clear);
return this;
}
@Override
public ChunkBuffer put(ByteBuffer that) {
// 1）判断待put操作的ByteBuffer中的剩余数据空间是否超过增量ByteBuffer当前剩余空间
// 如果超过，则抛出BufferOverflowException异常
if (that.remaining() > this.remaining()) {
final BufferOverflowException boe = new BufferOverflowException();
boe.initCause(new IllegalArgumentException(
"Failed to put since that.remaining() = " + that.remaining()
+ " > this.remaining() = " + this.remaining()));
throw boe;
}
// 2）得到待写入ByteBuffer的边界值
final int thatLimit = that.limit();
for(int p = position(); that.position() < thatLimit;) {
// 3）得到当前增量Buffer内正在提供写操作的ByteBuffer
final ByteBuffer b = getAndAllocateAtPosition(p);
// 4）比较剩余数据空间的大小，选取上述2个Buffer的较小值
final int min = Math.min(b.remaining(), thatLimit - that.position());
that.limit(that.position() + min);
// 进行ByteBuffer的写入操作
b.put(that);
// 更新增量ByteBuffer当前的position位置
p += min;
}
return this;
}
@Override
public ChunkBuffer duplicate(int newPosition, int newLimit) {
Preconditions.checkArgument(newPosition >= 0);
Preconditions.checkArgument(newPosition <= newLimit);
Preconditions.checkArgument(newLimit <= limit);
final IncrementalChunkBuffer duplicated = new IncrementalChunkBuffer(
newLimit, increment, true);
final int pi = newPosition / increment;
final int pr = newPosition % increment;
final int li = newLimit / increment;
final int lr = newLimit % increment;
final int newSize = lr == 0? li: li + 1;
for (int i = 0; i < newSize; i++) {
final int pos = i < pi ? increment : i == pi ? pr : 0;
final int lim = i < li ? increment : i == li ? lr : 0;
duplicated.buffers.add(duplicate(i, pos, lim));
}
return duplicated;
}
private ByteBuffer duplicate(int i, int pos, int lim) {
final ByteBuffer ith = getAtIndex(i);
Objects.requireNonNull(ith, () -> "buffers[" + i + "] == null");
final ByteBuffer b = ith.duplicate();
b.position(pos).limit(lim);
return b;
}
/** Support only when bufferSize == increment. */
@Override
public Iterable<ByteBuffer> iterate(int bufferSize) {
if (bufferSize != increment) {
throw new UnsupportedOperationException(
"Buffer size and increment mismatched: bufferSize = " + bufferSize
+ " but increment = " + increment);
}
return asByteBufferList();
}
@Override
public List<ByteBuffer> asByteBufferList() {
return Collections.unmodifiableList(buffers);
}
@Override
public long writeTo(GatheringByteChannel channel) throws IOException {
return channel.write(buffers.toArray(new ByteBuffer[0]));
}
@Override
public ByteString toByteStringImpl(Function<ByteBuffer, ByteString> f) {
return buffers.stream().map(f).reduce(ByteString.EMPTY, ByteString::concat);
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
} else if (!(obj instanceof IncrementalChunkBuffer)) {
return false;
}
final IncrementalChunkBuffer that = (IncrementalChunkBuffer)obj;
return this.limit == that.limit && this.buffers.equals(that.buffers);
}
@Override
public int hashCode() {
return buffers.hashCode();
}
@Override
public String toString() {
return getClass().getSimpleName()
+ ":limit=" + limit + ",increment=" + increment;
}
}
引用
[1].https://github.com/apache/hadoop-ozone/blob/master/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/IncrementalChunkBuffer.java
来源：CSDN
作者：Android路上的人
链接：https://blog.csdn.net/Androidlushangderen/article/details/104808634