Skip to content

Commit

Permalink
ARROW-9: Replace straggler references to Drill
Browse files Browse the repository at this point in the history
- Renaming drill to arrow for TestBaseAllocator
- Fix ArrowBuffer as ArrowBuf
- Replace Drill with Arrow for ValueHolder

This closes #2.
  • Loading branch information
lw-lin authored and jacques-n committed Feb 20, 2016
1 parent e9cc8ce commit e6905ef
Show file tree
Hide file tree
Showing 17 changed files with 198 additions and 198 deletions.
36 changes: 18 additions & 18 deletions java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ public final class ArrowBuf extends AbstractByteBuf implements AutoCloseable {
private final boolean isEmpty;
private volatile int length;
private final HistoricalLog historicalLog = BaseAllocator.DEBUG ?
new HistoricalLog(BaseAllocator.DEBUG_LOG_LENGTH, "DrillBuf[%d]", id) : null;
new HistoricalLog(BaseAllocator.DEBUG_LOG_LENGTH, "ArrowBuf[%d]", id) : null;

public ArrowBuf(
final AtomicInteger refCnt,
Expand Down Expand Up @@ -155,18 +155,18 @@ private void ensure(int width) {
}

/**
* Create a new DrillBuf that is associated with an alternative allocator for the purposes of memory ownership and
* accounting. This has no impact on the reference counting for the current DrillBuf except in the situation where the
* Create a new ArrowBuf that is associated with an alternative allocator for the purposes of memory ownership and
* accounting. This has no impact on the reference counting for the current ArrowBuf except in the situation where the
* passed in Allocator is the same as the current buffer.
*
* This operation has no impact on the reference count of this DrillBuf. The newly created DrillBuf with either have a
* This operation has no impact on the reference count of this ArrowBuf. The newly created ArrowBuf with either have a
* reference count of 1 (in the case that this is the first time this memory is being associated with the new
* allocator) or the current value of the reference count + 1 for the other AllocationManager/BufferLedger combination
* in the case that the provided allocator already had an association to this underlying memory.
*
* @param target
* The target allocator to create an association with.
* @return A new DrillBuf which shares the same underlying memory as this DrillBuf.
* @return A new ArrowBuf which shares the same underlying memory as this ArrowBuf.
*/
public ArrowBuf retain(BufferAllocator target) {

Expand All @@ -178,17 +178,17 @@ public ArrowBuf retain(BufferAllocator target) {
historicalLog.recordEvent("retain(%s)", target.getName());
}
final BufferLedger otherLedger = this.ledger.getLedgerForAllocator(target);
return otherLedger.newDrillBuf(offset, length, null);
return otherLedger.newArrowBuf(offset, length, null);
}

/**
* Transfer the memory accounting ownership of this DrillBuf to another allocator. This will generate a new DrillBuf
* that carries an association with the underlying memory of this DrillBuf. If this DrillBuf is connected to the
* Transfer the memory accounting ownership of this ArrowBuf to another allocator. This will generate a new ArrowBuf
* that carries an association with the underlying memory of this ArrowBuf. If this ArrowBuf is connected to the
* owning BufferLedger of this memory, that memory ownership/accounting will be transferred to the taret allocator. If
* this DrillBuf does not currently own the memory underlying it (and is only associated with it), this does not
* transfer any ownership to the newly created DrillBuf.
* this ArrowBuf does not currently own the memory underlying it (and is only associated with it), this does not
* transfer any ownership to the newly created ArrowBuf.
*
* This operation has no impact on the reference count of this DrillBuf. The newly created DrillBuf with either have a
* This operation has no impact on the reference count of this ArrowBuf. The newly created ArrowBuf with either have a
* reference count of 1 (in the case that this is the first time this memory is being associated with the new
* allocator) or the current value of the reference count for the other AllocationManager/BufferLedger combination in
* the case that the provided allocator already had an association to this underlying memory.
Expand All @@ -203,7 +203,7 @@ public ArrowBuf retain(BufferAllocator target) {
* @param target
* The allocator to transfer ownership to.
* @return A new transfer result with the impact of the transfer (whether it was overlimit) as well as the newly
* created DrillBuf.
* created ArrowBuf.
*/
public TransferResult transferOwnership(BufferAllocator target) {

Expand All @@ -212,7 +212,7 @@ public TransferResult transferOwnership(BufferAllocator target) {
}

final BufferLedger otherLedger = this.ledger.getLedgerForAllocator(target);
final ArrowBuf newBuf = otherLedger.newDrillBuf(offset, length, null);
final ArrowBuf newBuf = otherLedger.newArrowBuf(offset, length, null);
final boolean allocationFit = this.ledger.transferBalance(otherLedger);
return new TransferResult(allocationFit, newBuf);
}
Expand Down Expand Up @@ -267,7 +267,7 @@ public boolean release(int decrement) {

if (refCnt < 0) {
throw new IllegalStateException(
String.format("DrillBuf[%d] refCnt has gone negative. Buffer Info: %s", id, toVerboseString()));
String.format("ArrowBuf[%d] refCnt has gone negative. Buffer Info: %s", id, toVerboseString()));
}

return refCnt == 0;
Expand Down Expand Up @@ -370,7 +370,7 @@ public ArrowBuf slice(int index, int length) {
* Re the behavior of reference counting, see http://netty.io/wiki/reference-counted-objects.html#wiki-h3-5, which
* explains that derived buffers share their reference count with their parent
*/
final ArrowBuf newBuf = ledger.newDrillBuf(offset + index, length);
final ArrowBuf newBuf = ledger.newArrowBuf(offset + index, length);
newBuf.writerIndex(length);
return newBuf;
}
Expand Down Expand Up @@ -437,7 +437,7 @@ public long memoryAddress() {

@Override
public String toString() {
return String.format("DrillBuf[%d], udle: [%d %d..%d]", id, udle.id, offset, offset + capacity());
return String.format("ArrowBuf[%d], udle: [%d %d..%d]", id, udle.id, offset, offset + capacity());
}

@Override
Expand Down Expand Up @@ -782,7 +782,7 @@ public void close() {
}

/**
* Returns the possible memory consumed by this DrillBuf in the worse case scenario. (not shared, connected to larger
* Returns the possible memory consumed by this ArrowBuf in the worse case scenario. (not shared, connected to larger
* underlying buffer of allocated memory)
*
* @return Size in bytes.
Expand Down Expand Up @@ -833,7 +833,7 @@ public String toHexString(final int start, final int length) {
}

/**
* Get the integer id assigned to this DrillBuf for debugging purposes.
* Get the integer id assigned to this ArrowBuf for debugging purposes.
*
* @return integer id
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
import org.apache.arrow.memory.BufferAllocator;

/**
* Allows us to decorate DrillBuf to make it expandable so that we can use them in the context of the Netty framework
* Allows us to decorate ArrowBuf to make it expandable so that we can use them in the context of the Netty framework
* (thus supporting RPC level memory accounting).
*/
public class ExpandableByteBuf extends MutableWrappedByteBuf {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
import com.codahale.metrics.MetricRegistry;

/**
* The base allocator that we use for all of Drill's memory management. Returns UnsafeDirectLittleEndian buffers.
* The base allocator that we use for all of Arrow's memory management. Returns UnsafeDirectLittleEndian buffers.
*/
public class PooledByteBufAllocatorL {
private static final org.slf4j.Logger memoryLogger = org.slf4j.LoggerFactory.getLogger("drill.allocator");
Expand Down Expand Up @@ -184,7 +184,7 @@ private UnsafeDirectLittleEndian newDirectBufferL(int initialCapacity, int maxCa

private UnsupportedOperationException fail() {
return new UnsupportedOperationException(
"Drill requries that the JVM used supports access sun.misc.Unsafe. This platform didn't provide that functionality.");
"Arrow requries that the JVM used supports access sun.misc.Unsafe. This platform didn't provide that functionality.");
}

public UnsafeDirectLittleEndian directBuffer(int initialCapacity, int maxCapacity) {
Expand All @@ -197,7 +197,7 @@ public UnsafeDirectLittleEndian directBuffer(int initialCapacity, int maxCapacit

@Override
public ByteBuf heapBuffer(int initialCapacity, int maxCapacity) {
throw new UnsupportedOperationException("Drill doesn't support using heap buffers.");
throw new UnsupportedOperationException("Arrow doesn't support using heap buffers.");
}


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
import java.util.concurrent.atomic.AtomicLong;

/**
* The underlying class we use for little-endian access to memory. Is used underneath DrillBufs to abstract away the
* The underlying class we use for little-endian access to memory. Is used underneath ArrowBufs to abstract away the
* Netty classes and underlying Netty memory management.
*/
public final class UnsafeDirectLittleEndian extends WrappedByteBuf {
Expand Down Expand Up @@ -55,7 +55,7 @@ public final class UnsafeDirectLittleEndian extends WrappedByteBuf {
private UnsafeDirectLittleEndian(AbstractByteBuf buf, boolean fake, AtomicLong bufferCount, AtomicLong bufferSize) {
super(buf);
if (!NATIVE_ORDER || buf.order() != ByteOrder.BIG_ENDIAN) {
throw new IllegalStateException("Drill only runs on LittleEndian systems.");
throw new IllegalStateException("Arrow only runs on LittleEndian systems.");
}

this.bufferCount = bufferCount;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
* This class is also responsible for managing when memory is allocated and returned to the Netty-based
* PooledByteBufAllocatorL.
*
* The only reason that this isn't package private is we're forced to put DrillBuf in Netty's package which need access
* The only reason that this isn't package private is we're forced to put ArrowBuf in Netty's package which need access
* to these objects or methods.
*
* Threading: AllocationManager manages thread-safety internally. Operations within the context of a single BufferLedger
Expand Down Expand Up @@ -185,8 +185,8 @@ public void release() {

/**
* The reference manager that binds an allocator manager to a particular BaseAllocator. Also responsible for creating
* a set of DrillBufs that share a common fate and set of reference counts.
* As with AllocationManager, the only reason this is public is due to DrillBuf being in io.netty.buffer package.
* a set of ArrowBufs that share a common fate and set of reference counts.
* As with AllocationManager, the only reason this is public is due to ArrowBuf being in io.netty.buffer package.
*/
public class BufferLedger {

Expand Down Expand Up @@ -322,7 +322,7 @@ public int decrement(int decrement) {
/**
* Returns the ledger associated with a particular BufferAllocator. If the BufferAllocator doesn't currently have a
* ledger associated with this AllocationManager, a new one is created. This is placed on BufferLedger rather than
* AllocationManager directly because DrillBufs don't have access to AllocationManager and they are the ones
* AllocationManager directly because ArrowBufs don't have access to AllocationManager and they are the ones
* responsible for exposing the ability to associate multiple allocators with a particular piece of underlying
* memory. Note that this will increment the reference count of this ledger by one to ensure the ledger isn't
* destroyed before use.
Expand All @@ -335,32 +335,32 @@ public BufferLedger getLedgerForAllocator(BufferAllocator allocator) {
}

/**
* Create a new DrillBuf associated with this AllocationManager and memory. Does not impact reference count.
* Create a new ArrowBuf associated with this AllocationManager and memory. Does not impact reference count.
* Typically used for slicing.
* @param offset
* The offset in bytes to start this new DrillBuf.
* The offset in bytes to start this new ArrowBuf.
* @param length
* The length in bytes that this DrillBuf will provide access to.
* @return A new DrillBuf that shares references with all DrillBufs associated with this BufferLedger
* The length in bytes that this ArrowBuf will provide access to.
* @return A new ArrowBuf that shares references with all ArrowBufs associated with this BufferLedger
*/
public ArrowBuf newDrillBuf(int offset, int length) {
public ArrowBuf newArrowBuf(int offset, int length) {
allocator.assertOpen();
return newDrillBuf(offset, length, null);
return newArrowBuf(offset, length, null);
}

/**
* Create a new DrillBuf associated with this AllocationManager and memory.
* Create a new ArrowBuf associated with this AllocationManager and memory.
* @param offset
* The offset in bytes to start this new DrillBuf.
* The offset in bytes to start this new ArrowBuf.
* @param length
* The length in bytes that this DrillBuf will provide access to.
* The length in bytes that this ArrowBuf will provide access to.
* @param manager
* An optional BufferManager argument that can be used to manage expansion of this DrillBuf
* An optional BufferManager argument that can be used to manage expansion of this ArrowBuf
* @param retain
* Whether or not the newly created buffer should get an additional reference count added to it.
* @return A new DrillBuf that shares references with all DrillBufs associated with this BufferLedger
* @return A new ArrowBuf that shares references with all ArrowBufs associated with this BufferLedger
*/
public ArrowBuf newDrillBuf(int offset, int length, BufferManager manager) {
public ArrowBuf newArrowBuf(int offset, int length, BufferManager manager) {
allocator.assertOpen();

final ArrowBuf buf = new ArrowBuf(
Expand All @@ -375,7 +375,7 @@ public ArrowBuf newDrillBuf(int offset, int length, BufferManager manager) {

if (BaseAllocator.DEBUG) {
historicalLog.recordEvent(
"DrillBuf(BufferLedger, BufferAllocator[%s], UnsafeDirectLittleEndian[identityHashCode == "
"ArrowBuf(BufferLedger, BufferAllocator[%s], UnsafeDirectLittleEndian[identityHashCode == "
+ "%d](%s)) => ledger hc == %d",
allocator.name, System.identityHashCode(buf), buf.toString(),
System.identityHashCode(this));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,19 +23,19 @@
import io.netty.buffer.ExpandableByteBuf;

/**
* An implementation of ByteBufAllocator that wraps a Drill BufferAllocator. This allows the RPC layer to be accounted
* and managed using Drill's BufferAllocator infrastructure. The only thin different from a typical BufferAllocator is
* An implementation of ByteBufAllocator that wraps a Arrow BufferAllocator. This allows the RPC layer to be accounted
* and managed using Arrow's BufferAllocator infrastructure. The only thin different from a typical BufferAllocator is
* the signature and the fact that this Allocator returns ExpandableByteBufs which enable otherwise non-expandable
* DrillBufs to be expandable.
* ArrowBufs to be expandable.
*/
public class DrillByteBufAllocator implements ByteBufAllocator {
public class ArrowByteBufAllocator implements ByteBufAllocator {

private static final int DEFAULT_BUFFER_SIZE = 4096;
private static final int DEFAULT_MAX_COMPOSITE_COMPONENTS = 16;

private final BufferAllocator allocator;

public DrillByteBufAllocator(BufferAllocator allocator) {
public ArrowByteBufAllocator(BufferAllocator allocator) {
this.allocator = allocator;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ protected BaseAllocator(
this.parentAllocator = parentAllocator;
this.name = name;

this.thisAsByteBufAllocator = new DrillByteBufAllocator(this);
this.thisAsByteBufAllocator = new ArrowByteBufAllocator(this);

if (DEBUG) {
childAllocators = new IdentityHashMap<>();
Expand Down Expand Up @@ -236,7 +236,7 @@ private ArrowBuf bufferWithoutReservation(final int size, BufferManager bufferMa

final AllocationManager manager = new AllocationManager(this, size);
final BufferLedger ledger = manager.associate(this); // +1 ref cnt (required)
final ArrowBuf buffer = ledger.newDrillBuf(0, size, bufferManager);
final ArrowBuf buffer = ledger.newArrowBuf(0, size, bufferManager);

// make sure that our allocation is equal to what we expected.
Preconditions.checkArgument(buffer.capacity() == size,
Expand Down Expand Up @@ -314,9 +314,9 @@ public ArrowBuf allocateBuffer() {
Preconditions.checkState(!closed, "Attempt to allocate after closed");
Preconditions.checkState(!used, "Attempt to allocate more than once");

final ArrowBuf drillBuf = allocate(nBytes);
final ArrowBuf arrowBuf = allocate(nBytes);
used = true;
return drillBuf;
return arrowBuf;
}

public int getSize() {
Expand Down Expand Up @@ -397,13 +397,13 @@ private ArrowBuf allocate(int nBytes) {
* as well, so we need to return the same number back to avoid double-counting them.
*/
try {
final ArrowBuf drillBuf = BaseAllocator.this.bufferWithoutReservation(nBytes, null);
final ArrowBuf arrowBuf = BaseAllocator.this.bufferWithoutReservation(nBytes, null);

if (DEBUG) {
historicalLog.recordEvent("allocate() => %s", String.format("DrillBuf[%d]", drillBuf.getId()));
historicalLog.recordEvent("allocate() => %s", String.format("ArrowBuf[%d]", arrowBuf.getId()));
}
success = true;
return drillBuf;
return arrowBuf;
} finally {
if (!success) {
releaseBytes(nBytes);
Expand Down Expand Up @@ -565,7 +565,7 @@ void verifyAllocator() {
* Verifies the accounting state of the allocator. Only works for DEBUG.
*
* <p>
* This overload is used for recursive calls, allowing for checking that DrillBufs are unique across all allocators
* This overload is used for recursive calls, allowing for checking that ArrowBufs are unique across all allocators
* that are checked.
* </p>
*
Expand Down Expand Up @@ -594,7 +594,7 @@ private void verifyAllocator(final IdentityHashMap<UnsafeDirectLittleEndian, Bas
* Verify my relationships with my descendants.
*
* The sum of direct child allocators' owned memory must be <= my allocated memory; my allocated memory also
* includes DrillBuf's directly allocated by me.
* includes ArrowBuf's directly allocated by me.
*/
long childTotal = 0;
for (final BaseAllocator childAllocator : childSet) {
Expand Down Expand Up @@ -623,11 +623,11 @@ private void verifyAllocator(final IdentityHashMap<UnsafeDirectLittleEndian, Bas

final UnsafeDirectLittleEndian udle = ledger.getUnderlying();
/*
* Even when shared, DrillBufs are rewrapped, so we should never see the same instance twice.
* Even when shared, ArrowBufs are rewrapped, so we should never see the same instance twice.
*/
final BaseAllocator otherOwner = buffersSeen.get(udle);
if (otherOwner != null) {
throw new IllegalStateException("This allocator's drillBuf already owned by another allocator");
throw new IllegalStateException("This allocator's ArrowBuf already owned by another allocator");
}
buffersSeen.put(udle, this);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ public interface BufferAllocator extends AutoCloseable {
*
* @param size
* The size in bytes.
* @return a new DrillBuf, or null if the request can't be satisfied
* @return a new ArrowBuf, or null if the request can't be satisfied
* @throws OutOfMemoryException
* if buffer cannot be allocated
*/
Expand All @@ -44,7 +44,7 @@ public interface BufferAllocator extends AutoCloseable {
* The size in bytes.
* @param manager
* A buffer manager to manage reallocation.
* @return a new DrillBuf, or null if the request can't be satisfied
* @return a new ArrowBuf, or null if the request can't be satisfied
* @throws OutOfMemoryException
* if buffer cannot be allocated
*/
Expand Down
Loading

0 comments on commit e6905ef

Please sign in to comment.