Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(server): support MemoryManagement for graph query framework #2649

Merged
merged 51 commits into from
Nov 5, 2024
Merged
Changes from 1 commit
Commits
Show all changes
51 commits
Select commit Hold shift + click to select a range
7a652be
feat: framework for memoryManagement
Pengzna Aug 25, 2024
ccc918b
wip: memory pool and manager framework
Pengzna Aug 25, 2024
b266358
wip: memory allocation
Pengzna Oct 8, 2024
ff10c3a
wip: memory reclaim
Pengzna Oct 9, 2024
26b0cfe
rename class
Pengzna Oct 9, 2024
f6aeace
fix review
Pengzna Oct 9, 2024
72e5bf3
remove useless allocator
Pengzna Oct 9, 2024
48f4817
netty allocator
Pengzna Oct 10, 2024
d906d04
revert config.properties
Pengzna Oct 13, 2024
b308be0
fix and improvement for allocation and deallocation
Pengzna Oct 22, 2024
09367a1
Merge remote-tracking branch 'refs/remotes/base/master' into memory/m…
Pengzna Oct 22, 2024
ea9a459
move monitor
Pengzna Oct 22, 2024
f552fd2
Revert "move monitor"
Pengzna Oct 22, 2024
8a2c65c
improve memory arbitration
Pengzna Oct 22, 2024
c37f869
suspend query when arbitration & kill query when OOM
Pengzna Oct 23, 2024
5904909
fix review
Pengzna Oct 23, 2024
0e70e44
fury test
Pengzna Oct 23, 2024
5d71541
offHeap magic
Pengzna Oct 23, 2024
f73f0ab
Revert "fury test"
Pengzna Oct 23, 2024
871015e
offHeap magic util
Pengzna Oct 23, 2024
8344443
complete adoption for all id
Pengzna Oct 25, 2024
d9cf408
complete property adoption
Pengzna Oct 26, 2024
aaeacb5
release ByteBuf off heap memory block
Pengzna Oct 26, 2024
ef0d629
complete allocate memory test and fix bug
Pengzna Oct 27, 2024
54d1fd8
fix some bugs: arbitration & suspend
Pengzna Oct 27, 2024
bfe75c0
complete OOM UT and fix bugs
Pengzna Oct 27, 2024
ab1bcde
complete memory management framework UT and fix all bugs
Pengzna Oct 27, 2024
91df57a
fix ut
Pengzna Oct 27, 2024
52ca7af
keep format consistent with original version
Pengzna Oct 28, 2024
ee8e125
fix review
Pengzna Oct 28, 2024
1a7d461
Merge branch 'master' into memory/management
imbajin Oct 28, 2024
de9d7a1
wip: adoption to query chain & introduce factory
Pengzna Oct 28, 2024
46066eb
fix concurrent bug when local arbitrate
Pengzna Oct 29, 2024
4f1e966
add comments
Pengzna Oct 29, 2024
7be5069
Merge remote-tracking branch 'origin/memory/management' into memory/m…
Pengzna Oct 29, 2024
231b647
feat: off-heap object factory
Pengzna Oct 29, 2024
865f1fb
fix gc child bugs and add consumer test
Pengzna Oct 29, 2024
f34e233
fix deallocate netty memory block bug & add complexId test
Pengzna Oct 29, 2024
879390b
complete all ut
Pengzna Oct 29, 2024
a96e9ee
fix all bugs
Pengzna Oct 29, 2024
7c86e84
dependency
Pengzna Oct 29, 2024
5af2cb9
add comments
Pengzna Oct 29, 2024
5e47bb0
add private constructor for singleton
Pengzna Oct 29, 2024
b77346b
add memory management config
Pengzna Oct 29, 2024
d00a8df
improve robustness
Pengzna Oct 29, 2024
fecc909
remove duplicate
Pengzna Oct 29, 2024
31f1feb
improve condition usage
Pengzna Oct 29, 2024
d4035bd
improve log
Pengzna Oct 29, 2024
d87388b
fix memory conservation bug
Pengzna Oct 29, 2024
d25396d
Revert "dependency"
Pengzna Nov 4, 2024
b334c61
revert duplicate known-dependencies.txt under huge-common
Pengzna Nov 4, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
feat: framework for memoryManagement
Pengzna committed Aug 25, 2024
commit 7a652bef69828f31d4ff6d203f5577b53ded5e35
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
package org.apache.hugegraph.memory;

import org.apache.hugegraph.memory.pool.IMemoryPool;
import org.apache.hugegraph.memory.pool.impl.RootMemoryPool;

public class MemoryManager {

private static final String ROOT_POOL_NAME = "RootQueryMemoryPool";
private static final IMemoryPool ROOT = new RootMemoryPool(null, ROOT_POOL_NAME);

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
package org.apache.hugegraph.memory.allocator;

public abstract class AbstractAllocator implements IAllocator {

@Override
public long tryToAllocateOnHeap(long size) {
Pengzna marked this conversation as resolved.
Show resolved Hide resolved
return 0;
}

@Override
public long forceAllocateOnHeap(long size) {
return 0;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
package org.apache.hugegraph.memory.allocator;

public interface IAllocator {

long tryToAllocateOnHeap(long size);

long forceAllocateOnHeap(long size);

long tryToAllocateOffHeap(long size);

long forceAllocateOffHeap(long size);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
package org.apache.hugegraph.memory.arbitrator;

public interface IMemoryArbitrator {

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
package org.apache.hugegraph.memory.consumer;

public interface IMemoryConsumer {

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
package org.apache.hugegraph.memory.pool;

import java.util.Set;
import java.util.concurrent.CopyOnWriteArraySet;

import org.apache.hugegraph.memory.pool.impl.MemoryPoolStats;

public abstract class AbstractMemoryPool implements IMemoryPool {
private final IMemoryPool parent;
private final Set<IMemoryPool> children = new CopyOnWriteArraySet<>();
private final MemoryPoolStats stats;

public AbstractMemoryPool(IMemoryPool parent, String memoryPoolName) {
this.parent = parent;
this.stats = new MemoryPoolStats(memoryPoolName);
}

@Override
public MemoryPoolStats getSnapShot() {
return stats;
}

@Override
public IMemoryPool getParentPool() {
return parent;
}

@Override
public String getName() {
return stats.getMemoryPoolName();
Pengzna marked this conversation as resolved.
Show resolved Hide resolved
}

@Override
public Set<IMemoryPool> getChildrenPools() {
return children;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
package org.apache.hugegraph.memory.pool;

import java.util.Set;

import org.apache.hugegraph.memory.pool.impl.MemoryPoolStats;

public interface IMemoryPool {

MemoryPoolStats getSnapShot();

String getName();

IMemoryPool getParentPool();

Set<IMemoryPool> getChildrenPools();

long allocate(long bytes);

long free(long bytes);

long requestMemory(long bytes);

long reclaimMemory(long bytes, long maxWaitMs);

// visitChildren,传递方法引用
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
package org.apache.hugegraph.memory.pool.impl;

public class MemoryPoolStats {

private final String memoryPoolName;
private long usedBytes;
private long reservedBytes;
private long cumulativeBytes;
private long allocatedBytes;

private long numShrinks;
private long numExpands;
private long numAborts;

public MemoryPoolStats(String MemoryPoolName) {
this.memoryPoolName = MemoryPoolName;
}

@Override
public String toString() {
return String.format("IMemoryPool-%s: {usedBytes[%d], reservedBytes[%d], " +
"cumulativeBytes[%d], allocatedBytes[%d], numShrinks[%d], " +
"numExpands[%d], numAborts[%d]}.", memoryPoolName, usedBytes,
reservedBytes,
cumulativeBytes, allocatedBytes, numShrinks, numExpands,
numAborts);
}

public String getMemoryPoolName() {
return memoryPoolName;
Pengzna marked this conversation as resolved.
Show resolved Hide resolved
}

public long getUsedBytes() {
return usedBytes;
}

public void setUsedBytes(long usedBytes) {
this.usedBytes = usedBytes;
}

public long getReservedBytes() {
return reservedBytes;
}

public void setReservedBytes(long reservedBytes) {
this.reservedBytes = reservedBytes;
}

public long getCumulativeBytes() {
return cumulativeBytes;
}

public void setCumulativeBytes(long cumulativeBytes) {
this.cumulativeBytes = cumulativeBytes;
}

public long getAllocatedBytes() {
return allocatedBytes;
}

public void setAllocatedBytes(long allocatedBytes) {
this.allocatedBytes = allocatedBytes;
}

public long getNumShrinks() {
return numShrinks;
}

public void setNumShrinks(long numShrinks) {
this.numShrinks = numShrinks;
}

public long getNumExpands() {
return numExpands;
}

public void setNumExpands(long numExpands) {
this.numExpands = numExpands;
}

public long getNumAborts() {
return numAborts;
}

public void setNumAborts(long numAborts) {
this.numAborts = numAborts;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
package org.apache.hugegraph.memory.pool.impl;

public class OperatorMemoryPool {

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
package org.apache.hugegraph.memory.pool.impl;

import org.apache.hugegraph.memory.pool.AbstractMemoryPool;
import org.apache.hugegraph.memory.pool.IMemoryPool;

public class RootMemoryPool extends AbstractMemoryPool {

public RootMemoryPool(IMemoryPool parent, String name) {
super(parent, name);
}

@Override
public long reclaimMemory(long bytes, long maxWaitMs) {
return 0;
}

@Override
public long requestMemory(long bytes) {
return 0;
}

@Override
public long free(long bytes) {
return 0;
}

@Override
public long allocate(long bytes) {
return 0;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
package org.apache.hugegraph.memory.pool.impl;

public class TaskMemoryPool {

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
package org.apache.hugegraph.memory.reclaimer;

public interface IMemoryReclaimer {

}
Original file line number Diff line number Diff line change
@@ -19,7 +19,7 @@ edge.cache_type=l2

#vertex.default_label=vertex

backend=hstore
Pengzna marked this conversation as resolved.
Show resolved Hide resolved
backend=rocksdb
serializer=binary

store=hugegraph