feat(server): support MemoryManagement for graph query framework

hugegraph-commons/hugegraph-dist/scripts/dependency/known-dependencies.txt

@@ -1,4 +1,7 @@
+animal-sniffer-annotations-1.18.jar
 annotations-13.0.jar
+annotations-4.1.1.4.jar
+bolt-1.6.2.jar
 checker-qual-3.5.0.jar
 commons-beanutils-1.9.4.jar
 commons-codec-1.13.jar
@@ -10,14 +13,25 @@ commons-lang-2.6.jar
 commons-lang3-3.12.0.jar
 commons-logging-1.1.1.jar
 commons-text-1.9.jar
+disruptor-3.3.7.jar
 error_prone_annotations-2.3.4.jar
 failureaccess-1.0.1.jar
+grpc-api-1.28.1.jar
+grpc-context-1.28.1.jar
+grpc-core-1.28.1.jar
+grpc-netty-shaded-1.28.0.jar
+grpc-protobuf-1.28.0.jar
+grpc-protobuf-lite-1.28.0.jar
+grpc-stub-1.28.0.jar
+gson-2.8.6.jar
 guava-30.0-jre.jar
 hamcrest-core-1.3.jar
+hessian-3.3.7.jar
 j2objc-annotations-1.3.jar
 jackson-annotations-2.14.0-rc1.jar
 jackson-core-2.14.0-rc1.jar
 jackson-databind-2.14.0-rc1.jar
+jackson-dataformat-yaml-2.9.3.jar
 jackson-jaxrs-base-2.14.0-rc1.jar
 jackson-jaxrs-json-provider-2.14.0-rc1.jar
 jackson-module-jaxb-annotations-2.14.0-rc1.jar
@@ -39,7 +53,23 @@ log4j-api-2.18.0.jar
 log4j-core-2.18.0.jar
 log4j-slf4j-impl-2.18.0.jar
 logging-interceptor-4.10.0.jar
-lombok-1.18.8.jar
+lookout-api-1.4.1.jar
+netty-all-4.1.42.Final.jar
 okhttp-4.10.0.jar
 okio-jvm-3.0.0.jar
+opentracing-api-0.22.0.jar
+opentracing-mock-0.22.0.jar
+opentracing-noop-0.22.0.jar
+opentracing-util-0.22.0.jar
+perfmark-api-0.19.0.jar
+proto-google-common-protos-1.17.0.jar
+protobuf-java-3.11.0.jar
 slf4j-api-1.7.25.jar
+snakeyaml-1.18.jar
+sofa-common-tools-1.0.12.jar
+sofa-rpc-all-5.7.6.jar
+swagger-annotations-1.5.18.jar
+swagger-core-1.5.18.jar
+swagger-models-1.5.18.jar
+tracer-core-3.0.8.jar
+validation-api-1.1.0.Final.jar

hugegraph-server/hugegraph-core/pom.xml

@@ -96,6 +96,11 @@
             <groupId>org.apache.tinkerpop</groupId>
             <artifactId>gremlin-driver</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.apache.fury</groupId>
+            <artifactId>fury-core</artifactId>
+            <version>0.9.0-SNAPSHOT</version>
+        </dependency>
 
         <!-- jraft -->
         <dependency>

hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/StandardHugeGraph.java

@@ -70,6 +70,8 @@
 import org.apache.hugegraph.masterelection.RoleElectionStateMachine;
 import org.apache.hugegraph.masterelection.StandardClusterRoleStore;
 import org.apache.hugegraph.masterelection.StandardRoleElectionStateMachine;
+import org.apache.hugegraph.memory.MemoryManager;
+import org.apache.hugegraph.memory.util.RoundUtil;
 import org.apache.hugegraph.meta.MetaManager;
 import org.apache.hugegraph.perf.PerfUtil.Watched;
 import org.apache.hugegraph.rpc.RpcServiceConfig4Client;
@@ -215,6 +217,13 @@ public StandardHugeGraph(HugeConfig config) {
         this.readMode = GraphReadMode.OLTP_ONLY;
         this.schedulerType = config.get(CoreOptions.SCHEDULER_TYPE);
 
+        MemoryManager.setMemoryMode(
+                MemoryManager.MemoryMode.fromValue(config.get(CoreOptions.MEMORY_MODE)));
+        MemoryManager.setMaxMemoryCapacityInBytes(config.get(CoreOptions.MAX_MEMORY_CAPACITY));
+        MemoryManager.setMaxMemoryCapacityForOneQuery(
+                config.get(CoreOptions.ONE_QUERY_MAX_MEMORY_CAPACITY));
+        RoundUtil.setAlignment(config.get(CoreOptions.MEMORY_ALIGNMENT));
+
         LockUtil.init(this.name);
 
         try {
@@ -477,8 +486,8 @@ private ISchemaTransaction openSchemaTransaction() throws HugeException {
         try {
             if (isHstore()) {
                 return new CachedSchemaTransactionV2(
-                    MetaManager.instance().metaDriver(),
-                    MetaManager.instance().cluster(), this.params);
+                        MetaManager.instance().metaDriver(),
+                        MetaManager.instance().cluster(), this.params);
             }
             return new CachedSchemaTransaction(this.params, loadSchemaStore());
         } catch (BackendException e) {

hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/CachedBackendStore.java

@@ -215,16 +215,21 @@ public Number queryNumber(Query query) {
     /**
      * Query as an Id for cache
      */
-    static class QueryId implements Id {
+    public static class QueryId implements Id {
 
-        private String query;
-        private int hashCode;
+        protected String query;
+        protected int hashCode;
 
         public QueryId(Query q) {
             this.query = q.toString();
             this.hashCode = q.hashCode();
         }
 
+        public QueryId(String query, int hashCode) {
+            this.query = query;
+            this.hashCode = hashCode;
+        }
+
         @Override
         public IdType type() {
             return IdType.UNKNOWN;

hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/id/EdgeId.java

@@ -32,8 +32,8 @@
  * > sortKeys > target-vertex-id }
  * NOTE:
  * <p>1. for edges with edgeLabelType = NORMAL: edgeLabelId = parentEdgeLabelId = subEdgeLabelId;
- *       for edges with edgeLabelType = PARENT: edgeLabelId = subEdgeLabelId, parentEdgeLabelId =
- *       edgeLabelId.fatherId
+ * for edges with edgeLabelType = PARENT: edgeLabelId = subEdgeLabelId, parentEdgeLabelId =
+ * edgeLabelId.fatherId
  * <p>2.if we use `entry.type()` which is IN or OUT as a part of id,
  * an edge's id will be different due to different directions (belongs
  * to 2 owner vertex)
@@ -49,15 +49,14 @@ public class EdgeId implements Id {
             HugeKeys.OTHER_VERTEX
     };
 
-    private final Id ownerVertexId;
-    private final Directions direction;
-    private final Id edgeLabelId;
-    private final Id subLabelId;
-    private final String sortValues;
-    private final Id otherVertexId;
-
-    private final boolean directed;
-    private String cache;
+    protected final Id ownerVertexId;
+    protected final Id edgeLabelId;
+    protected final Id subLabelId;
+    protected final Id otherVertexId;
+    protected final Directions direction;
+    protected final boolean directed;
+    protected String sortValues;
+    protected String cache;
 
     public EdgeId(HugeVertex ownerVertex, Directions direction,
                   Id edgeLabelId, Id subLabelId, String sortValues, HugeVertex otherVertex) {

hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/id/IdGenerator.java

@@ -119,15 +119,15 @@ public static IdType idType(Id id) {
         return IdType.UNKNOWN;
     }
 
-    private static int compareType(Id id1, Id id2) {
+    public static int compareType(Id id1, Id id2) {
         return idType(id1).ordinal() - idType(id2).ordinal();
     }
 
     /****************************** id defines ******************************/
 
-    public static final class StringId implements Id {
+    public static class StringId implements Id {
 
-        private final String id;
+        protected String id;
 
         public StringId(String id) {
             E.checkArgument(!id.isEmpty(), "The id can't be empty");
@@ -196,11 +196,11 @@ public String toString() {
         }
     }
 
-    public static final class LongId extends Number implements Id {
+    public static class LongId extends Number implements Id {
 
         private static final long serialVersionUID = -7732461469037400190L;
 
-        private final long id;
+        protected Long id;
 
         public LongId(long id) {
             this.id = id;
@@ -270,7 +270,7 @@ public String toString() {
 
         @Override
         public int intValue() {
-            return (int) this.id;
+            return this.id.intValue();
         }
 
         @Override
@@ -289,9 +289,9 @@ public double doubleValue() {
         }
     }
 
-    public static final class UuidId implements Id {
+    public static class UuidId implements Id {
 
-        private final UUID uuid;
+        protected UUID uuid;
 
         public UuidId(String string) {
             this(StringEncoding.uuid(string));
@@ -379,9 +379,9 @@ public String toString() {
     /**
      * This class is just used by backend store for wrapper object as Id
      */
-    public static final class ObjectId implements Id {
+    public static class ObjectId implements Id {
 
-        private final Object object;
+        protected Object object;
 
         public ObjectId(Object object) {
             E.checkNotNull(object, "object");

hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/BinaryBackendEntry.java

@@ -208,10 +208,10 @@ public int hashCode() {
         return this.id().hashCode() ^ this.columns.size();
     }
 
-    public static final class BinaryId implements Id {
+    public static class BinaryId implements Id {
 
-        private final byte[] bytes;
-        private final Id id;
+        protected byte[] bytes;
+        protected Id id;
 
         public BinaryId(byte[] bytes, Id id) {
             this.bytes = bytes;

hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/config/CoreOptions.java

@@ -20,6 +20,7 @@
 import static org.apache.hugegraph.backend.tx.GraphTransaction.COMMIT_BATCH;
 import static org.apache.hugegraph.config.OptionChecker.allowValues;
 import static org.apache.hugegraph.config.OptionChecker.disallowEmpty;
+import static org.apache.hugegraph.config.OptionChecker.nonNegativeInt;
 import static org.apache.hugegraph.config.OptionChecker.positiveInt;
 import static org.apache.hugegraph.config.OptionChecker.rangeInt;
 
@@ -681,4 +682,32 @@ public static synchronized CoreOptions instance() {
             disallowEmpty(),
             "127.0.0.1:8686"
     );
+
+    public static final ConfigOption<String> MEMORY_MODE = new ConfigOption<>(
+            "memory.mode",
+            "The memory mode used for query in HugeGraph.",
+            disallowEmpty(),
+            "off-heap"
+    );
+
+    public static final ConfigOption<Long> MAX_MEMORY_CAPACITY = new ConfigOption<>(
+            "memory.max_capacity",
+            "The maximum memory capacity that can be managed for all queries in HugeGraph.",
+            nonNegativeInt(),
+            Bytes.GB
+    );
+
+    public static final ConfigOption<Long> ONE_QUERY_MAX_MEMORY_CAPACITY = new ConfigOption<>(
+            "memory.one_query_max_capacity",
+            "The maximum memory capacity that can be managed for a query in HugeGraph.",
+            nonNegativeInt(),
+            Bytes.MB * 100
+    );
+
+    public static final ConfigOption<Long> MEMORY_ALIGNMENT = new ConfigOption<>(
+            "memory.alignment",
+            "The alignment used for round memory size.",
+            nonNegativeInt(),
+            8L
+    );
 }