CI: Call malloc_trim

.github/workflows/tpch.yml

@@ -37,7 +37,6 @@ jobs:
       - run: free
       - run: sudo apt-get install cmake
       - run: sudo apt-get install libboost-all-dev
-      - run: sudo apt-get install libjemalloc-dev
       - name: Install OAP optimized Arrow
         run: |
           cd /tmp
@@ -59,7 +58,7 @@ jobs:
           cd core/
           mvn test -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DmembersOnlySuites=com.intel.oap.tpch -DtagsToInclude=com.intel.oap.tags.TestAndWriteLogs
         env:
-          LD_PRELOAD: "/usr/lib/x86_64-linux-gnu/libjemalloc.so"
+          MALLOC_ARENA_MAX: "4"
           MAVEN_OPTS: "-Xmx2048m"
           COMMENT_TEXT_OUTPUT_PATH: "/tmp/comment_text.txt"
           COMMENT_IMAGE_OUTPUT_PATH: "/tmp/comment_image.png"

core/src/test/java/com/intel/oap/tpch/MallocUtils.java

@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.intel.oap.tpch;
+
+import com.intel.oap.vectorized.JniUtils;
+
+import java.io.IOException;
+
+public class MallocUtils {
+
+  static {
+    try {
+      JniUtils.getInstance();
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  /**
+   * Visible for testing: Try turning back allocated native memory to OS. This might have no effect
+   * when using Jemalloc.
+   */
+  static native void mallocTrim();
+
+  /**
+   * Visible for testing: Print malloc statistics.
+   */
+  static native void mallocStats();
+}

core/src/test/scala/com/intel/oap/tpch/MemoryUsageTest.scala

@@ -484,10 +484,13 @@ class MemoryUsageTest extends QueryTest with SharedSparkSession {
         createTPCHTables()
         writeCommentLine("```")
         writeCommentLine("Before suite starts: %s".format(genReportLine()))
-        (1 to 30).foreach { executionId =>
+        (1 to 20).foreach { executionId =>
           writeCommentLine("Iteration %d:".format(executionId))
           (1 to 22).foreach(i => {
             runTPCHQuery(i, executionId)
+            MallocUtils.mallocTrim()
+            System.gc()
+            System.gc()
             writeCommentLine("  Query %d: %s".format(i, genReportLine()))
             ramMonitor.writeImage(commentImageOutputPath)
           })

cpp/src/jni/jni_wrapper.cc

@@ -26,6 +26,7 @@
 #include <arrow/record_batch.h>
 #include <arrow/util/compression.h>
 #include <jni.h>
+#include <malloc.h>
 
 #include <iostream>
 #include <memory>
@@ -1602,6 +1603,20 @@ JNIEXPORT void JNICALL Java_com_intel_oap_vectorized_ShuffleDecompressionJniWrap
   decompression_schema_holder_.Erase(schema_holder_id);
 }
 
+JNIEXPORT void JNICALL
+Java_com_intel_oap_tpch_MallocUtils_mallocTrim(JNIEnv* env, jobject obj) {
+//  malloc_stats_print(statsPrint, nullptr, nullptr);
+  std::cout << "Calling malloc_trim... " << std::endl;
+  malloc_trim(0);
+}
+
+JNIEXPORT void JNICALL
+Java_com_intel_oap_tpch_MallocUtils_mallocStats(JNIEnv* env, jobject obj) {
+//  malloc_stats_print(statsPrint, nullptr, nullptr);
+  std::cout << "Calling malloc_stats... " << std::endl;
+  malloc_stats();
+}
+
 #ifdef __cplusplus
 }
 #endif