feat: support mallopt

.github/workflows/nodejs.yml

@@ -20,6 +20,11 @@ jobs:
         ]
         node-version: [ 12, 14, 16, 18, 20, 21 ]
     steps:
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+
     - name: Checkout Git Source
       uses: actions/checkout@master
 

binding.gyp

@@ -48,6 +48,7 @@
                 "src/jsapi/export_environment.cc",
                 "src/jsapi/export_configure.cc",
                 "src/jsapi/export_logger.cc",
+                "src/jsapi/export_mallopt.cc",
                 "src/jsapi/export_hooks.cc",
                 "src/jsapi/export_http.cc",
                 "src/jsapi/export_thread_logbypass.cc",
@@ -78,7 +79,8 @@
                         "src/platform/unix/core/linux/coredumper.cc",
                         "src/platform/unix/core/linux/elfcore.cc",
                         "src/platform/unix/core/linux/linuxthreads.cc",
-                        "src/platform/unix/core/linux/thread_lister.cc"
+                        "src/platform/unix/core/linux/thread_lister.cc",
+                        "src/platform/unix/mallopt/linux.cc",
                     ]
                 }],
                 ["OS == 'mac'", {
@@ -100,6 +102,7 @@
                         "src/platform/unix/ipc.cc",
                         "src/platform/unix/report.cc",
                         "src/platform/unix/core/darwin.cc",
+                        "src/platform/unix/mallopt/darwin.cc",
                     ]
                 }],
                 ["OS == 'win'", {
@@ -120,7 +123,8 @@
                         "src/platform/win/utils_win.cc",
                         "src/platform/win/ipc_win.cc",
                         "src/platform/win/report_win.cc",
-                        "src/platform/win/core_win.cc"
+                        "src/platform/win/core_win.cc",
+                        "src/platform/win/mallopt_win.cc",
                     ]
                 }],
             ],

configuration.js

@@ -86,5 +86,15 @@ module.exports = () => {
       ...xprofctl(true, enable => `${enable ? '启用' : '禁用'} Node.js 自动增加堆上限`),
       ...config('enable_auto_incr_heap_limit', 'XPROFILER_ENABLE_AUTO_INCR_HEAP_LIMIT', 'boolean', false),
     },
+
+    {
+      ...xprofctl(false),
+      ...config('enable_avoid_rss_leak', 'XPROFILER_ENABLE_AVOID_RSS_LEAK', 'boolean', false),
+    },
+
+    {
+      ...xprofctl(false),
+      ...config('m_mmap_threshold', 'XPROFILER_M_MMAP_THRESHOLD', 'number', 128), // KB
+    },
   ];
 };

package.json

@@ -55,20 +55,20 @@
   },
   "homepage": "https://github.com/X-Profiler/xprofiler#readme",
   "dependencies": {
-    "@xprofiler/node-pre-gyp": "^1.0.10",
-    "moment": "^2.29.4",
-    "nan": "^2.17.0",
-    "uuid": "^9.0.0",
-    "yargs": "^17.7.1"
+    "@xprofiler/node-pre-gyp": "^1.0.11",
+    "moment": "^2.30.1",
+    "nan": "^2.18.0",
+    "uuid": "^9.0.1",
+    "yargs": "^17.7.2"
   },
   "devDependencies": {
     "@istanbuljs/schema": "^0.1.3",
     "autod": "^3.1.2",
     "clang-format": "^1.8.0",
     "codecov": "^3.8.3",
-    "eslint": "^8.38.0",
+    "eslint": "^8.56.0",
     "expect.js": "^0.3.1",
-    "mm": "^3.2.2",
+    "mm": "^3.4.0",
     "mocha": "^10.2.0",
     "nyc": "^15.1.0"
   },

src/jsapi/export_mallopt.cc

@@ -0,0 +1,17 @@
+#include "export_mallopt.h"
+
+#include "configure-inl.h"
+#include "platform/platform.h"
+
+namespace xprofiler {
+using Nan::FunctionCallbackInfo;
+using v8::Value;
+
+void InitMallopt(const FunctionCallbackInfo<Value>& info) {
+  if (GetConfig<bool>("enable_avoid_rss_leak")) {
+    int threshold = GetConfig<int>("m_mmap_threshold");
+    threshold = (threshold > 128 ? threshold : 128) * 1024;
+    AvoidRssLeak(threshold);
+  }
+}
+}  // namespace xprofiler

src/jsapi/export_mallopt.h

@@ -0,0 +1,10 @@
+#ifndef XPROFILER_SRC_JSAPI_MALLOC_H
+#define XPROFILER_SRC_JSAPI_MALLOC_H
+
+#include "nan.h"
+
+namespace xprofiler {
+void InitMallopt(const Nan::FunctionCallbackInfo<v8::Value>& info);
+}
+
+#endif /* XPROFILER_SRC_JSAPI_MALLOC_H */

src/platform/platform.h

@@ -28,6 +28,9 @@ void PrintLoadedLibraries(JSONWriter* writer);
 
 // coredumper
 void WriteCore(std::string filename);
+
+// mallopt
+int AvoidRssLeak(int threshold);
 }  // namespace xprofiler
 
 #endif /* XPROFILER_SRC_PLATFORM_PLATFORM_H */

src/platform/unix/mallopt/darwin.cc

@@ -0,0 +1,7 @@
+#ifdef __APPLE__
+
+namespace xprofiler {
+int AvoidRssLeak(int threshold) { return 0; };
+}  // namespace xprofiler
+
+#endif

src/platform/unix/mallopt/linux.cc

@@ -0,0 +1,11 @@
+#ifdef __linux__
+#include <malloc.h>
+
+namespace xprofiler {
+int AvoidRssLeak(int threshold) {
+  int rc = mallopt(M_MMAP_THRESHOLD, threshold);
+  return rc;
+};
+}  // namespace xprofiler
+
+#endif

src/platform/win/mallopt_win.cc

@@ -0,0 +1,7 @@
+#ifdef _WIN32
+
+namespace xprofiler {
+int AvoidRssLeak(int threshold) { return 0; };
+}  // namespace xprofiler
+
+#endif

src/xprofiler.cc

@@ -3,6 +3,7 @@
 #include "jsapi/export_hooks.h"
 #include "jsapi/export_http.h"
 #include "jsapi/export_logger.h"
+#include "jsapi/export_mallopt.h"
 #include "jsapi/export_thread_listener.h"
 #include "jsapi/export_thread_logbypass.h"
 #include "jsapi/export_utils.h"
@@ -37,6 +38,9 @@ NAN_MODULE_INIT(Initialize) {
   // set hooks
   CREATE_JS_BINDING(setHooks, SetHooks);
 
+  // mallopt
+  CREATE_JS_BINDING(initMallopt, InitMallopt);
+
   // utils
   CREATE_JS_BINDING(checkSocketPath, CheckSocketPath);
 

test/fixtures/cases/command.js

@@ -236,12 +236,15 @@ exports = module.exports = function (logdir) {
         { key: 'data.enable_fatal_error_coredump', rule: { label: 'false', test: value => value === false } },
         { key: 'data.enable_http_profiling', rule: { label: 'false', test: value => value === false } },
         { key: 'data.enable_auto_incr_heap_limit', rule: { label: 'false', test: value => value === false } },
+        { key: 'data.enable_avoid_rss_leak', rule: { label: 'false', test: value => value === false } },
+        { key: 'data.m_mmap_threshold', rule: /^128$/ },
       ],
       xprofctlRules(data) {
         return [new RegExp(`^X-Profiler 当前配置\\(pid ${data.pid}\\):\n`
           + '  - auto_incr_heap_limit_size: 256\n'
           + '  - check_throw: false\n'
           + '  - enable_auto_incr_heap_limit: false\n'
+          + '  - enable_avoid_rss_leak: false\n'
           + '  - enable_fatal_error_coredump: false\n'
           + '  - enable_fatal_error_hook: true\n'
           + '  - enable_fatal_error_report: true\n'
@@ -252,6 +255,7 @@ exports = module.exports = function (logdir) {
           + '  - log_interval: 60\n'
           + '  - log_level: 2\n'
           + '  - log_type: 1\n'
+          + '  - m_mmap_threshold: 128\n'
           + '  - patch_http: true\n'
           + '  - patch_http_timeout: 30')
         ];

test/fixtures/cases/config.js

@@ -148,6 +148,18 @@ const configure = {
     envValue: 'YES',
     userValue: true
   },
+  enable_avoid_rss_leak: {
+    defaultValue: false,
+    envKey: 'XPROFILER_ENABLE_AVOID_RSS_LEAK',
+    envValue: 'YES',
+    userValue: true
+  },
+  m_mmap_threshold: {
+    defaultValue: 128,
+    envKey: 'XPROFILER_M_MMAP_THRESHOLD',
+    envValue: 2024,
+    userValue: 4048,
+  }
 };
 
 module.exports = getTestKeys(configure);

test/fixtures/scripts/rss.js

@@ -0,0 +1,44 @@
+'use strict';
+
+const xprofiler = require('../../../');
+xprofiler.start();
+
+const bs = 4 * 1024 * 1024; // 4 MiB
+const retained = [];
+let i = 0, flag = false;
+
+const rssMap = {
+  before: [],
+  after: [],
+};
+
+function tick() {
+  i++;
+  const rss = Math.round(process.memoryUsage().rss / 1024 / 1024);
+  if (i % 1000 === 0) {
+    console.log(`RSS [${i}]: ${rss} MiB`);
+    if (flag) {
+      rssMap.after.push(rss);
+    } else {
+      rssMap.before.push(rss);
+    }
+  }
+  retained.push(Buffer.allocUnsafe(bs));
+  if (i === 5000) {
+    console.log('Clearing retained and enabling alloc');
+    retained.length = 0;
+    flag = true;
+  }
+  if (flag) {
+    // Buffer.alloc(bs - 10) seems to be fine here
+    Buffer.alloc(bs);
+  }
+
+  if (i < 10000 && rss < 2 * 1024) {
+    setImmediate(tick);
+  } else {
+    process.send(rssMap);
+  }
+}
+
+tick();

test/mallopt.test.js

@@ -0,0 +1,56 @@
+'use strict';
+
+const os = require('os');
+const cp = require('child_process');
+const path = require('path');
+const expect = require('expect.js');
+const moment = require('moment');
+const utils = require('./fixtures/utils');
+
+const logdir = utils.createLogDir('logdir_mallopt');
+const rssPath = path.join(__dirname, 'fixtures/scripts/rss.js');
+
+(os.platform() === 'linux' ? describe : describe.skip)('avoid rss leak by mallopt', function () {
+  let rssMap;
+  let exitInfo = { code: null, signal: null };
+  before(async function () {
+    const p = cp.fork(rssPath, {
+      env: Object.assign({
+        XPROFILER_LOG_DIR: logdir,
+        XPROFILER_LOG_LEVEL: 2,
+        XPROFILER_LOG_TYPE: 1,
+        XPROFILER_ENABLE_AVOID_RSS_LEAK: 'YES',
+      })
+    });
+    p.on('message', msg => rssMap = msg);
+    exitInfo = await utils.getChildProcessExitInfo(p);
+  });
+
+  after(function () {
+    utils.cleanDir(logdir);
+  });
+
+  it('child process should be exited with code 0', function () {
+    console.log(`[${moment().format('YYYY-MM-DD HH:mm:ss')}]`, `exit info: ${JSON.stringify(exitInfo)}`);
+    utils.checkChildProcessExitInfo(expect, exitInfo);
+  });
+
+  it('should avoid rss leak', function () {
+    console.log(`process rss map: ${JSON.stringify(rssMap)}`);
+    const threshold = 200;
+    const list = Object.keys(rssMap).map(key => ({
+      key,
+      value: rssMap[key]
+    }));
+
+    for (const pair of list) {
+      describe(pair.key, function () {
+        for (const item of pair.value) {
+          it(`should ${item} < ${threshold} (MiB)`, function () {
+            expect(Number(item) < threshold).to.be.ok();
+          });
+        }
+      });
+    }
+  });
+});

xprofiler.js

@@ -22,7 +22,8 @@ xprofiler.setup({
 const runOnceStatus = {
   bypassLogThreadStarted: false,
   commandsListenerThreadStarted: false,
-  hooksSetted: false
+  hooksSetted: false,
+  malloptInited: false,
 };
 
 let configured = false;
@@ -75,6 +76,8 @@ function start(config = {}) {
     if (!singleModuleMode) {
       // start commands listener thread if needed
       exports.runCommandsListener();
+      // init mallopt
+      exports.initMallopt();
     }
   }
 
@@ -128,3 +131,5 @@ exports.runLogBypass = runOnce.bind(null, 'bypassLogThreadStarted', xprofiler.ru
 exports.runCommandsListener = runOnce.bind(null, 'commandsListenerThreadStarted', xprofiler.runCommandsListener);
 
 exports.setHooks = runOnce.bind(null, 'hooksSetted', xprofiler.setHooks);
+
+exports.initMallopt = runOnce.bind(null, 'malloptInited', xprofiler.initMallopt);