feat: support multiple node processes

endorlabs · Sep 18, 2024 · 84748cf · 84748cf
1 parent 6b50b6c
commit 84748cf
Show file tree

Hide file tree

Showing 9 changed files with 371 additions and 23 deletions.
diff --git a/README.md b/README.md
@@ -4,7 +4,9 @@ This repository contains a fork of Node.js v20 (commit: efbec04208a3c8588d4e7f07
 
 ## Overview
 
-Node.js Trace enhances the standard Node.js runtime to generate detailed call graphs during execution. This fork produces two primary output files:
+Node.js Trace enhances the standard Node.js runtime to generate detailed call graphs during execution. 
+Each node process will create a `cg.tsv` and `funcs.tsv`. Those files can be combained into one callgraph using the script in `nodeCG2endor`.
+This fork produces two primary output files:
 
 1. `cg.tsv`: Contains the call graph data
 2. `funcs.tsv`: Contains function information (name, position, and path)
@@ -24,7 +26,7 @@ Configure Node.js Trace using environment variables:
 | Variable | Description | Default |
 |----------|-------------|---------|
 | `TRACE_ALL` | Set to `1` to trace all functions, including Node.js internals | `0` |
-| `TRACE_DEPTH` | Maximum depth of the collected stack (deeper = slower) | `Infinity` |
+| `TRACE_DEPTH` | Maximum depth of the collected stack (deeper = slower) | `100` |
 
 **Note**: If `TRACE_DEPTH` is set to a value less than 2, the call graph will be based on function entry and exit points.
 
@@ -47,8 +49,118 @@ Configure Node.js Trace using environment variables:
    TRACE_ALL=1 TRACE_DEPTH=10 ./node your-app.js
    ```
 
+However, the recommend usage is to put this version of node inside your path since some testing framework or typescript are lunching multiple processes of node.
+Adding node in the path garantee that the full execution will be executed.
+
 4. Analyze the generated `cg.tsv` and `funcs.tsv` files.
 
+
+## Format
+
+### `func.tsv`
+```tsv
+func id    name    func file id    start    end    path
+```
+```tsv
+0               0       0       195     /private/tmp/js_vuln_test/simple.js
+1               1       0       195     /private/tmp/js_vuln_test/simple.js
+2       one     2       12      60      /private/tmp/js_vuln_test/simple.js
+3       two     3       74      109     /private/tmp/js_vuln_test/simple.js
+4       three   4       125     166     /private/tmp/js_vuln_test/simple.js
+```
+
+
+### `cg.tsv`
+
+Stack mode
+```tsv
+stack IDs   func id
+```
+
+```tsv
+0,1,2,0	0
+4,0	1
+5,4	2
+6,5	3
+6	3
+4	1
+5	2
+6	3
+6	3
+5	3
+4	3
+```
+
+IN OUT mode
+
+```tsv
+I0
+I1
+I2
+I3
+O3
+I3
+O3
+O2
+O1
+I1
+I2
+I3
+O3
+I3
+O3
+O2
+I3
+O3
+O1
+I3
+O3
+O0
+```
+
+## Convert Node CG to Endor CG
+
+The folder `nodeCG2endor` contains the script to convert the collected callgraph to Endor callgraph format.
+The script will combine all the `cg.tsv` and `func.tsv` present in the project folder. 
+
+```bash
+python3 node nodeCG2endor/nodeCG2endor.py <path_to_project>
+```
+
+### Output
+
+```json
+{
+  "packageName": "<name>@<version>",
+  "createTime": "",
+  "functionMap": {
+    "0": "javascript://js_vuln_test$1.0.0/[js_vuln_test:1.0.0:js_vuln_test/simple]/()",
+    "1": "javascript://js_vuln_test$1.0.0/[js_vuln_test:1.0.0:js_vuln_test/simple]/()",
+    "2": "javascript://js_vuln_test$1.0.0/[js_vuln_test:1.0.0:js_vuln_test/simple]/one()",
+    "3": "javascript://js_vuln_test$1.0.0/[js_vuln_test:1.0.0:js_vuln_test/simple]/two()",
+    "4": "javascript://js_vuln_test$1.0.0/[js_vuln_test:1.0.0:js_vuln_test/simple]/three()"
+  },
+  "callgraph": {
+    "1": {
+      "nodes": [
+        2,
+        4
+      ]
+    },
+    "2": {
+      "nodes": [
+        3,
+        4
+      ]
+    },
+    "3": {
+      "nodes": [
+        4
+      ]
+    }
+  }
+}
+```
 ## Performance Considerations
 
 - Enabling tracing will impact performance. The impact varies based on the chosen mode and trace depth.

diff --git a/deps/v8/src/interpreter/bytecode-generator.cc b/deps/v8/src/interpreter/bytecode-generator.cc
@@ -1400,14 +1400,16 @@ std::map<std::string, int> funcMap;
 // the incremental ID of the functions
 int funcID = 0;
 // the depth of the stack to collect
-int stackDepth = 2;
+int stackDepth = 100;
 // should node functions be traced
 bool traceNode = false;
 
 bool isInit = false;
 void CleanupAtExit() {
+  std::string filename = "func_" + std::to_string(getpid()) + ".tsv";
+
   // Open the file with the constructed filename
-  FILE* cgFile = fopen("func.tsv", "w");
+  FILE* cgFile = fopen(filename.c_str(), "w");
   // fprintf(cgFile, "stack\tfunc_id\n");
   for (const auto& cg : funcs) {
     fprintf(cgFile, "%s\n", cg.c_str());
@@ -1437,10 +1439,10 @@ void create_and_add_func_name(uint32_t func_id, const FunctionLiteral* literal,
   funcName.push_back('\t');
 
   // Append name
+  funcName.append(debug_name);
   if (isConstructor) {
-    funcName.append("new ");
+    funcName.append(".constructor");
   }
-  funcName.append(debug_name);
   funcName.push_back('\t');
 
   // Append function_key
@@ -1523,7 +1525,8 @@ void BytecodeGenerator::GenerateBytecodeBody() {
     if (toTrace) {
       int start_position = literal->start_position();
       int end_position = literal->end_position();
-      std::string functionKey = std::to_string(start_position) + "\t" +
+      std::string functionKey = std::to_string(literal->function_literal_id()) +
+                                "\t" + std::to_string(start_position) + "\t" +
                                 std::to_string(end_position) + "\t" + path;
       int func_id = -1;
       if (funcMap.find(functionKey) == funcMap.end()) {
@@ -1565,7 +1568,8 @@ void BytecodeGenerator::BuildReturn(int source_position) {
       auto literal = info()->literal();
       int start_position = literal->start_position();
       int end_position = literal->end_position();
-      std::string functionKey = std::to_string(start_position) + "\t" +
+      std::string functionKey = std::to_string(literal->function_literal_id()) +
+                                "\t" + std::to_string(start_position) + "\t" +
                                 std::to_string(end_position) + "\t" + path;
       int func_id = funcMap[functionKey];
       RegisterAllocationScope register_scope(this);
@@ -1578,13 +1582,6 @@ void BytecodeGenerator::BuildReturn(int source_position) {
           .CallRuntime(Runtime::kTraceExit, result);
     }
   }
-  // if (v8_flags.trace) {
-  //   RegisterAllocationScope register_scope(this);
-  //   Register result = register_allocator()->NewRegister();
-  //   // Runtime returns {result} value, preserving accumulator.
-  //   builder()->StoreAccumulatorInRegister(result).CallRuntime(
-  //       Runtime::kTraceExit, result);
-  // }
   builder()->SetStatementPosition(source_position);
   builder()->Return();
 }

diff --git a/deps/v8/src/runtime/runtime-test.cc b/deps/v8/src/runtime/runtime-test.cc
@@ -1245,10 +1245,12 @@ std::ofstream outFile;
 uint32_t collectStackDepth = 0;
 bool isInit = false;
 
-constexpr size_t BUFFER_SIZE = 5 * 1024 * 1024;  // 5 MB buffer
+constexpr size_t BUFFER_SIZE = 1024 * 1024;  // 512 MB buffer
 std::array<char, BUFFER_SIZE> buffer;
 size_t bufferPos = 0;
+
 std::map<StackFrameId, int> frameMap;
+std::unordered_set<std::string> printedTraces;
 
 void FlushBuffer() {
   if (bufferPos > 0) {
@@ -1296,7 +1298,11 @@ RUNTIME_FUNCTION(Runtime_TraceEnter) {
 
   if (!isInit) {
     isInit = true;
-    outFile.open("cg.tsv", std::ios::out | std::ios::binary);
+    // Construct the filename with the time postfix
+    std::string filename = "cg_" + std::to_string(getpid()) + ".tsv";
+
+    // Open the file with the constructed filename
+    outFile.open(filename);
     outFile.rdbuf()->pubsetbuf(nullptr, 0);  // Disable stream buffering
     collectStackDepth = NumberToUint32(args[0]);
     std::atexit(SaveAtExit);
@@ -1305,30 +1311,39 @@ RUNTIME_FUNCTION(Runtime_TraceEnter) {
   uint32_t funcId = NumberToUint32(args[1]);
 
   if (collectStackDepth > 1) {
+    std::string stackTrace;
+    stackTrace.reserve(40);
+
     JavaScriptStackFrameIterator it(isolate);
     int level = 0;
     while (!it.done()) {
       if (level > 0) {
-        AppendCharToBuffer(',');
+        stackTrace += ',';
       }
       StackFrameId frameId = it.frame()->id();
       if (frameMap.find(frameId) != frameMap.end() ||
           level > collectStackDepth) {
-        AppendIntToBuffer(frameMap[frameId]);
+        stackTrace += std::to_string(frameMap[frameId]);
         break;
       }
       int new_frame_id = frameMap.size();
       frameMap[frameId] = new_frame_id;
-      AppendIntToBuffer(new_frame_id);
+      stackTrace += std::to_string(new_frame_id);
       ++level;
       it.Advance();
     }
-    AppendCharToBuffer('\t');
+    stackTrace += '\t';
+    stackTrace += std::to_string(funcId);
+    if (printedTraces.insert(stackTrace).second) {
+      // This is a new stack trace, so we print it
+    }
+    AppendToBuffer(stackTrace.c_str(), stackTrace.length());
+    AppendCharToBuffer('\n');
   } else {
     AppendCharToBuffer('I');
+    AppendIntToBuffer(funcId);
+    AppendCharToBuffer('\n');
   }
-  AppendIntToBuffer(funcId);
-  AppendCharToBuffer('\n');
 
   return ReadOnlyRoots(isolate).undefined_value();
 }