Skip to content

Commit

Permalink
src: port bootstrap/cache.js to C++
Browse files Browse the repository at this point in the history
This allows us to query the categories of modules in C++
so we can implement the code cache generator in C++ that
does not depend on a Node.js binary.

PR-URL: #27046
Refs: #21563
Reviewed-By: Anna Henningsen <anna@addaleax.net>
Reviewed-By: Benjamin Gruenbaum <benjamingr@gmail.com>
  • Loading branch information
joyeecheung authored and BethGriggs committed Apr 4, 2019
1 parent 2e8bcbb commit c1cbc1e
Show file tree
Hide file tree
Showing 7 changed files with 140 additions and 107 deletions.
11 changes: 4 additions & 7 deletions benchmark/fixtures/require-cachable.js
Original file line number Diff line number Diff line change
@@ -1,13 +1,10 @@
'use strict';

const list = require('internal/bootstrap/cache');
const { internalBinding } = require('internal/test/binding');
const {
isMainThread
} = require('worker_threads');
moduleCategories: { canBeRequired }
} = internalBinding('native_module');

for (const key of list.cachableBuiltins) {
if (!isMainThread && key === 'trace_events') {
continue;
}
for (const key of canBeRequired) {
require(key);
}
85 changes: 0 additions & 85 deletions lib/internal/bootstrap/cache.js

This file was deleted.

1 change: 0 additions & 1 deletion node.gyp
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
'node_intermediate_lib_type%': 'static_library',
'library_files': [
'lib/internal/bootstrap/primordials.js',
'lib/internal/bootstrap/cache.js',
'lib/internal/bootstrap/environment.js',
'lib/internal/bootstrap/loaders.js',
'lib/internal/bootstrap/node.js',
Expand Down
114 changes: 114 additions & 0 deletions src/node_native_module.cc
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,75 @@ using v8::String;
using v8::Uint8Array;
using v8::Value;

void NativeModuleLoader::InitializeModuleCategories() {
if (module_categories_.is_initialized) {
DCHECK(!module_categories_.can_be_required.empty());
return;
}

std::vector<std::string> prefixes = {
#if !HAVE_OPENSSL
"internal/crypto/",
#endif // !HAVE_OPENSSL

"internal/bootstrap/",
"internal/per_context/",
"internal/deps/",
"internal/main/"
};

module_categories_.cannot_be_required = std::set<std::string> {
#if !HAVE_INSPECTOR
"inspector",
"internal/util/inspector",
#endif // !HAVE_INSPECTOR

#if !NODE_USE_V8_PLATFORM || !defined(NODE_HAVE_I18N_SUPPORT)
"trace_events",
#endif // !NODE_USE_V8_PLATFORM

#if !HAVE_OPENSSL
"crypto",
"https",
"http2",
"tls",
"_tls_common",
"_tls_wrap",
"internal/http2/core",
"internal/http2/compat",
"internal/policy/manifest",
"internal/process/policy",
"internal/streams/lazy_transform",
#endif // !HAVE_OPENSSL

"sys", // Deprecated.
"internal/test/binding",
"internal/v8_prof_polyfill",
"internal/v8_prof_processor",
};

for (auto const& x : source_) {
const std::string& id = x.first;
for (auto const& prefix : prefixes) {
if (prefix.length() > id.length()) {
continue;
}
if (id.find(prefix) == 0) {
module_categories_.cannot_be_required.emplace(id);
}
}
}

for (auto const& x : source_) {
const std::string& id = x.first;
if (0 == module_categories_.cannot_be_required.count(id)) {
module_categories_.can_be_required.emplace(id);
}
}

module_categories_.is_initialized = true;
}

// TODO(joyeecheung): make these more general and put them into util.h
Local<Object> MapToObject(Local<Context> context,
const NativeModuleRecordMap& in) {
Expand Down Expand Up @@ -63,6 +132,39 @@ bool NativeModuleLoader::Exists(const char* id) {
return source_.find(id) != source_.end();
}

void NativeModuleLoader::GetModuleCategories(
Local<Name> property, const PropertyCallbackInfo<Value>& info) {
per_process::native_module_loader.InitializeModuleCategories();

Environment* env = Environment::GetCurrent(info);
Isolate* isolate = env->isolate();
Local<Context> context = env->context();
Local<Object> result = Object::New(isolate);

// Copy from the per-process categories
std::set<std::string> cannot_be_required =
per_process::native_module_loader.module_categories_.cannot_be_required;
std::set<std::string> can_be_required =
per_process::native_module_loader.module_categories_.can_be_required;

if (!env->owns_process_state()) {
can_be_required.erase("trace_events");
cannot_be_required.insert("trace_events");
}

result
->Set(context,
OneByteString(isolate, "cannotBeRequired"),
ToJsSet(context, cannot_be_required))
.FromJust();
result
->Set(context,
OneByteString(isolate, "canBeRequired"),
ToJsSet(context, can_be_required))
.FromJust();
info.GetReturnValue().Set(result);
}

void NativeModuleLoader::GetCacheUsage(
const FunctionCallbackInfo<Value>& args) {
Environment* env = Environment::GetCurrent(args);
Expand Down Expand Up @@ -298,6 +400,18 @@ void NativeModuleLoader::Initialize(Local<Object> target,
SideEffectType::kHasNoSideEffect)
.FromJust());

CHECK(target
->SetAccessor(
env->context(),
FIXED_ONE_BYTE_STRING(env->isolate(), "moduleCategories"),
GetModuleCategories,
nullptr,
env->as_callback_data(),
DEFAULT,
None,
SideEffectType::kHasNoSideEffect)
.FromJust());

env->SetMethod(
target, "getCacheUsage", NativeModuleLoader::GetCacheUsage);
env->SetMethod(
Expand Down
12 changes: 12 additions & 0 deletions src/node_native_module.h
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,9 @@ class NativeModuleLoader {
Environment* optional_env);

private:
static void GetModuleCategories(
v8::Local<v8::Name> property,
const v8::PropertyCallbackInfo<v8::Value>& info);
static void GetCacheUsage(const v8::FunctionCallbackInfo<v8::Value>& args);
// Passing ids of builtin module source code into JS land as
// internalBinding('native_module').moduleIds
Expand Down Expand Up @@ -84,6 +87,15 @@ class NativeModuleLoader {
static v8::MaybeLocal<v8::Function> CompileAsModule(Environment* env,
const char* id);

void InitializeModuleCategories();
struct ModuleCategories {
bool is_initialized = false;
std::set<std::string> can_be_required;
std::set<std::string> cannot_be_required;
};

ModuleCategories module_categories_;

NativeModuleRecordMap source_;
NativeModuleCacheMap code_cache_;
UnionBytes config_;
Expand Down
15 changes: 4 additions & 11 deletions test/code-cache/test-code-cache.js
Original file line number Diff line number Diff line change
Expand Up @@ -7,22 +7,15 @@

const { isMainThread } = require('../common');
const assert = require('assert');
const {
cachableBuiltins,
cannotBeRequired
} = require('internal/bootstrap/cache');

const {
internalBinding
} = require('internal/test/binding');
const {
getCacheUsage
getCacheUsage,
moduleCategories: { canBeRequired, cannotBeRequired }
} = internalBinding('native_module');

for (const key of cachableBuiltins) {
if (!isMainThread && key === 'trace_events') {
continue; // Cannot load trace_events in workers
}
for (const key of canBeRequired) {
require(key);
}

Expand Down Expand Up @@ -60,7 +53,7 @@ if (process.config.variables.node_code_cache_path === undefined) {
);

for (const key of loadedModules) {
if (cannotBeRequired.includes(key)) {
if (cannotBeRequired.has(key)) {
assert(compiledWithoutCache.has(key),
`"${key}" should've been compiled without code cache`);
} else {
Expand Down
9 changes: 6 additions & 3 deletions tools/generate_code_cache.js
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,12 @@
// compiled into the binary using the `--code-cache-path` option
// of `configure`.

const { internalBinding } = require('internal/test/binding');
const {
moduleCategories: { canBeRequired },
getCodeCache,
compileFunction,
cachableBuiltins
} = require('internal/bootstrap/cache');
} = internalBinding('native_module');

const {
types: {
Expand Down Expand Up @@ -85,7 +86,9 @@ function lexical(a, b) {
return 0;
}

for (const key of cachableBuiltins.sort(lexical)) {
// TODO(joyeecheung): support non-modules that require different
// parameters in the wrapper.
for (const key of [...canBeRequired].sort(lexical)) {
compileFunction(key); // compile it
const cachedData = getCodeCache(key);
if (!isUint8Array(cachedData)) {
Expand Down

0 comments on commit c1cbc1e

Please sign in to comment.