diff --git a/lib/internal/main/prof_process.js b/lib/internal/main/prof_process.js index 612e3b469d84f4..c56ddc1d3dc4b7 100644 --- a/lib/internal/main/prof_process.js +++ b/lib/internal/main/prof_process.js @@ -1,10 +1,34 @@ 'use strict'; +// This main script is used by --prof-process to process logs generated by --prof. +// The tick processor implementation is vendor-in from V8, placed in deps/v8/tools, +// and they have different resolution rules compared to normal Node.js internal builtins, +// despite being embedded into the binary as a built-in as well. + +// TODO(joyeecheung): put the context locals in import.meta. +const { + ObjectAssign, + globalThis, +} = primordials; + const { prepareMainThreadExecution, markBootstrapComplete, } = require('internal/process/pre_execution'); +const { importBuiltinSourceTextModule } = internalBinding('builtins'); + prepareMainThreadExecution(); markBootstrapComplete(); -require('internal/v8_prof_processor'); + +// Polyfill the context with globals needed by the tick processor. +const { globals, openFile } = require('internal/v8_prof_polyfill'); +ObjectAssign(globalThis, globals); +openFile(process.argv[process.argv.length - 1]); + +(async () => { + // Load and evaluate the V8 tick processor as a source text module. + // Evaluation is asynchronous because the tick processor contains top level await. + const { promise } = importBuiltinSourceTextModule('internal/deps/v8/tools/tickprocessor-driver'); + await promise; +})(); diff --git a/lib/internal/v8_prof_polyfill.js b/lib/internal/v8_prof_polyfill.js index 8d047a530d799c..4922d448cfec50 100644 --- a/lib/internal/v8_prof_polyfill.js +++ b/lib/internal/v8_prof_polyfill.js @@ -30,17 +30,17 @@ /* eslint-disable node-core/prefer-primordials, no-restricted-globals */ /* global console */ -module.exports = { versionCheck }; +const { + ArrayPrototypePush, + ArrayPrototypePushApply, + ArrayPrototypeSlice, +} = primordials; -// Don't execute when required directly instead of being eval'd from -// lib/internal/v8_prof_processor.js. This way we can test functions -// from this file in isolation. -if (module.id === 'internal/v8_prof_polyfill') return; - -// Node polyfill const fs = require('fs'); const cp = require('child_process'); const { Buffer } = require('buffer'); +const { StringDecoder } = require('string_decoder'); + const os = { system: function(name, args) { if (process.platform === 'linux' && name === 'nm') { @@ -68,24 +68,40 @@ const os = { }, }; const print = console.log; + function read(fileName) { return fs.readFileSync(fileName, 'utf8'); } const quit = process.exit; -// Polyfill "readline()". -const logFile = globalThis.arguments[globalThis.arguments.length - 1]; -try { - fs.accessSync(logFile); -} catch { - console.error('Please provide a valid isolate file as the final argument.'); - process.exit(1); + +const tickArguments = []; +if (process.platform === 'darwin') { + ArrayPrototypePush(tickArguments, '--mac'); +} else if (process.platform === 'win32') { + ArrayPrototypePush(tickArguments, '--windows'); } -const fd = fs.openSync(logFile, 'r'); +ArrayPrototypePushApply(tickArguments, + ArrayPrototypeSlice(process.argv, 1)); + +function write(str) { process.stdout.write(str); }; +function printErr(str) { process.stderr.write(str); }; + +let logFile; +let fd; const buf = Buffer.allocUnsafe(4096); -const dec = new (require('string_decoder').StringDecoder)('utf-8'); +const dec = new StringDecoder('utf-8'); let line = ''; -{ +function openFile(filename) { + logFile = filename; + try { + fd = fs.openSync(logFile, 'r'); + } catch (e) { + console.error(`Cannot access log file: ${logFile}`); + console.error('Please provide a valid isolate file as the final argument.'); + throw e; + } + const message = versionCheck(peekline(), process.versions.v8); if (message) console.log(message); } @@ -162,10 +178,17 @@ function macCppfiltNm(out) { }); } -Object.assign(globalThis, { - os, - print, - read, - quit, - readline, -}); +module.exports = { + globals: { + arguments: tickArguments, + write, + printErr, + os, + print, + read, + quit, + readline, + }, + openFile, + versionCheck, +}; diff --git a/lib/internal/v8_prof_processor.js b/lib/internal/v8_prof_processor.js deleted file mode 100644 index 1d4f33418ff733..00000000000000 --- a/lib/internal/v8_prof_processor.js +++ /dev/null @@ -1,54 +0,0 @@ -'use strict'; - -const { - ArrayPrototypePush, - ArrayPrototypePushApply, - ArrayPrototypeSlice, - StringPrototypeSlice, -} = primordials; - -const Buffer = require('buffer').Buffer; -const console = require('internal/console/global'); -const vm = require('vm'); -const { SourceTextModule } = require('internal/vm/module'); - -const { natives } = internalBinding('builtins'); - -async function linker(specifier, referencingModule) { - // Transform "./file.mjs" to "file" - const file = StringPrototypeSlice(specifier, 2, -4); - const code = natives[`internal/deps/v8/tools/${file}`]; - return new SourceTextModule(code, { context: referencingModule.context }); -} - -(async () => { - const tickArguments = []; - if (process.platform === 'darwin') { - ArrayPrototypePush(tickArguments, '--mac'); - } else if (process.platform === 'win32') { - ArrayPrototypePush(tickArguments, '--windows'); - } - ArrayPrototypePushApply(tickArguments, - ArrayPrototypeSlice(process.argv, 1)); - - const context = vm.createContext({ - arguments: tickArguments, - write(s) { process.stdout.write(s); }, - printErr(err) { console.error(err); }, - console, - process, - Buffer, - }); - - const polyfill = natives['internal/v8_prof_polyfill']; - const script = `(function(module, require) { - ${polyfill} - })`; - - vm.runInContext(script, context)(module, require); - - const tickProcessor = natives['internal/deps/v8/tools/tickprocessor-driver']; - const tickprocessorDriver = new SourceTextModule(tickProcessor, { context }); - await tickprocessorDriver.link(linker); - await tickprocessorDriver.evaluate(); -})(); diff --git a/node.gyp b/node.gyp index a80260ba173fc9..2dfbaea8cd68ad 100644 --- a/node.gyp +++ b/node.gyp @@ -73,6 +73,7 @@ 'src/async_context_frame.cc', 'src/async_wrap.cc', 'src/base_object.cc', + 'src/builtin_info.cc', 'src/cares_wrap.cc', 'src/cleanup_queue.cc', 'src/compile_cache.cc', @@ -214,6 +215,7 @@ 'src/base_object_types.h', 'src/blob_serializer_deserializer.h', 'src/blob_serializer_deserializer-inl.h', + "src/builtin_info.h", 'src/callback_queue.h', 'src/callback_queue-inl.h', 'src/cleanup_queue.h', @@ -1381,6 +1383,8 @@ 'tools/executable_wrapper.h', 'src/embedded_data.h', 'src/embedded_data.cc', + 'src/builtin_info.h', + 'src/builtin_info.cc', ], 'conditions': [ [ 'node_shared_simdutf=="false"', { diff --git a/src/api/environment.cc b/src/api/environment.cc index 8126749cc2034b..16ac44938d7947 100644 --- a/src/api/environment.cc +++ b/src/api/environment.cc @@ -914,7 +914,7 @@ Maybe InitializePrimordials(Local context, exports, primordials, private_symbols, per_isolate_symbols}; if (builtin_loader - .CompileAndCall( + .CompileAndCallWith( context, *module, arraysize(arguments), arguments, nullptr) .IsEmpty()) { // Execution failed during context creation. diff --git a/src/builtin_info.cc b/src/builtin_info.cc new file mode 100644 index 00000000000000..d5309265ac3733 --- /dev/null +++ b/src/builtin_info.cc @@ -0,0 +1,48 @@ +#include "builtin_info.h" + +namespace node { +namespace builtins { + +BuiltinSourceType GetBuiltinSourceType(const std::string& id, + const std::string& filename) { + if (filename.ends_with(".mjs")) { + return BuiltinSourceType::kSourceTextModule; + } + if (id.starts_with("internal/bootstrap/realm")) { + return BuiltinSourceType::kBootstrapRealm; + } + if (id.starts_with("internal/bootstrap/")) { + return BuiltinSourceType::kBootstrapScript; + } + if (id.starts_with("internal/per_context/")) { + return BuiltinSourceType::kPerContextScript; + } + if (id.starts_with("internal/main/")) { + return BuiltinSourceType::kMainScript; + } + if (id.starts_with("internal/deps/v8/tools/")) { + return BuiltinSourceType::kSourceTextModule; + } + return BuiltinSourceType::kFunction; +} + +std::string GetBuiltinSourceTypeName(BuiltinSourceType type) { + switch (type) { + case BuiltinSourceType::kBootstrapRealm: + return "kBootstrapRealm"; + case BuiltinSourceType::kBootstrapScript: + return "kBootstrapScript"; + case BuiltinSourceType::kPerContextScript: + return "kPerContextScript"; + case BuiltinSourceType::kMainScript: + return "kMainScript"; + case BuiltinSourceType::kFunction: + return "kFunction"; + case BuiltinSourceType::kSourceTextModule: + return "kSourceTextModule"; + } + abort(); +} + +} // namespace builtins +} // namespace node diff --git a/src/builtin_info.h b/src/builtin_info.h new file mode 100644 index 00000000000000..0aec3cecfe2bc5 --- /dev/null +++ b/src/builtin_info.h @@ -0,0 +1,57 @@ +#ifndef SRC_BUILTIN_INFO_H_ +#define SRC_BUILTIN_INFO_H_ + +#include +#include +#include +#include + +// This file must not depend on node.h or other code that depends on +// the full Node.js implementation because it is used during the +// compilation of the Node.js implementation itself (especially js2c). + +namespace node { +namespace builtins { + +enum class BuiltinSourceType { + kBootstrapRealm, // internal/bootstrap/realm + kBootstrapScript, // internal/bootstrap/* + kPerContextScript, // internal/per_context/* + kMainScript, // internal/main/* + kFunction, // others + kSourceTextModule, // selected modules, ends with .mjs in source +}; + +struct BuiltinInfo { + // C++17 inline static + inline static const std::unordered_map> + parameter_map{ + {BuiltinSourceType::kBootstrapRealm, + {"process", + "getLinkedBinding", + "getInternalBinding", + "primordials"}}, + {BuiltinSourceType::kBootstrapScript, + {"process", "require", "internalBinding", "primordials"}}, + {BuiltinSourceType::kPerContextScript, + {"exports", "primordials", "privateSymbols", "perIsolateSymbols"}}, + {BuiltinSourceType::kMainScript, + {"process", "require", "internalBinding", "primordials"}}, + {BuiltinSourceType::kFunction, + {"exports", + "require", + "module", + "process", + "internalBinding", + "primordials"}}, + }; +}; + +std::string GetBuiltinSourceTypeName(BuiltinSourceType type); +BuiltinSourceType GetBuiltinSourceType(const std::string& id, + const std::string& filename); +} // namespace builtins +} // namespace node + +#endif // SRC_BUILTIN_INFO_H_ diff --git a/src/env_properties.h b/src/env_properties.h index 114f23c9860dcf..5675e391820a4c 100644 --- a/src/env_properties.h +++ b/src/env_properties.h @@ -57,6 +57,7 @@ V(onpskexchange_symbol, "onpskexchange") \ V(resource_symbol, "resource_symbol") \ V(trigger_async_id_symbol, "trigger_async_id_symbol") \ + V(builtin_source_text_module_hdo, "builtin_source_text_module_hdo") \ V(source_text_module_default_hdo, "source_text_module_default_hdo") \ V(vm_context_no_contextify, "vm_context_no_contextify") \ V(vm_dynamic_import_default_internal, "vm_dynamic_import_default_internal") \ diff --git a/src/node_builtins.cc b/src/node_builtins.cc index 7e962e7ecdabb2..2a77bf6d7a4715 100644 --- a/src/node_builtins.cc +++ b/src/node_builtins.cc @@ -1,6 +1,8 @@ #include "node_builtins.h" #include "debug_utils-inl.h" #include "env-inl.h" +#include "module_wrap.h" +#include "node_errors.h" #include "node_external_reference.h" #include "node_internals.h" #include "node_threadsafe_cow-inl.h" @@ -11,10 +13,12 @@ namespace node { namespace builtins { +using loader::HostDefinedOptions; using v8::Boolean; using v8::Context; +using v8::Data; using v8::EscapableHandleScope; -using v8::Exception; +using v8::FixedArray; using v8::Function; using v8::FunctionCallbackInfo; using v8::IntegrityLevel; @@ -22,11 +26,13 @@ using v8::Isolate; using v8::Local; using v8::LocalVector; using v8::MaybeLocal; +using v8::Module; +using v8::ModuleRequest; using v8::Name; -using v8::NewStringType; using v8::None; using v8::Object; using v8::ObjectTemplate; +using v8::PrimitiveArray; using v8::PropertyCallbackInfo; using v8::ScriptCompiler; using v8::ScriptOrigin; @@ -75,9 +81,12 @@ bool BuiltinLoader::Exists(const char* id) { return source->find(id) != source->end(); } -bool BuiltinLoader::Add(const char* id, const UnionBytes& source) { - auto result = source_.write()->emplace(id, source); - return result.second; +const BuiltinSource* BuiltinLoader::AddFromDisk(const char* id, + const std::string& filename, + const UnionBytes& source) { + BuiltinSourceType type = GetBuiltinSourceType(id, filename); + auto result = source_.write()->emplace(id, BuiltinSource{id, source, type}); + return &(result.first->second); } void BuiltinLoader::GetNatives(Local property, @@ -90,7 +99,8 @@ void BuiltinLoader::GetNatives(Local property, auto source = env->builtin_loader()->source_.read(); for (auto const& x : *source) { Local key = OneByteString(isolate, x.first); - if (out->Set(context, key, x.second.ToStringChecked(isolate)).IsNothing()) { + if (out->Set(context, key, x.second.source.ToStringChecked(isolate)) + .IsNothing()) { return; } } @@ -150,7 +160,6 @@ BuiltinLoader::BuiltinCategories BuiltinLoader::GetBuiltinCategories() const { "internal/webstorage", // Experimental. #endif "internal/test/binding", "internal/v8_prof_polyfill", - "internal/v8_prof_processor", }; auto source = source_.read(); @@ -188,38 +197,30 @@ static std::string OnDiskFileName(const char* id) { filename += "lib/"; } filename += id; - filename += ".js"; + if (strncmp(id, "deps/v8/tools", strlen("deps/v8/tools")) == 0) { + // V8 tools scripts are .mjs files. + filename += ".mjs"; + } else { + filename += ".js"; + } return filename; } #endif // NODE_BUILTIN_MODULES_PATH -MaybeLocal BuiltinLoader::LoadBuiltinSource(Isolate* isolate, - const char* id) const { - auto source = source_.read(); +const BuiltinSource* BuiltinLoader::LoadBuiltinSource(Isolate* isolate, + const char* id) { #ifndef NODE_BUILTIN_MODULES_PATH + auto source = source_.read(); const auto source_it = source->find(id); if (source_it == source->end()) [[unlikely]] { fprintf(stderr, "Cannot find native builtin: \"%s\".\n", id); ABORT(); } - return source_it->second.ToStringChecked(isolate); + return &(source_it->second); #else // !NODE_BUILTIN_MODULES_PATH std::string filename = OnDiskFileName(id); - - std::string contents; - int r = ReadFileSync(&contents, filename.c_str()); - if (r != 0) { - const std::string buf = SPrintF("Cannot read local builtin. %s: %s \"%s\"", - uv_err_name(r), - uv_strerror(r), - filename); - Local message = OneByteString(isolate, buf); - isolate->ThrowException(Exception::Error(message)); - return MaybeLocal(); - } - return String::NewFromUtf8( - isolate, contents.c_str(), NewStringType::kNormal, contents.length()); + return AddExternalizedBuiltin(id, filename.c_str()); #endif // NODE_BUILTIN_MODULES_PATH } @@ -229,8 +230,8 @@ std::unordered_map> externalized_builtin_sources; } // namespace -void BuiltinLoader::AddExternalizedBuiltin(const char* id, - const char* filename) { +const BuiltinSource* BuiltinLoader::AddExternalizedBuiltin( + const char* id, const char* filename) { StaticExternalTwoByteResource* resource; { Mutex::ScopedLock lock(externalized_builtins_mutex); @@ -267,26 +268,28 @@ void BuiltinLoader::AddExternalizedBuiltin(const char* id, resource = it->second.get(); } - Add(id, UnionBytes(resource)); + return AddFromDisk(id, filename, UnionBytes(resource)); } -MaybeLocal BuiltinLoader::LookupAndCompileInternal( +MaybeLocal BuiltinLoader::LookupAndCompile(Local context, + const char* id, + Realm* optional_realm) { + Isolate* isolate = Isolate::GetCurrent(); + const BuiltinSource* builtin_source = LoadBuiltinSource(isolate, id); + if (builtin_source == nullptr) { + THROW_ERR_MODULE_NOT_FOUND(isolate, "Cannot find module %s", id); + return MaybeLocal(); + } + return LookupAndCompile(context, builtin_source, optional_realm); +} + +MaybeLocal BuiltinLoader::LookupAndCompile( Local context, - const char* id, - LocalVector* parameters, + const BuiltinSource* builtin_source, Realm* optional_realm) { Isolate* isolate = Isolate::GetCurrent(); EscapableHandleScope scope(isolate); - Local source; - if (!LoadBuiltinSource(isolate, id).ToLocal(&source)) { - return {}; - } - - std::string filename_s = std::string("node:") + id; - Local filename = OneByteString(isolate, filename_s); - ScriptOrigin origin(filename, 0, 0, true); - BuiltinCodeCacheData cached_data{}; { // Note: The lock here should not extend into the @@ -294,7 +297,7 @@ MaybeLocal BuiltinLoader::LookupAndCompileInternal( // there is a syntax error during bootstrap (because the fatal exception // handler is invoked, which may load built-in modules). RwLock::ScopedLock lock(code_cache_->mutex); - auto cache_it = code_cache_->map.find(id); + auto cache_it = code_cache_->map.find(builtin_source->id); if (cache_it != code_cache_->map.end()) { // Transfer ownership to ScriptCompiler::Source later. cached_data = cache_it->second; @@ -308,64 +311,113 @@ MaybeLocal BuiltinLoader::LookupAndCompileInternal( if (should_eager_compile_) { options = ScriptCompiler::kEagerCompile; } else if (!to_eager_compile_.empty()) { - if (to_eager_compile_.contains(id)) { + if (to_eager_compile_.contains(builtin_source->id)) { options = ScriptCompiler::kEagerCompile; } } - ScriptCompiler::Source script_source( - source, - origin, - has_cache ? cached_data.AsCachedData().release() : nullptr); per_process::Debug( DebugCategory::CODE_CACHE, "Compiling %s %s code cache %s\n", - id, + builtin_source->id, has_cache ? "with" : "without", options == ScriptCompiler::kEagerCompile ? "eagerly" : "lazily"); - MaybeLocal maybe_fun = - ScriptCompiler::CompileFunction(context, - &script_source, - parameters->size(), - parameters->data(), - 0, - nullptr, - options); - - // This could fail when there are early errors in the built-in modules, - // e.g. the syntax errors - Local fun; - if (!maybe_fun.ToLocal(&fun)) { - // In the case of early errors, v8 is already capable of - // decorating the stack for us - note that we use CompileFunction - // so there is no need to worry about wrappers. - return MaybeLocal(); + // The ownership of cached_data_ptr will be transferred to + // ScriptCompiler::Source. + ScriptCompiler::CachedData* cached_data_ptr = + has_cache ? cached_data.AsCachedData().release() : nullptr; + std::string filename_s = std::string("node:") + builtin_source->id; + Local filename = OneByteString(isolate, filename_s); + Local source = builtin_source->source.ToStringChecked(isolate); + + Local compiled_result; + // This needs to be retrieved after compilation, as the ownership of + // cached data will be transferred to script source. + bool is_cache_accepted = false; + bool is_buffer_owned = cached_data_ptr + ? (cached_data_ptr->buffer_policy == + ScriptCompiler::CachedData::BufferOwned) + : false; + if (builtin_source->type == BuiltinSourceType::kSourceTextModule) { + Local host_defined_options; + if (optional_realm != nullptr) { + host_defined_options = + PrimitiveArray::New(isolate, HostDefinedOptions::kLength); + host_defined_options->Set( + isolate, + HostDefinedOptions::kID, + optional_realm->isolate_data()->builtin_source_text_module_hdo()); + } + ScriptOrigin origin(filename, + 0, + 0, + true, // is cross origin + -1, // script id + Local(), // source map URL + false, // is opaque + false, // is WASM + true, // is ES Module + host_defined_options); + ScriptCompiler::Source script_source(source, origin, cached_data_ptr); + MaybeLocal maybe_mod = + ScriptCompiler::CompileModule(isolate, &script_source, options); + if (maybe_mod.IsEmpty()) { + // In the case of early errors, v8 is already capable of + // decorating the stack for us. + return MaybeLocal(); + } + compiled_result = maybe_mod.ToLocalChecked(); + is_cache_accepted = has_cache && !script_source.GetCachedData()->rejected; + } else { + ScriptOrigin origin(filename, 0, 0, true); + ScriptCompiler::Source script_source(source, origin, cached_data_ptr); + LocalVector parameters(isolate); + auto params_it = BuiltinInfo::parameter_map.find(builtin_source->type); + CHECK_NE(params_it, BuiltinInfo::parameter_map.end()); + parameters.reserve(params_it->second.size()); + for (const std::string& param : params_it->second) { + parameters.push_back(OneByteString(isolate, param)); + } + MaybeLocal maybe_fun = + ScriptCompiler::CompileFunction(context, + &script_source, + parameters.size(), + parameters.data(), + 0, + nullptr, + options); + // This could fail when there are early errors in the built-in modules, + // e.g. the syntax errors + Local fun; + if (!maybe_fun.ToLocal(&fun)) { + // In the case of early errors, v8 is already capable of + // decorating the stack for us - note that we use CompileFunction + // so there is no need to worry about wrappers. + return MaybeLocal(); + } + + compiled_result = fun; + is_cache_accepted = has_cache && !script_source.GetCachedData()->rejected; } // XXX(joyeecheung): this bookkeeping is not exactly accurate because // it only starts after the Environment is created, so the per_context.js // will never be in any of these two sets, but the two sets are only for // testing anyway. - - Result result = (has_cache && !script_source.GetCachedData()->rejected) - ? Result::kWithCache - : Result::kWithoutCache; + Result result = + is_cache_accepted ? Result::kWithCache : Result::kWithoutCache; if (optional_realm != nullptr) { DCHECK_EQ(this, optional_realm->env()->builtin_loader()); - RecordResult(id, result, optional_realm); + RecordResult(builtin_source->id, result, optional_realm); } if (has_cache) { per_process::Debug(DebugCategory::CODE_CACHE, "Code cache of %s (%s) %s\n", - id, - script_source.GetCachedData()->buffer_policy == - ScriptCompiler::CachedData::BufferNotOwned - ? "BufferNotOwned" - : "BufferOwned", - script_source.GetCachedData()->rejected ? "is rejected" - : "is accepted"); + builtin_source->id, + is_buffer_owned ? "BufferOwned" : "BufferNotOwned", + is_cache_accepted ? "is accepted" : "is rejected"); } if (result == Result::kWithoutCache && optional_realm != nullptr && @@ -376,15 +428,22 @@ MaybeLocal BuiltinLoader::LookupAndCompileInternal( // This is only done when the isolate is not being serialized because // V8 does not support serializing code cache with an unfinalized read-only // space (which is what isolates pending to be serialized have). - SaveCodeCache(id, fun); + SaveCodeCache(builtin_source->id, compiled_result); } - return scope.Escape(fun); + return scope.Escape(compiled_result); } -void BuiltinLoader::SaveCodeCache(const char* id, Local fun) { - std::shared_ptr new_cached_data( - ScriptCompiler::CreateCodeCacheForFunction(fun)); +void BuiltinLoader::SaveCodeCache(const std::string& id, Local data) { + std::shared_ptr new_cached_data; + if (data->IsModule()) { + Local mod = data.As(); + new_cached_data.reset( + ScriptCompiler::CreateCodeCache(mod->GetUnboundModuleScript())); + } else { + Local fun = data.As(); + new_cached_data.reset(ScriptCompiler::CreateCodeCacheForFunction(fun)); + } CHECK_NOT_NULL(new_cached_data); { @@ -394,115 +453,93 @@ void BuiltinLoader::SaveCodeCache(const char* id, Local fun) { } } -MaybeLocal BuiltinLoader::LookupAndCompile(Local context, - const char* id, - Realm* optional_realm) { +MaybeLocal BuiltinLoader::LookupAndCompileFunction( + Local context, const char* id, Realm* optional_realm) { Isolate* isolate = Isolate::GetCurrent(); LocalVector parameters(isolate); - // Detects parameters of the scripts based on module ids. - // internal/bootstrap/realm: process, getLinkedBinding, - // getInternalBinding, primordials - if (strcmp(id, "internal/bootstrap/realm") == 0) { - parameters = { - FIXED_ONE_BYTE_STRING(isolate, "process"), - FIXED_ONE_BYTE_STRING(isolate, "getLinkedBinding"), - FIXED_ONE_BYTE_STRING(isolate, "getInternalBinding"), - FIXED_ONE_BYTE_STRING(isolate, "primordials"), - }; - } else if (strncmp(id, - "internal/per_context/", - strlen("internal/per_context/")) == 0) { - // internal/per_context/*: global, exports, primordials - parameters = { - FIXED_ONE_BYTE_STRING(isolate, "exports"), - FIXED_ONE_BYTE_STRING(isolate, "primordials"), - FIXED_ONE_BYTE_STRING(isolate, "privateSymbols"), - FIXED_ONE_BYTE_STRING(isolate, "perIsolateSymbols"), - }; - } else if (strncmp(id, "internal/main/", strlen("internal/main/")) == 0 || - strncmp(id, - "internal/bootstrap/", - strlen("internal/bootstrap/")) == 0) { - // internal/main/*, internal/bootstrap/*: process, require, - // internalBinding, primordials - parameters = { - FIXED_ONE_BYTE_STRING(isolate, "process"), - FIXED_ONE_BYTE_STRING(isolate, "require"), - FIXED_ONE_BYTE_STRING(isolate, "internalBinding"), - FIXED_ONE_BYTE_STRING(isolate, "primordials"), - }; - } else { - // others: exports, require, module, process, internalBinding, primordials - parameters = { - FIXED_ONE_BYTE_STRING(isolate, "exports"), - FIXED_ONE_BYTE_STRING(isolate, "require"), - FIXED_ONE_BYTE_STRING(isolate, "module"), - FIXED_ONE_BYTE_STRING(isolate, "process"), - FIXED_ONE_BYTE_STRING(isolate, "internalBinding"), - FIXED_ONE_BYTE_STRING(isolate, "primordials"), - }; + + Local data; + if (!LookupAndCompile(context, id, optional_realm).ToLocal(&data)) { + return MaybeLocal(); } - MaybeLocal maybe = - LookupAndCompileInternal(context, id, ¶meters, optional_realm); - return maybe; + CHECK(data->IsValue()); + Local value = data.As(); + CHECK(value->IsFunction()); + return value.As(); } MaybeLocal BuiltinLoader::CompileAndCall(Local context, const char* id, Realm* realm) { Isolate* isolate = Isolate::GetCurrent(); - // Detects parameters of the scripts based on module ids. - // internal/bootstrap/realm: process, getLinkedBinding, - // getInternalBinding, primordials - if (strcmp(id, "internal/bootstrap/realm") == 0) { - Local get_linked_binding; - Local get_internal_binding; - if (!NewFunctionTemplate(isolate, binding::GetLinkedBinding) - ->GetFunction(context) - .ToLocal(&get_linked_binding) || - !NewFunctionTemplate(isolate, binding::GetInternalBinding) - ->GetFunction(context) - .ToLocal(&get_internal_binding)) { + const BuiltinSource* builtin_source = LoadBuiltinSource(isolate, id); + if (builtin_source == nullptr) { + THROW_ERR_MODULE_NOT_FOUND(isolate, "Cannot find module %s", id); + return MaybeLocal(); + } + + if (builtin_source->type == BuiltinSourceType::kSourceTextModule) { + Local value; + if (!ImportBuiltinSourceTextModule(realm, id).ToLocal(&value)) { return MaybeLocal(); } - Local arguments[] = {realm->process_object(), - get_linked_binding, - get_internal_binding, - realm->primordials()}; - return CompileAndCall( - context, id, arraysize(arguments), &arguments[0], realm); - } else if (strncmp(id, "internal/main/", strlen("internal/main/")) == 0 || - strncmp(id, - "internal/bootstrap/", - strlen("internal/bootstrap/")) == 0) { - // internal/main/*, internal/bootstrap/*: process, require, - // internalBinding, primordials - Local arguments[] = {realm->process_object(), - realm->builtin_module_require(), - realm->internal_binding_loader(), - realm->primordials()}; - return CompileAndCall( - context, id, arraysize(arguments), &arguments[0], realm); - } - - // This should be invoked with the other CompileAndCall() methods, as - // we are unable to generate the arguments. - // Currently there are two cases: - // internal/per_context/*: the arguments are generated in - // InitializePrimordials() - // all the other cases: the arguments are generated in the JS-land loader. - UNREACHABLE(); + return value; + } + + Local data; + if (!LookupAndCompile(context, builtin_source, realm).ToLocal(&data)) { + return MaybeLocal(); + } + CHECK(data->IsValue()); + Local value = data.As(); + CHECK(value->IsFunction()); + Local fn = value.As(); + Local receiver = Undefined(Isolate::GetCurrent()); + + switch (builtin_source->type) { + case BuiltinSourceType::kBootstrapRealm: { + Local get_linked_binding; + Local get_internal_binding; + if (!NewFunctionTemplate(isolate, binding::GetLinkedBinding) + ->GetFunction(context) + .ToLocal(&get_linked_binding) || + !NewFunctionTemplate(isolate, binding::GetInternalBinding) + ->GetFunction(context) + .ToLocal(&get_internal_binding)) { + return MaybeLocal(); + } + Local arguments[] = {realm->process_object(), + get_linked_binding, + get_internal_binding, + realm->primordials()}; + return fn->Call(context, receiver, 4, arguments); + } + case BuiltinSourceType::kMainScript: + case BuiltinSourceType::kBootstrapScript: { + Local arguments[] = {realm->process_object(), + realm->builtin_module_require(), + realm->internal_binding_loader(), + realm->primordials()}; + return fn->Call(context, receiver, 4, arguments); + } + case BuiltinSourceType::kFunction: // Handled in JS land. + case BuiltinSourceType::kPerContextScript: // Handled by + // InitializePrimordials + default: + UNREACHABLE(); + } } -MaybeLocal BuiltinLoader::CompileAndCall(Local context, - const char* id, - int argc, - Local argv[], - Realm* optional_realm) { +MaybeLocal BuiltinLoader::CompileAndCallWith(Local context, + const char* id, + int argc, + Local argv[], + Realm* optional_realm) { // Arguments must match the parameters specified in // BuiltinLoader::LookupAndCompile(). - MaybeLocal maybe_fn = LookupAndCompile(context, id, optional_realm); + MaybeLocal maybe_fn = + LookupAndCompileFunction(context, id, optional_realm); Local fn; if (!maybe_fn.ToLocal(&fn)) { return MaybeLocal(); @@ -511,21 +548,12 @@ MaybeLocal BuiltinLoader::CompileAndCall(Local context, return fn->Call(context, undefined, argc, argv); } -MaybeLocal BuiltinLoader::LookupAndCompile( - Local context, - const char* id, - LocalVector* parameters, - Realm* optional_realm) { - return LookupAndCompileInternal(context, id, parameters, optional_realm); -} - bool BuiltinLoader::CompileAllBuiltinsAndCopyCodeCache( Local context, const std::vector& eager_builtins, std::vector* out) { auto ids = GetBuiltinIds(); bool all_succeeded = true; - constexpr std::string_view v8_tools_prefix = "internal/deps/v8/tools/"; constexpr std::string_view primordial_prefix = "internal/per_context/"; constexpr std::string_view bootstrap_prefix = "internal/bootstrap/"; constexpr std::string_view main_prefix = "internal/main/"; @@ -533,11 +561,6 @@ bool BuiltinLoader::CompileAllBuiltinsAndCopyCodeCache( std::unordered_set(eager_builtins.begin(), eager_builtins.end()); for (const auto& id : ids) { - if (id.starts_with(v8_tools_prefix)) { - // No need to generate code cache for v8 scripts. - continue; - } - // Eagerly compile primordials/boostrap/main scripts during code cache // generation. if (id.starts_with(primordial_prefix) || id.starts_with(bootstrap_prefix) || @@ -546,7 +569,7 @@ bool BuiltinLoader::CompileAllBuiltinsAndCopyCodeCache( } TryCatch bootstrapCatch(Isolate::GetCurrent()); - auto fn = LookupAndCompile(context, id.data(), nullptr); + auto data = LookupAndCompile(context, id.data(), nullptr); if (bootstrapCatch.HasCaught()) { per_process::Debug(DebugCategory::CODE_CACHE, "Failed to compile code cache for %s\n", @@ -556,7 +579,7 @@ bool BuiltinLoader::CompileAllBuiltinsAndCopyCodeCache( } else { // This is used by the snapshot builder, so save the code cache // unconditionally. - SaveCodeCache(id.data(), fn.ToLocalChecked()); + SaveCodeCache(id, data.ToLocalChecked()); } } @@ -671,7 +694,7 @@ void BuiltinLoader::ConfigStringGetter( env->builtin_loader()->GetConfigString(info.GetIsolate())); } -void BuiltinLoader::RecordResult(const char* id, +void BuiltinLoader::RecordResult(const std::string& id, BuiltinLoader::Result result, Realm* realm) { if (result == BuiltinLoader::Result::kWithCache) { @@ -686,14 +709,124 @@ void BuiltinLoader::CompileFunction(const FunctionCallbackInfo& args) { CHECK(args[0]->IsString()); node::Utf8Value id_v(realm->isolate(), args[0].As()); const char* id = *id_v; - MaybeLocal maybe = realm->env()->builtin_loader()->LookupAndCompile( - realm->context(), id, realm); Local fn; - if (maybe.ToLocal(&fn)) { + if (realm->env() + ->builtin_loader() + ->LookupAndCompileFunction(realm->context(), id, realm) + .ToLocal(&fn)) { args.GetReturnValue().Set(fn); } } +std::string ResolveRequestForBuiltin(const std::string& specifier) { + // Currently this is only ever hit by V8 tools. Importing any other specifiers + // from a builtin would result in a module not found error later. + if (specifier[0] == '.' && specifier[1] == '/') { + // Currently the V8 tool sources are all flat. + // ./file.mjs -> internal/deps/v8/tools/file + DCHECK(specifier.ends_with(".mjs")); + return std::string("internal/deps/v8/tools/") + + specifier.substr(2, specifier.length() - 6); + } + return specifier; +} + +MaybeLocal BuiltinLoader::LoadBuiltinSourceTextModule(Realm* realm, + const char* id) { + Isolate* isolate = realm->isolate(); + if (module_cache_.find(id) != module_cache_.end()) { + return module_cache_[id].Get(isolate); + } + + Local context = realm->context(); + Local data; + + if (!LookupAndCompile(context, id, realm).ToLocal(&data)) { + return MaybeLocal(); + } + + CHECK(data->IsModule()); + Local mod = data.As(); + + auto pair = module_cache_.emplace(id, v8::Global(isolate, mod)); + CHECK(pair.second); + Local requests = mod->GetModuleRequests(); + + // Pre-fetch all dependencies. + if (requests->Length() > 0) { + for (int i = 0; i < requests->Length(); i++) { + Local req = requests->Get(context, i).As(); + std::string specifier = + Utf8Value(isolate, req->GetSpecifier()).ToString(); + std::string resolved_id = ResolveRequestForBuiltin(specifier); + if (LoadBuiltinSourceTextModule(realm, resolved_id.c_str()).IsEmpty()) { + return MaybeLocal(); + } + } + } + + return mod; +} + +MaybeLocal BuiltinLoader::ImportBuiltinSourceTextModule(Realm* realm, + const char* id) { + Isolate* isolate = realm->isolate(); + EscapableHandleScope scope(isolate); + Local mod; + if (!LoadBuiltinSourceTextModule(realm, id).ToLocal(&mod)) { + return MaybeLocal(); + } + Local context = realm->context(); + if (mod->InstantiateModule(context, ResolveModuleCallback).IsEmpty()) { + return MaybeLocal(); + } + Local promise; + if (!mod->Evaluate(context).ToLocal(&promise)) { + return MaybeLocal(); + } + + LocalVector names{isolate, + { + OneByteString(isolate, "promise"), + OneByteString(isolate, "namespace"), + }}; + LocalVector values{isolate, + { + promise, + mod->GetModuleNamespace(), + }}; + auto result = Object::New( + isolate, v8::Null(isolate), names.data(), values.data(), names.size()); + return scope.Escape(result); +} + +MaybeLocal BuiltinLoader::ResolveModuleCallback( + Local context, + Local specifier, + Local import_attributes, + Local referrer) { + Realm* realm = Realm::GetCurrent(context); + + Utf8Value specifier_v(realm->isolate(), specifier); + std::string resolved = ResolveRequestForBuiltin(specifier_v.ToString()); + return realm->env()->builtin_loader()->LoadBuiltinSourceTextModule( + realm, resolved.c_str()); +} + +void BuiltinLoader::ImportBuiltinSourceTextModule( + const FunctionCallbackInfo& args) { + Realm* realm = Realm::GetCurrent(args); + CHECK(args[0]->IsString()); + node::Utf8Value id(realm->isolate(), args[0].As()); + Local value; + if (realm->env() + ->builtin_loader() + ->ImportBuiltinSourceTextModule(realm, *id) + .ToLocal(&value)) { + args.GetReturnValue().Set(value); + } +} + void BuiltinLoader::HasCachedBuiltins(const FunctionCallbackInfo& args) { auto instance = Environment::GetCurrent(args)->builtin_loader(); RwLock::ScopedReadLock lock(instance->code_cache_->mutex); @@ -754,6 +887,10 @@ void BuiltinLoader::CreatePerIsolateProperties(IsolateData* isolate_data, SetMethod(isolate, target, "compileFunction", BuiltinLoader::CompileFunction); SetMethod(isolate, target, "hasCachedBuiltins", HasCachedBuiltins); SetMethod(isolate, target, "setInternalLoaders", SetInternalLoaders); + SetMethod(isolate, + target, + "importBuiltinSourceTextModule", + ImportBuiltinSourceTextModule); } void BuiltinLoader::CreatePerContextProperties(Local target, @@ -774,6 +911,7 @@ void BuiltinLoader::RegisterExternalReferences( registry->Register(HasCachedBuiltins); registry->Register(SetInternalLoaders); registry->Register(GetNatives); + registry->Register(ImportBuiltinSourceTextModule); RegisterExternalReferencesForInternalizedBuiltinCode(registry); } diff --git a/src/node_builtins.h b/src/node_builtins.h index 7a7b84337feb67..e4af1f42f4442b 100644 --- a/src/node_builtins.h +++ b/src/node_builtins.h @@ -11,6 +11,7 @@ #include #include #include +#include "builtin_info.h" #include "node_external_reference.h" #include "node_mutex.h" #include "node_threadsafe_cow.h" @@ -68,7 +69,13 @@ struct CodeCacheInfo { BuiltinCodeCacheData data; }; -using BuiltinSourceMap = std::map; +struct BuiltinSource { + std::string id; + UnionBytes source; + BuiltinSourceType type; +}; + +using BuiltinSourceMap = std::map; using BuiltinCodeCacheMap = std::unordered_map; @@ -94,21 +101,14 @@ class NODE_EXTERN_PRIVATE BuiltinLoader { // The parameters used to compile the scripts are detected based on // the pattern of the id. - v8::MaybeLocal LookupAndCompile(v8::Local context, - const char* id, - Realm* optional_realm); + v8::MaybeLocal LookupAndCompileFunction( + v8::Local context, const char* id, Realm* optional_realm); - v8::MaybeLocal LookupAndCompile( - v8::Local context, - const char* id, - v8::LocalVector* parameters, - Realm* optional_realm); - - v8::MaybeLocal CompileAndCall(v8::Local context, - const char* id, - int argc, - v8::Local argv[], - Realm* optional_realm); + v8::MaybeLocal CompileAndCallWith(v8::Local context, + const char* id, + int argc, + v8::Local argv[], + Realm* optional_realm); v8::MaybeLocal CompileAndCall(v8::Local context, const char* id, @@ -117,7 +117,9 @@ class NODE_EXTERN_PRIVATE BuiltinLoader { // Returns config.gypi as a JSON string v8::Local GetConfigString(v8::Isolate* isolate); bool Exists(const char* id); - bool Add(const char* id, const UnionBytes& source); + const BuiltinSource* AddFromDisk(const char* id, + const std::string& filename, + const UnionBytes& source); bool CompileAllBuiltinsAndCopyCodeCache( v8::Local context, @@ -152,18 +154,18 @@ class NODE_EXTERN_PRIVATE BuiltinLoader { const v8::ScriptCompiler::CachedData* GetCodeCache(const char* id) const; enum class Result { kWithCache, kWithoutCache }; - v8::MaybeLocal LoadBuiltinSource(v8::Isolate* isolate, - const char* id) const; + const BuiltinSource* LoadBuiltinSource(v8::Isolate* isolate, const char* id); // If an exception is encountered (e.g. source code contains // syntax error), the returned value is empty. - v8::MaybeLocal LookupAndCompileInternal( - v8::Local context, - const char* id, - v8::LocalVector* parameters, - Realm* optional_realm); - void SaveCodeCache(const char* id, v8::Local fn); - - static void RecordResult(const char* id, + v8::MaybeLocal LookupAndCompile(v8::Local context, + const char* id, + Realm* optional_realm); + v8::MaybeLocal LookupAndCompile(v8::Local context, + const BuiltinSource* builtin_source, + Realm* optional_realm); + void SaveCodeCache(const std::string& id, v8::Local data); + + static void RecordResult(const std::string& id, BuiltinLoader::Result result, Realm* realm); static void GetBuiltinCategories( @@ -180,13 +182,26 @@ class NODE_EXTERN_PRIVATE BuiltinLoader { const v8::PropertyCallbackInfo& info); // Compile a specific built-in as a function static void CompileFunction(const v8::FunctionCallbackInfo& args); + v8::MaybeLocal LoadBuiltinSourceTextModule(Realm* realm, + const char* id); + v8::MaybeLocal ImportBuiltinSourceTextModule(Realm* realm, + const char* id); + static void ImportBuiltinSourceTextModule( + const v8::FunctionCallbackInfo& args); + + static v8::MaybeLocal ResolveModuleCallback( + v8::Local context, + v8::Local specifier, + v8::Local import_attributes, + v8::Local referrer); static void HasCachedBuiltins( const v8::FunctionCallbackInfo& args); // For legacy process.binding('natives') static void GetNatives(v8::Local property, const v8::PropertyCallbackInfo& info); - void AddExternalizedBuiltin(const char* id, const char* filename); + const BuiltinSource* AddExternalizedBuiltin(const char* id, + const char* filename); ThreadsafeCopyOnWrite source_; @@ -211,6 +226,7 @@ class NODE_EXTERN_PRIVATE BuiltinLoader { }; std::shared_ptr code_cache_; + std::unordered_map> module_cache_; friend class ::PerProcessTest; }; diff --git a/test/README.md b/test/README.md index a79c97e941d542..02359ca6826501 100644 --- a/test/README.md +++ b/test/README.md @@ -46,7 +46,7 @@ For the tests to run on Windows, be sure to clone Node.js source code with the [^3]: All tests inside of this directory are expected to fail. If a test doesn't fail on certain platforms, those should be skipped via `known_issues.status`. -[^4]: The tests are for the logic in `lib/internal/v8_prof_processor.js` and `lib/internal/v8_prof_polyfill.js`. +[^4]: The tests are for the logic in `lib/internal/main/prof_process.js` and `lib/internal/v8_prof_polyfill.js`. The tests confirm that the profile processor packages the correct set of scripts from V8 and introduces the correct platform specific logic. diff --git a/test/cctest/test_per_process.cc b/test/cctest/test_per_process.cc index 34cf163add7904..7a6f53d56222c1 100644 --- a/test/cctest/test_per_process.cc +++ b/test/cctest/test_per_process.cc @@ -21,11 +21,11 @@ namespace { TEST_F(PerProcessTest, EmbeddedSources) { const auto& sources = PerProcessTest::get_sources_for_test(); ASSERT_TRUE(std::any_of(sources.cbegin(), sources.cend(), [](auto p) { - return p.second.is_one_byte(); + return p.second.source.is_one_byte(); })) << "BuiltinLoader::source_ should have some 8bit items"; ASSERT_TRUE(std::any_of(sources.cbegin(), sources.cend(), [](auto p) { - return !p.second.is_one_byte(); + return !p.second.source.is_one_byte(); })) << "BuiltinLoader::source_ should have some 16bit items"; } diff --git a/test/parallel/test-tick-processor-arguments.js b/test/parallel/test-tick-processor-arguments.js index 1448ea46dbaeb4..406b13b676d79c 100644 --- a/test/parallel/test-tick-processor-arguments.js +++ b/test/parallel/test-tick-processor-arguments.js @@ -3,7 +3,7 @@ const common = require('../common'); const tmpdir = require('../common/tmpdir'); const fs = require('fs'); const assert = require('assert'); -const { spawnSync } = require('child_process'); +const { spawnSyncAndAssert, spawnSyncAndExitWithoutError } = require('../common/child_process'); if (!common.enoughTestMem) common.skip('skipped due to memory requirements'); @@ -13,7 +13,7 @@ if (common.isAIX) tmpdir.refresh(); // Generate log file. -spawnSync(process.execPath, [ '--prof', '-p', '42' ], { cwd: tmpdir.path }); +spawnSyncAndExitWithoutError(process.execPath, [ '--prof', '-p', '42' ], { cwd: tmpdir.path }); const files = fs.readdirSync(tmpdir.path); const logfile = files.find((name) => /\.log$/.test(name)); @@ -23,10 +23,14 @@ assert(logfile); // as an example flag listed in deps/v8/tools/tickprocessor.js. // Any of the other flags there should work for this test too, if --preprocess // is ever removed. -const { stdout } = spawnSync( +spawnSyncAndAssert( process.execPath, [ '--prof-process', '--preprocess', logfile ], - { cwd: tmpdir.path, encoding: 'utf8', maxBuffer: Infinity }); - -// Make sure that the result is valid JSON. -JSON.parse(stdout); + { cwd: tmpdir.path, encoding: 'utf8', maxBuffer: Infinity }, + { + stdout(output) { + // Make sure that the result is valid JSON. + JSON.parse(output); + } + } +); diff --git a/tools/js2c.cc b/tools/js2c.cc index 21992cbe894a88..8b0b76643d7a7d 100644 --- a/tools/js2c.cc +++ b/tools/js2c.cc @@ -9,6 +9,7 @@ #include #include #include +#include "builtin_info.h" #include "embedded_data.h" #include "executable_wrapper.h" #include "simdutf.h" @@ -687,15 +688,26 @@ int AddModule(const std::string& filename, std::string var = GetVariableName(file_id); definitions->emplace_back(GetDefinition(var, code)); - + std::string source_type = builtins::GetBuiltinSourceTypeName( + builtins::GetBuiltinSourceType(file_id, filename)); // Initializers of the BuiltinSourceMap: - // {"fs", UnionBytes{&fs_resource}}, - Fragment& init_buf = initializers->emplace_back(Fragment(256, 0)); + // {"fs", + // BuiltinSource{UnionBytes(&fs_resource), BuiltinSourceType::kFunction}}, + // {"internal/deps/v8/tools/tickprocessor-driver", + // BuiltinSource{UnionBytes(&fs_resource), + // BuiltinSourceType::kSourceTextModule}}, + Fragment& init_buf = initializers->emplace_back(Fragment(512, 0)); int init_size = snprintf(init_buf.data(), init_buf.size(), - " {\"%s\", UnionBytes(&%s_resource) },", + " {\"%s\"," + " BuiltinSource{" + " \"%s\"," + " UnionBytes(&%s_resource)," + " BuiltinSourceType::%s} },", + file_id.c_str(), file_id.c_str(), - var.c_str()); + var.c_str(), + source_type.c_str()); init_buf.resize(init_size); // Registrations: diff --git a/unofficial.gni b/unofficial.gni index c742b62c484e9d..f654a15da347a3 100644 --- a/unofficial.gni +++ b/unofficial.gni @@ -323,6 +323,8 @@ template("node_gn_build") { "tools/executable_wrapper.h", "src/embedded_data.cc", "src/embedded_data.h", + "src/builtin_info.cc", + "src/builtin_info.h", ] include_dirs = [ "src" ] }