Skip to content

Commit 7db8fc7

Browse files
jasnelljasnell
authored
Merge pull request #5088 from cloudflare/jasnell/start-consolidation-of-module-code
2 parents 05c42b8 + 0186e1f commit 7db8fc7

3 files changed

Lines changed: 120 additions & 92 deletions

File tree

‎src/workerd/io/worker-modules.h‎

Lines changed: 99 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
#pragma once
22

3+
#include <workerd/api/commonjs.h>
34
#include <workerd/api/modules.h>
45
#include <workerd/io/io-context.h>
56
#include <workerd/io/worker.h>
@@ -8,16 +9,17 @@
89

910
#include <pyodide/python-entrypoint.embed.h>
1011

12+
#include <capnp/blob.h>
1113
#include <capnp/schema-loader.h>
1214
#include <capnp/schema.h>
1315

1416
// This header provides utilities for setting up the ModuleRegistry for a worker.
1517
// It is meant to be included in only two places; workerd-api.c++ and the equivalent
1618
// file in the internal repo. It is templated on the TypeWrapper and JsgIsolate types.
19+
1720
namespace workerd {
1821
namespace api {
1922
class ServiceWorkerGlobalScope;
20-
class CommonJsModuleContext;
2123
} // namespace api
2224

2325
WD_STRONG_BOOL(IsPythonWorker);
@@ -260,4 +262,100 @@ static kj::Arc<jsg::modules::ModuleRegistry> newWorkerModuleRegistry(
260262
return builder.finish();
261263
}
262264

265+
// ======================================================================================
266+
// Legacy module registry support
267+
268+
namespace modules::legacy {
269+
270+
template <typename JsgIsolate>
271+
static v8::Local<v8::String> compileTextGlobal(
272+
typename JsgIsolate::Lock& lock, ::capnp::Text::Reader reader) {
273+
return lock.wrapNoContext(reader);
274+
};
275+
276+
template <typename JsgIsolate>
277+
static v8::Local<v8::ArrayBuffer> compileDataGlobal(
278+
typename JsgIsolate::Lock& lock, ::capnp::Data::Reader reader) {
279+
return lock.wrapNoContext(kj::heapArray(reader));
280+
};
281+
282+
template <typename JsgIsolate>
283+
static v8::Local<v8::WasmModuleObject> compileWasmGlobal(typename JsgIsolate::Lock& lock,
284+
::capnp::Data::Reader reader,
285+
const jsg::CompilationObserver& observer) {
286+
lock.setAllowEval(true);
287+
KJ_DEFER(lock.setAllowEval(false));
288+
289+
// Allow Wasm compilation to spawn a background thread for tier-up, i.e. recompiling
290+
// Wasm with optimizations in the background. Otherwise Wasm startup is way too slow.
291+
// Until tier-up finishes, requests will be handled using Liftoff-generated code, which
292+
// compiles fast but runs slower.
293+
AllowV8BackgroundThreadsScope scope;
294+
295+
return jsg::compileWasmModule(lock, reader, observer);
296+
};
297+
298+
template <typename JsgIsolate>
299+
static v8::Local<v8::Value> compileJsonGlobal(
300+
typename JsgIsolate::Lock& lock, ::capnp::Text::Reader reader) {
301+
return jsg::check(v8::JSON::Parse(lock.v8Context(), lock.wrapNoContext(reader)));
302+
};
303+
304+
// Compiles a module for the legacy module registry, returning kj::none if the module
305+
// is a Python module or Python requirement, which are handled elsewhere.
306+
template <typename JsgIsolate>
307+
kj::Maybe<jsg::ModuleRegistry::ModuleInfo> tryCompileLegacyModule(jsg::Lock& js,
308+
kj::StringPtr name,
309+
const Worker::Script::ModuleContent& moduleContent,
310+
const jsg::CompilationObserver& observer,
311+
CompatibilityFlags::Reader featureFlags) {
312+
auto& lock = kj::downcast<typename JsgIsolate::Lock>(js);
313+
KJ_SWITCH_ONEOF(moduleContent) {
314+
KJ_CASE_ONEOF(content, Worker::Script::TextModule) {
315+
return jsg::ModuleRegistry::ModuleInfo(js, name, kj::none,
316+
jsg::ModuleRegistry::TextModuleInfo(
317+
js, modules::legacy::compileTextGlobal<JsgIsolate>(lock, content.body)));
318+
}
319+
KJ_CASE_ONEOF(content, Worker::Script::DataModule) {
320+
return jsg::ModuleRegistry::ModuleInfo(js, name, kj::none,
321+
jsg::ModuleRegistry::DataModuleInfo(
322+
js, modules::legacy::compileDataGlobal<JsgIsolate>(lock, content.body)));
323+
}
324+
KJ_CASE_ONEOF(content, Worker::Script::WasmModule) {
325+
return jsg::ModuleRegistry::ModuleInfo(js, name, kj::none,
326+
jsg::ModuleRegistry::WasmModuleInfo(
327+
js, modules::legacy::compileWasmGlobal<JsgIsolate>(lock, content.body, observer)));
328+
}
329+
KJ_CASE_ONEOF(content, Worker::Script::JsonModule) {
330+
return jsg::ModuleRegistry::ModuleInfo(js, name, kj::none,
331+
jsg::ModuleRegistry::JsonModuleInfo(
332+
js, modules::legacy::compileJsonGlobal<JsgIsolate>(lock, content.body)));
333+
}
334+
KJ_CASE_ONEOF(content, Worker::Script::EsModule) {
335+
// TODO(soon): Make sure passing nullptr to compile cache is desired.
336+
return jsg::ModuleRegistry::ModuleInfo(js, name, content.body, nullptr /* compile cache */,
337+
jsg::ModuleInfoCompileOption::BUNDLE, observer);
338+
}
339+
KJ_CASE_ONEOF(content, Worker::Script::CommonJsModule) {
340+
return jsg::ModuleRegistry::ModuleInfo(js, name, content.namedExports,
341+
jsg::ModuleRegistry::CommonJsModuleInfo(lock, name, content.body,
342+
kj::heap<api::CommonJsImpl<typename JsgIsolate::Lock>>(js, kj::Path::parse(name))));
343+
}
344+
KJ_CASE_ONEOF(content, Worker::Script::PythonModule) {
345+
// Nothing to do. Handled elsewhere.
346+
return kj::none;
347+
}
348+
KJ_CASE_ONEOF(content, Worker::Script::PythonRequirement) {
349+
// Nothing to do. Handled elsewhere.
350+
return kj::none;
351+
}
352+
KJ_CASE_ONEOF(content, Worker::Script::CapnpModule) {
353+
return workerd::modules::capnp::addCapnpModule<JsgIsolate>(lock, content.typeId, name);
354+
}
355+
}
356+
KJ_UNREACHABLE;
357+
}
358+
359+
} // namespace modules::legacy
360+
263361
} // namespace workerd

‎src/workerd/server/workerd-api.c++‎

Lines changed: 20 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -405,36 +405,6 @@ struct WorkerdApi::Impl final {
405405
}
406406
});
407407
}
408-
409-
static v8::Local<v8::String> compileTextGlobal(
410-
JsgWorkerdIsolate::Lock& lock, capnp::Text::Reader reader) {
411-
return lock.wrapNoContext(reader);
412-
};
413-
414-
static v8::Local<v8::ArrayBuffer> compileDataGlobal(
415-
JsgWorkerdIsolate::Lock& lock, capnp::Data::Reader reader) {
416-
return lock.wrapNoContext(kj::heapArray(reader));
417-
};
418-
419-
static v8::Local<v8::WasmModuleObject> compileWasmGlobal(JsgWorkerdIsolate::Lock& lock,
420-
capnp::Data::Reader reader,
421-
const jsg::CompilationObserver& observer) {
422-
lock.setAllowEval(true);
423-
KJ_DEFER(lock.setAllowEval(false));
424-
425-
// Allow Wasm compilation to spawn a background thread for tier-up, i.e. recompiling
426-
// Wasm with optimizations in the background. Otherwise Wasm startup is way too slow.
427-
// Until tier-up finishes, requests will be handled using Liftoff-generated code, which
428-
// compiles fast but runs slower.
429-
AllowV8BackgroundThreadsScope scope;
430-
431-
return jsg::compileWasmModule(lock, reader, observer);
432-
};
433-
434-
static v8::Local<v8::Value> compileJsonGlobal(
435-
JsgWorkerdIsolate::Lock& lock, capnp::Text::Reader reader) {
436-
return jsg::check(v8::JSON::Parse(lock.v8Context(), lock.wrapNoContext(reader)));
437-
};
438408
};
439409

440410
WorkerdApi::WorkerdApi(jsg::V8System& v8System,
@@ -583,7 +553,8 @@ kj::Array<Worker::Script::CompiledGlobal> WorkerdApi::compileServiceWorkerGlobal
583553
for (auto& global: source.globals) {
584554
KJ_IF_SOME(wasm, global.content.tryGet<Worker::Script::WasmModule>()) {
585555
auto name = lock.str(global.name);
586-
auto value = Impl::compileWasmGlobal(lock, wasm.body, *impl->observer);
556+
auto value =
557+
modules::legacy::compileWasmGlobal<JsgWorkerdIsolate>(lock, wasm.body, *impl->observer);
587558

588559
compiledGlobals.add(Worker::Script::CompiledGlobal{
589560
{lock.v8Isolate, name},
@@ -597,64 +568,24 @@ kj::Array<Worker::Script::CompiledGlobal> WorkerdApi::compileServiceWorkerGlobal
597568
return compiledGlobals.finish();
598569
}
599570

600-
// Part of the original module registry implementation.
601-
kj::Maybe<jsg::ModuleRegistry::ModuleInfo> WorkerdApi::tryCompileModule(jsg::Lock& js,
602-
config::Worker::Module::Reader conf,
603-
jsg::CompilationObserver& observer,
571+
namespace {
572+
kj::Maybe<jsg::ModuleRegistry::ModuleInfo> tryCompileLegacyModule(jsg::Lock& js,
573+
kj::StringPtr name,
574+
const Worker::Script::ModuleContent& content,
575+
const jsg::CompilationObserver& observer,
604576
CompatibilityFlags::Reader featureFlags) {
605-
return tryCompileModule(js, readModuleConf(conf, featureFlags), observer, featureFlags);
577+
return modules::legacy::tryCompileLegacyModule<JsgWorkerdIsolate>(
578+
js, name, content, observer, featureFlags);
606579
}
580+
} // namespace
607581

582+
// Part of the original module registry implementation.
608583
kj::Maybe<jsg::ModuleRegistry::ModuleInfo> WorkerdApi::tryCompileModule(jsg::Lock& js,
609-
const Worker::Script::Module& module,
610-
jsg::CompilationObserver& observer,
584+
config::Worker::Module::Reader conf,
585+
const jsg::CompilationObserver& observer,
611586
CompatibilityFlags::Reader featureFlags) {
612-
TRACE_EVENT("workerd", "WorkerdApi::tryCompileModule()", "name", module.name);
613-
auto& lock = kj::downcast<JsgWorkerdIsolate::Lock>(js);
614-
KJ_SWITCH_ONEOF(module.content) {
615-
KJ_CASE_ONEOF(content, Worker::Script::TextModule) {
616-
return jsg::ModuleRegistry::ModuleInfo(lock, module.name, kj::none,
617-
jsg::ModuleRegistry::TextModuleInfo(lock, Impl::compileTextGlobal(lock, content.body)));
618-
}
619-
KJ_CASE_ONEOF(content, Worker::Script::DataModule) {
620-
return jsg::ModuleRegistry::ModuleInfo(lock, module.name, kj::none,
621-
jsg::ModuleRegistry::DataModuleInfo(
622-
lock, Impl::compileDataGlobal(lock, content.body).As<v8::ArrayBuffer>()));
623-
}
624-
KJ_CASE_ONEOF(content, Worker::Script::WasmModule) {
625-
return jsg::ModuleRegistry::ModuleInfo(lock, module.name, kj::none,
626-
jsg::ModuleRegistry::WasmModuleInfo(
627-
lock, Impl::compileWasmGlobal(lock, content.body, observer)));
628-
}
629-
KJ_CASE_ONEOF(content, Worker::Script::JsonModule) {
630-
return jsg::ModuleRegistry::ModuleInfo(lock, module.name, kj::none,
631-
jsg::ModuleRegistry::JsonModuleInfo(lock, Impl::compileJsonGlobal(lock, content.body)));
632-
}
633-
KJ_CASE_ONEOF(content, Worker::Script::EsModule) {
634-
// TODO(soon): Make sure passing nullptr to compile cache is desired.
635-
return jsg::ModuleRegistry::ModuleInfo(lock, module.name, content.body,
636-
nullptr /* compile cache */, jsg::ModuleInfoCompileOption::BUNDLE, observer);
637-
}
638-
KJ_CASE_ONEOF(content, Worker::Script::CommonJsModule) {
639-
return jsg::ModuleRegistry::ModuleInfo(lock, module.name, content.namedExports,
640-
jsg::ModuleRegistry::CommonJsModuleInfo(lock, module.name, content.body,
641-
kj::heap<api::CommonJsImpl<JsgWorkerdIsolate::Lock>>(
642-
lock, kj::Path::parse(module.name))));
643-
}
644-
KJ_CASE_ONEOF(content, Worker::Script::PythonModule) {
645-
// Nothing to do. Handled in compileModules.
646-
return kj::none;
647-
}
648-
KJ_CASE_ONEOF(content, Worker::Script::PythonRequirement) {
649-
// Nothing to do. Handled in compileModules.
650-
return kj::none;
651-
}
652-
KJ_CASE_ONEOF(content, Worker::Script::CapnpModule) {
653-
return workerd::modules::capnp::addCapnpModule<JsgWorkerdIsolate>(
654-
lock, content.typeId, module.name);
655-
}
656-
}
657-
KJ_UNREACHABLE;
587+
auto module = readModuleConf(conf, featureFlags);
588+
return tryCompileLegacyModule(js, module.name, module.content, observer, featureFlags);
658589
}
659590

660591
Worker::Script::Module WorkerdApi::readModuleConf(config::Worker::Module::Reader conf,
@@ -756,7 +687,9 @@ void WorkerdApi::compileModules(jsg::Lock& lockParam,
756687
Worker::Script::Module module{
757688
.name = source.mainModule, .content = Worker::Script::EsModule{PYTHON_ENTRYPOINT}};
758689

759-
auto info = tryCompileModule(lockParam, module, modules->getObserver(), featureFlags);
690+
auto info = tryCompileLegacyModule(
691+
lockParam, module.name, module.content, modules->getObserver(), featureFlags);
692+
760693
auto path = kj::Path::parse(source.mainModule);
761694
modules->add(path, kj::mv(KJ_REQUIRE_NONNULL(info)));
762695
}
@@ -792,7 +725,8 @@ void WorkerdApi::compileModules(jsg::Lock& lockParam,
792725

793726
for (auto& module: source.modules) {
794727
auto path = kj::Path::parse(module.name);
795-
auto maybeInfo = tryCompileModule(lockParam, module, modules->getObserver(), featureFlags);
728+
auto maybeInfo = tryCompileLegacyModule(
729+
lockParam, module.name, module.content, modules->getObserver(), featureFlags);
796730
KJ_IF_SOME(info, maybeInfo) {
797731
modules->add(path, kj::mv(info));
798732
}

‎src/workerd/server/workerd-api.h‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -320,11 +320,7 @@ class WorkerdApi final: public Worker::Api {
320320
// Part of the original module registry API.
321321
static kj::Maybe<jsg::ModuleRegistry::ModuleInfo> tryCompileModule(jsg::Lock& js,
322322
config::Worker::Module::Reader conf,
323-
jsg::CompilationObserver& observer,
324-
CompatibilityFlags::Reader featureFlags);
325-
static kj::Maybe<jsg::ModuleRegistry::ModuleInfo> tryCompileModule(jsg::Lock& js,
326-
const Worker::Script::Module& module,
327-
jsg::CompilationObserver& observer,
323+
const jsg::CompilationObserver& observer,
328324
CompatibilityFlags::Reader featureFlags);
329325

330326
// Convert a module definition from workerd config to a Worker::Script::Module (which may contain

0 commit comments

Comments
 (0)