Add an embed_module!() macro to boa_interop (#3784)

* Add an embed_module!() macro to boa_interop

The macro creates a ModuleLoader that includes all JS files from a directory.

* Add description of function

* Run prettier

* Remove reference to a unaccessible crate

* Remove one more reference to a unaccessible crate

* Disable test that plays with paths on Windows

* Block the whole test module instead of just the fn

* This is a bit insane

* Replace path separators into JavaScript specifier separators

* cargo fmt

* cargo fmt part deux

* fix some issues with relative path and pathing on windows

* fix module resolver when there are no base path

* use the platform's path separator

* cargo fmt

* prettier

* Remove caching of the error

* Pedantic clippy gonna pedant

core/engine/src/module/loader.rs

@@ -48,7 +48,7 @@ pub fn resolve_module_specifier(
     referrer: Option<&Path>,
     _context: &mut Context,
 ) -> JsResult<PathBuf> {
-    let base = base.map_or_else(|| PathBuf::from(""), PathBuf::from);
+    let base_path = base.map_or_else(|| PathBuf::from(""), PathBuf::from);
     let referrer_dir = referrer.and_then(|p| p.parent());
 
     let specifier = specifier.to_std_string_escaped();
@@ -65,17 +65,17 @@ pub fn resolve_module_specifier(
 
     let long_path = if is_relative {
         if let Some(r_path) = referrer_dir {
-            base.join(r_path).join(short_path)
+            base_path.join(r_path).join(short_path)
         } else {
             return Err(JsError::from_opaque(
                 js_string!("relative path without referrer").into(),
             ));
         }
     } else {
-        base.join(&specifier)
+        base_path.join(&specifier)
     };
 
-    if long_path.is_relative() {
+    if long_path.is_relative() && base.is_some() {
         return Err(JsError::from_opaque(
             js_string!("resolved path is relative").into(),
         ));
@@ -100,7 +100,7 @@ pub fn resolve_module_specifier(
             Ok(acc)
         })?;
 
-    if path.starts_with(&base) {
+    if path.starts_with(&base_path) {
         Ok(path)
     } else {
         Err(JsError::from_opaque(
@@ -371,6 +371,39 @@ mod tests {
         assert_eq!(actual.map_err(|_| ()), expected.map(PathBuf::from));
     }
 
+    // This tests the same cases as the previous test, but without a base path.
+    #[rustfmt::skip]
+    #[cfg(target_family = "unix")]
+    #[test_case(Some("hello/ref.js"),       "a.js",             Ok("a.js"))]
+    #[test_case(Some("base/ref.js"),        "./b.js",           Ok("base/b.js"))]
+    #[test_case(Some("base/other/ref.js"),  "./c.js",           Ok("base/other/c.js"))]
+    #[test_case(Some("base/other/ref.js"),  "../d.js",          Ok("base/d.js"))]
+    #[test_case(Some("base/ref.js"),        "e.js",             Ok("e.js"))]
+    #[test_case(Some("base/ref.js"),        "./f.js",           Ok("base/f.js"))]
+    #[test_case(Some("./ref.js"),           "./g.js",           Ok("g.js"))]
+    #[test_case(Some("./other/ref.js"),     "./other/h.js",     Ok("other/other/h.js"))]
+    #[test_case(Some("./other/ref.js"),     "./other/../h1.js", Ok("other/h1.js"))]
+    #[test_case(Some("./other/ref.js"),     "./../h2.js",       Ok("h2.js"))]
+    #[test_case(None,                       "./i.js",           Err(()))]
+    #[test_case(None,                       "j.js",             Ok("j.js"))]
+    #[test_case(None,                       "other/k.js",       Ok("other/k.js"))]
+    #[test_case(None,                       "other/../../l.js", Err(()))]
+    #[test_case(Some("/base/ref.js"),       "other/../../m.js", Err(()))]
+    #[test_case(None,                       "../n.js",          Err(()))]
+    fn resolve_test_no_base(ref_path: Option<&str>, spec: &str, expected: Result<&str, ()>) {
+        let mut context = Context::default();
+        let spec = js_string!(spec);
+        let ref_path = ref_path.map(PathBuf::from);
+
+        let actual = resolve_module_specifier(
+            None,
+            &spec,
+            ref_path.as_deref(),
+            &mut context,
+        );
+        assert_eq!(actual.map_err(|_| ()), expected.map(PathBuf::from));
+    }
+
     #[rustfmt::skip]
     #[cfg(target_family = "windows")]
     #[test_case(Some("a:\\hello\\ref.js"),       "a.js",                Ok("a:\\base\\a.js"))]

core/interop/Cargo.toml

@@ -13,6 +13,7 @@ rust-version.workspace = true
 [dependencies]
 boa_engine.workspace = true
 boa_gc.workspace = true
+boa_macros.workspace = true
 rustc-hash = { workspace = true, features = ["std"] }
 
 [lints]

core/interop/src/lib.rs

@@ -3,6 +3,7 @@
 use boa_engine::module::SyntheticModuleInitializer;
 use boa_engine::value::TryFromJs;
 use boa_engine::{Context, JsResult, JsString, JsValue, Module, NativeFunction};
+pub use boa_macros;
 
 pub mod loaders;
 

core/interop/src/loaders.rs

@@ -3,4 +3,5 @@
 
 pub use hashmap::HashMapModuleLoader;
 
+pub mod embedded;
 pub mod hashmap;

core/interop/src/loaders/embedded.rs

@@ -0,0 +1,139 @@
+//! Embedded module loader. Creates a `ModuleLoader` instance that contains
+//! files embedded in the binary at build time.
+
+use std::cell::RefCell;
+use std::collections::HashMap;
+use std::path::Path;
+
+use boa_engine::module::{ModuleLoader, Referrer};
+use boa_engine::{Context, JsNativeError, JsResult, JsString, Module, Source};
+
+/// Create a module loader that embeds files from the filesystem at build
+/// time. This is useful for bundling assets with the binary.
+///
+/// By default, will error if the total file size exceeds 1MB. This can be
+/// changed by specifying the `max_size` parameter.
+///
+/// The embedded module will only contain files that have the `.js`, `.mjs`,
+/// or `.cjs` extension.
+#[macro_export]
+macro_rules! embed_module {
+    ($path: literal, max_size = $max_size: literal) => {
+        $crate::loaders::embedded::EmbeddedModuleLoader::from_iter(
+            $crate::boa_macros::embed_module_inner!($path, $max_size),
+        )
+    };
+    ($path: literal) => {
+        embed_module!($path, max_size = 1_048_576)
+    };
+}
+
+#[derive(Debug, Clone)]
+enum EmbeddedModuleEntry {
+    Source(JsString, &'static [u8]),
+    Module(Module),
+}
+
+impl EmbeddedModuleEntry {
+    fn from_source(path: JsString, source: &'static [u8]) -> Self {
+        Self::Source(path, source)
+    }
+
+    fn cache(&mut self, context: &mut Context) -> JsResult<&Module> {
+        if let Self::Source(path, source) = self {
+            let mut bytes: &[u8] = source;
+            let path = path.to_std_string_escaped();
+            let source = Source::from_reader(&mut bytes, Some(Path::new(&path)));
+            match Module::parse(source, None, context) {
+                Ok(module) => {
+                    *self = Self::Module(module);
+                }
+                Err(err) => {
+                    return Err(err);
+                }
+            }
+        };
+
+        match self {
+            Self::Module(module) => Ok(module),
+            EmbeddedModuleEntry::Source(_, _) => unreachable!(),
+        }
+    }
+
+    fn as_module(&self) -> Option<&Module> {
+        match self {
+            Self::Module(module) => Some(module),
+            Self::Source(_, _) => None,
+        }
+    }
+}
+
+/// The resulting type of creating an embedded module loader.
+#[derive(Debug, Clone)]
+#[allow(clippy::module_name_repetitions)]
+pub struct EmbeddedModuleLoader {
+    map: HashMap<JsString, RefCell<EmbeddedModuleEntry>>,
+}
+
+impl FromIterator<(&'static str, &'static [u8])> for EmbeddedModuleLoader {
+    fn from_iter<T: IntoIterator<Item = (&'static str, &'static [u8])>>(iter: T) -> Self {
+        Self {
+            map: iter
+                .into_iter()
+                .map(|(path, source)| {
+                    let p = JsString::from(path);
+                    (
+                        p.clone(),
+                        RefCell::new(EmbeddedModuleEntry::from_source(p, source)),
+                    )
+                })
+                .collect(),
+        }
+    }
+}
+
+impl ModuleLoader for EmbeddedModuleLoader {
+    fn load_imported_module(
+        &self,
+        referrer: Referrer,
+        specifier: JsString,
+        finish_load: Box<dyn FnOnce(JsResult<Module>, &mut Context)>,
+        context: &mut Context,
+    ) {
+        let Ok(specifier_path) = boa_engine::module::resolve_module_specifier(
+            None,
+            &specifier,
+            referrer.path(),
+            context,
+        ) else {
+            let err = JsNativeError::typ().with_message(format!(
+                "could not resolve module specifier `{}`",
+                specifier.to_std_string_escaped()
+            ));
+            finish_load(Err(err.into()), context);
+            return;
+        };
+
+        if let Some(module) = self
+            .map
+            .get(&JsString::from(specifier_path.to_string_lossy().as_ref()))
+        {
+            let mut embedded = module.borrow_mut();
+            let module = embedded.cache(context);
+
+            finish_load(module.cloned(), context);
+        } else {
+            let err = JsNativeError::typ().with_message(format!(
+                "could not find module `{}`",
+                specifier.to_std_string_escaped()
+            ));
+            finish_load(Err(err.into()), context);
+        }
+    }
+
+    fn get_module(&self, specifier: JsString) -> Option<Module> {
+        self.map
+            .get(&specifier)
+            .and_then(|module| module.borrow().as_module().cloned())
+    }
+}

core/interop/tests/embedded.rs

@@ -0,0 +1,67 @@
+#![allow(unused_crate_dependencies)]
+
+use std::rc::Rc;
+
+use boa_engine::builtins::promise::PromiseState;
+use boa_engine::module::ModuleLoader;
+use boa_engine::{js_string, Context, JsString, JsValue, Module, Source};
+use boa_interop::embed_module;
+
+#[test]
+fn simple() {
+    #[cfg(target_family = "unix")]
+    let module_loader = Rc::new(embed_module!("tests/embedded/"));
+    #[cfg(target_family = "windows")]
+    let module_loader = Rc::new(embed_module!("tests\\embedded\\"));
+
+    let mut context = Context::builder()
+        .module_loader(module_loader.clone())
+        .build()
+        .unwrap();
+
+    // Resolving modules that exist but haven't been cached yet should return None.
+    assert_eq!(module_loader.get_module(JsString::from("/file1.js")), None);
+    assert_eq!(
+        module_loader.get_module(JsString::from("/non-existent.js")),
+        None
+    );
+
+    let module = Module::parse(
+        Source::from_bytes(b"export { bar } from '/file1.js';"),
+        None,
+        &mut context,
+    )
+    .expect("failed to parse module");
+    let promise = module.load_link_evaluate(&mut context);
+    context.run_jobs();
+
+    match promise.state() {
+        PromiseState::Fulfilled(value) => {
+            assert!(
+                value.is_undefined(),
+                "Expected undefined, got {}",
+                value.display()
+            );
+
+            let bar = module
+                .namespace(&mut context)
+                .get(js_string!("bar"), &mut context)
+                .unwrap()
+                .as_callable()
+                .cloned()
+                .unwrap();
+            let value = bar.call(&JsValue::undefined(), &[], &mut context).unwrap();
+            assert_eq!(
+                value.as_number(),
+                Some(6.),
+                "Expected 6, got {}",
+                value.display()
+            );
+        }
+        PromiseState::Rejected(err) => panic!(
+            "promise was not fulfilled: {:?}",
+            err.to_string(&mut context)
+        ),
+        PromiseState::Pending => panic!("Promise was not settled"),
+    }
+}

core/interop/tests/embedded/dir1/file3.js

@@ -0,0 +1,2 @@
+// Enable this when https://github.com/boa-dev/boa/pull/3781 is fixed and merged.
+export { foo } from "./file4.js";

core/interop/tests/embedded/dir1/file4.js

@@ -0,0 +1,3 @@
+export function foo() {
+  return 3;
+}

core/interop/tests/embedded/file1.js

@@ -0,0 +1,6 @@
+import { foo } from "./file2.js";
+import { foo as foo2 } from "./dir1/file3.js";
+
+export function bar() {
+  return foo() + foo2() + 1;
+}

core/interop/tests/embedded/file2.js

@@ -0,0 +1,3 @@
+export function foo() {
+  return 2;
+}