From 1b0a3c265926b960bbebc62e726ec8e171c9e220 Mon Sep 17 00:00:00 2001 From: Joel Dice Date: Mon, 8 Jun 2026 17:09:32 -0600 Subject: [PATCH] add `--intersect-world` option This allows app developers to e.g. use a third-party library which makes use of a number of imports but specify that the application will only use a subset of those imports. That can be useful when targetting an environment which does not support all the imports the library would otherwise pull in. --- Cargo.lock | 86 ++++++++++---------------------- src/command.rs | 26 ++++++++++ src/lib.rs | 78 ++++++++++++++++++++++++++--- src/python.rs | 4 +- src/test.rs | 12 +++-- src/test/echoes.rs | 1 + src/test/foo_sdk/wit/world.wit | 14 ++++++ src/test/tests.rs | 90 ++++++++++++++++++++++++++++++++-- test-generator/src/lib.rs | 1 + 9 files changed, 236 insertions(+), 76 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d566c54..dd05b9a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,22 +2,13 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "addr2line" -version = "0.25.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" -dependencies = [ - "gimli 0.32.3", -] - [[package]] name = "addr2line" version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59317f77929f0e679d39364702289274de2f0f0b22cbf50b2b8cff2169a0b27a" dependencies = [ - "gimli 0.33.1", + "gimli", ] [[package]] @@ -123,12 +114,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.101" +version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f0e0fee31ef5ed1ba1316088939cea399010ed7731dba877ed44aeb407a75ea" -dependencies = [ - "backtrace", -] +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" [[package]] name = "arbitrary" @@ -198,21 +186,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" -[[package]] -name = "backtrace" -version = "0.3.76" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" -dependencies = [ - "addr2line 0.25.1", - "cfg-if", - "libc", - "miniz_oxide", - "object 0.37.3", - "rustc-demangle", - "windows-link", -] - [[package]] name = "base64" version = "0.22.1" @@ -637,7 +610,7 @@ dependencies = [ "cranelift-control", "cranelift-entity", "cranelift-isle", - "gimli 0.33.1", + "gimli", "hashbrown 0.16.1", "libm", "log", @@ -1125,12 +1098,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "gimli" -version = "0.32.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" - [[package]] name = "gimli" version = "0.33.1" @@ -1164,6 +1131,12 @@ dependencies = [ "serde_core", ] +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + [[package]] name = "heck" version = "0.5.0" @@ -1436,12 +1409,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.13.0" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "serde", "serde_core", ] @@ -1718,15 +1691,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "object" -version = "0.37.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" -dependencies = [ - "memchr", -] - [[package]] name = "object" version = "0.38.1" @@ -3246,7 +3210,7 @@ version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54fa9f298901a64ed3eae16b130f0b30c80dbb74a9e7f129a791f4e74649b917" dependencies = [ - "addr2line 0.26.1", + "addr2line", "async-trait", "bitflags", "bumpalo", @@ -3256,13 +3220,13 @@ dependencies = [ "encoding_rs", "futures", "fxprof-processed-profile", - "gimli 0.33.1", + "gimli", "ittapi", "libc", "log", "mach2", "memfd", - "object 0.38.1", + "object", "once_cell", "postcard", "pulley-interpreter", @@ -3305,11 +3269,11 @@ dependencies = [ "cranelift-bforest", "cranelift-bitset", "cranelift-entity", - "gimli 0.33.1", + "gimli", "hashbrown 0.16.1", "indexmap", "log", - "object 0.38.1", + "object", "postcard", "rustc-demangle", "semver", @@ -3390,10 +3354,10 @@ dependencies = [ "cranelift-entity", "cranelift-frontend", "cranelift-native", - "gimli 0.33.1", + "gimli", "itertools", "log", - "object 0.38.1", + "object", "pulley-interpreter", "smallvec", "target-lexicon", @@ -3427,7 +3391,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fe23134536b9883ffc2afcffae23f7ffbcb1791e2d9fac6d6464a37ea4c8fdd" dependencies = [ "cc", - "object 0.38.1", + "object", "rustix 1.1.3", "wasmtime-internal-versioned-export-macros", ] @@ -3453,7 +3417,7 @@ dependencies = [ "cfg-if", "cranelift-codegen", "log", - "object 0.38.1", + "object", "wasmtime-environ", ] @@ -3475,9 +3439,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d556c3b176aba3cce565b2bafcdc049e7410ac1d86bf1ef663a035d9ded0dddc" dependencies = [ "cranelift-codegen", - "gimli 0.33.1", + "gimli", "log", - "object 0.38.1", + "object", "target-lexicon", "wasmparser 0.245.1", "wasmtime-environ", @@ -3680,7 +3644,7 @@ checksum = "1ca3d76763e4ddc48ede73792d067396ba5ee74c3c581db90e6638fe6b46bf52" dependencies = [ "cranelift-assembler-x64", "cranelift-codegen", - "gimli 0.33.1", + "gimli", "regalloc2", "smallvec", "target-lexicon", diff --git a/src/command.rs b/src/command.rs index d206abe..be886b6 100644 --- a/src/command.rs +++ b/src/command.rs @@ -151,6 +151,29 @@ pub struct Componentize { #[arg(short = 'm', long, value_parser = parse_key_value)] pub module_worlds: Vec<(String, String)>, + /// Specify a world to *intersect* with the world(s) specified elsewhere. + /// + /// The `--world` option, `--module-worlds` option, and + /// `componentize-py.toml` files may be used to specify any number of worlds + /// for the component to target. If more than one are specified, they are + /// unioned together to compute the target world. This option extends that + /// computation to calculate the *intersection* (i.e. shared subset) of that + /// union and the specified world. + /// + /// This can be useful in cases where an application is using an SDK lbirary + /// containing a `componentize-py.toml` and pre-generated bindings, but the + /// target environment for the app only supports a subset of the features + /// covered by the SDK. By default, using that SDK would pull in all the + /// imports of the world(s) for which it was intended, but with this option, + /// we can exclude imports that are unsupported by the target environment + /// and still use the subset of the SDK which does not require those + /// excluded imports. + /// + /// Note that this may lead to pre-init or runtime errors if the application + /// does in fact end up using any imports which were excluded. + #[arg(short = 'i', long)] + pub intersect_world: Option, + /// Output file to which to write the resulting component #[arg(short = 'o', long, default_value = "index.wasm")] pub output: PathBuf, @@ -271,6 +294,7 @@ fn componentize(common: Common, componentize: Componentize) -> Result<()> { .map(|(a, b)| (a.as_str(), b.as_str())) .collect(), full_names: common.full_names, + intersect_world: componentize.intersect_world.as_deref(), } .generate(), )?; @@ -526,6 +550,7 @@ class Bindings(bindings.Bindings): module_worlds: vec![], output: out_dir.path().join("app.wasm"), stub_wasi: false, + intersect_world: None, }; componentize(common, componentize_opts) } @@ -639,6 +664,7 @@ world lib-world { module_worlds: vec![], output: dir.path().join("app.wasm"), stub_wasi: false, + intersect_world: None, }, ) } diff --git a/src/lib.rs b/src/lib.rs index f9315e4..c0b52fe 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,7 +10,7 @@ use { serde::Deserialize, std::{ borrow::Cow, - collections::HashMap, + collections::{HashMap, HashSet}, fs, io::Cursor, iter, @@ -280,6 +280,7 @@ pub struct ComponentGenerator<'a> { pub import_interface_names: &'a HashMap<&'a str, &'a str>, pub export_interface_names: &'a HashMap<&'a str, &'a str>, pub full_names: bool, + pub intersect_world: Option<&'a str>, } impl ComponentGenerator<'_> { @@ -443,6 +444,24 @@ impl ComponentGenerator<'_> { let worlds = select_worlds(&resolve, self.worlds, &packages)?; + let intersector = if let Some(world) = self.intersect_world { + let intersector = select_world(&resolve, Some(world), &packages)?; + + for &world in &worlds { + intersect_world(&mut resolve, intersector, world); + } + + for (_, worlds) in configs.values() { + for &world in worlds { + intersect_world(&mut resolve, intersector, world); + } + } + + Some(intersector) + } else { + None + }; + let mut all_worlds = worlds .iter() .copied() @@ -459,10 +478,13 @@ impl ComponentGenerator<'_> { if all_worlds.is_empty() { // No worlds specified; pick the default one, if available: - all_worlds.insert( - select_world(&resolve, None, &packages)?, - Naming::from_full(self.full_names), - ); + let world = select_world(&resolve, None, &packages)?; + + if let Some(intersector) = intersector { + intersect_world(&mut resolve, intersector, world); + } + + all_worlds.insert(world, Naming::from_full(self.full_names)); } // Now that we've parsed all known WIT files and resolved all relevant @@ -758,7 +780,10 @@ impl ComponentGenerator<'_> { for (mounts, world_dir) in world_dir_mounts.iter() { for mount in mounts { if DEBUG_PYTHON_BINDINGS { - eprintln!("world dir path: {}", world_dir.path().display()); + eprintln!( + "world dir path: {}; mount: {mount}", + world_dir.path().display() + ); } wasi.preopened_dir(world_dir.path(), mount, DirPerms::all(), FilePerms::all())?; } @@ -970,6 +995,47 @@ fn union_world( Ok(union_world) } +fn intersect_world(resolve: &mut Resolve, intersector: WorldId, world: WorldId) { + let (imports_to_remove, exports_to_remove) = { + let intersector = &resolve.worlds[intersector]; + let world = &resolve.worlds[world]; + + // Note that we make no effort to check whether the world items + // corresponding to the same keys found in `intersector` and `world` + // actually match. We only care about the keys in `intersector` and ignore + // the items they map to. + + ( + world + .imports + .keys() + .filter_map(|name| { + if intersector.imports.contains_key(name) { + None + } else { + Some(name.clone()) + } + }) + .collect::>(), + world + .exports + .keys() + .filter_map(|name| { + if intersector.exports.contains_key(name) { + None + } else { + Some(name.clone()) + } + }) + .collect::>(), + ) + }; + + let world = &mut resolve.worlds[world]; + world.imports.retain(|k, _| !imports_to_remove.contains(k)); + world.exports.retain(|k, _| !exports_to_remove.contains(k)); +} + fn add_wasi_and_stubs( resolve: &Resolve, worlds: &IndexSet, diff --git a/src/python.rs b/src/python.rs index e7aeb5b..ba09685 100644 --- a/src/python.rs +++ b/src/python.rs @@ -18,7 +18,7 @@ use { #[allow(clippy::too_many_arguments)] #[pyo3::pyfunction] #[pyo3(name = "componentize")] -#[pyo3(signature = (wit_path, worlds, features, all_features, world_module, python_path, module_worlds, app_name, output_path, stub_wasi, import_interface_names, export_interface_names, full_names))] +#[pyo3(signature = (wit_path, worlds, features, all_features, world_module, python_path, module_worlds, app_name, output_path, stub_wasi, import_interface_names, export_interface_names, full_names, intersect_world))] fn python_componentize( wit_path: Vec, worlds: Vec, @@ -33,6 +33,7 @@ fn python_componentize( import_interface_names: Vec<(PyBackedStr, PyBackedStr)>, export_interface_names: Vec<(PyBackedStr, PyBackedStr)>, full_names: bool, + intersect_world: Option<&str>, ) -> PyResult<()> { (|| { Runtime::new()?.block_on( @@ -63,6 +64,7 @@ fn python_componentize( .map(|(a, b)| (a.as_ref(), b.as_ref())) .collect(), full_names, + intersect_world, } .generate(), ) diff --git a/src/test.rs b/src/test.rs index b61c92c..9e6f8e7 100644 --- a/src/test.rs +++ b/src/test.rs @@ -42,7 +42,8 @@ static ENGINE: Lazy = Lazy::new(|| { Engine::new(&config).unwrap() }); -#[allow(clippy::type_complexity)] +#[expect(clippy::type_complexity)] +#[expect(clippy::too_many_arguments)] async fn make_component( wit: &str, worlds: &[&str], @@ -50,6 +51,7 @@ async fn make_component( guest_code: &[(&str, &str)], python_path: &[&str], module_worlds: &[(&str, &[&str])], + intersect_world: Option<&str>, add_to_linker: Option<&dyn Fn(&mut Linker) -> Result<()>>, ) -> Result> { let tempdir = tempfile::tempdir()?; @@ -82,6 +84,7 @@ async fn make_component( import_interface_names: &HashMap::new(), export_interface_names: &HashMap::new(), full_names: false, + intersect_world, } .generate() .await?; @@ -122,6 +125,7 @@ struct Tester { } impl Tester { + #[expect(clippy::too_many_arguments)] fn new( wit: &str, worlds: &[&str], @@ -129,6 +133,7 @@ impl Tester { guest_code: &[(&str, &str)], python_path: &[&str], module_worlds: &[(&str, &[&str])], + intersect_world: Option<&str>, seed: [u8; 32], ) -> Result { // TODO: create two versions of the component -- one with and one @@ -143,6 +148,7 @@ impl Tester { guest_code, python_path, module_worlds, + intersect_world, Some(&H::add_to_linker), ))?; let mut linker = Linker::::new(&ENGINE); @@ -191,9 +197,7 @@ impl Tester { ) }); - let world = runtime - .block_on(H1::instantiate_pre(&mut store, self.pre.clone())) - .unwrap(); + let world = runtime.block_on(H1::instantiate_pre(&mut store, self.pre.clone()))?; test(&world, &mut store, &runtime) } diff --git a/src/test/echoes.rs b/src/test/echoes.rs index 01a9bbd..c7095f5 100644 --- a/src/test/echoes.rs +++ b/src/test/echoes.rs @@ -326,6 +326,7 @@ static TESTER: Lazy> = Lazy::new(|| { GUEST_CODE, &[], &[], + None, *SEED, ) .unwrap() diff --git a/src/test/foo_sdk/wit/world.wit b/src/test/foo_sdk/wit/world.wit index 01aac48..d1e899f 100644 --- a/src/test/foo_sdk/wit/world.wit +++ b/src/test/foo_sdk/wit/world.wit @@ -4,7 +4,21 @@ interface foo-interface { test: func(s: string) -> string; } +interface foo-interface2 { + test: func(s: string) -> string; +} + world foo-world { import foo-interface; export foo-interface; } + +world foo-world2 { + import foo-interface2; + export foo-interface; +} + +world foo-world-union { + include foo-world; + include foo-world2; +} diff --git a/src/test/tests.rs b/src/test/tests.rs index 0927651..729182c 100644 --- a/src/test/tests.rs +++ b/src/test/tests.rs @@ -48,7 +48,7 @@ wasmtime::component::bindgen!({ mod foo_sdk { wasmtime::component::bindgen!({ path: "src/test/foo_sdk/wit", - world: "foo-world", + world: "foo-world-union", imports: { default: trappable }, exports: { default: async }, }); @@ -120,7 +120,7 @@ impl super::Host for Host { fn add_to_linker(linker: &mut Linker) -> Result<()> { wasmtime_wasi::p2::add_to_linker_async(linker)?; Tests::add_to_linker::<_, HasSelf<_>>(linker, |ctx| ctx)?; - foo_sdk::FooWorld::add_to_linker::<_, HasSelf<_>>(linker, |ctx| ctx)?; + foo_sdk::FooWorldUnion::add_to_linker::<_, HasSelf<_>>(linker, |ctx| ctx)?; Ok(()) } @@ -132,14 +132,14 @@ impl super::Host for Host { struct FooHost; impl super::Host for FooHost { - type World = foo_sdk::FooWorld; + type World = foo_sdk::FooWorldUnion; fn add_to_linker(_linker: &mut Linker) -> Result<()> { unreachable!() } async fn instantiate_pre(store: &mut Store, pre: InstancePre) -> Result { - Ok(foo_sdk::FooWorldPre::new(pre)? + Ok(foo_sdk::FooWorldUnionPre::new(pre)? .instantiate_async(store) .await?) } @@ -172,6 +172,7 @@ static TESTER: Lazy> = Lazy::new(|| { ("foo_sdk", &["foo:sdk/foo-world"]), ("bar_sdk", &["bar:sdk/bar-world"]), ], + None, *SEED, ) .unwrap() @@ -880,6 +881,87 @@ fn multiworld() -> Result<()> { }) } +#[test] +fn multiworld_intersect() -> Result<()> { + impl foo_sdk::foo::sdk::foo_interface2::Host for Ctx { + fn test(&mut self, s: String) -> wasmtime::Result { + Ok(format!("{s} HostFoo2::test")) + } + } + + let tester = Tester::::new( + "package dummy:dummy;", + &[], + None, + &[( + "app.py", + r#" +from foo_sdk.wit import exports as foo_exports +from foo_sdk.wit.imports.foo_interface2 import test as foo_test2 +try: + from foo_sdk.wit.imports.foo_interface import test as foo_test + raise AssertionError +except ModuleNotFoundError: + pass +try: + from bar_sdk.wit.imports.bar_interface import test as bar_test + raise AssertionError +except ModuleNotFoundError: + pass + +class FooInterface(foo_exports.FooInterface): + def test(self, s: str) -> str: + return foo_test2(f"{s} FooInterface.test") + +class BarSdkBarInterface: + def test(self, s: str) -> str: + raise AssertionError +"#, + )], + &["src/test"], + &[ + ("foo_sdk", &["foo:sdk/foo-world-union"]), + ("bar_sdk", &["bar:sdk/bar-world"]), + ], + Some("foo:sdk/foo-world2"), + *SEED, + )?; + + // This should work since `export foo:sdk/foo-interface` is included in the intersection. + tester.test_with::(|world, store, runtime| { + runtime.block_on(async { + let result = world + .foo_sdk_foo_interface() + .call_test(store, "Howdy") + .await?; + + assert_eq!("Howdy FooInterface.test HostFoo2::test", result); + + Ok(()) + }) + })?; + + // This should _not_ work since `export bar:sdk/bar-interface` is excluded from the intersection. + let result = tester.test_with::(|world, store, runtime| { + runtime.block_on(async { + let result = world + .bar_sdk_bar_interface() + .call_test(store, "Howdy") + .await?; + + assert_eq!("Howdy BarInterface.test HostFoo::test", result); + + Ok(()) + }) + }); + + assert!(matches!(result, Err(error) if format!("{error}").contains( + "no exported instance named `bar:sdk/bar-interface`" + ))); + + Ok(()) +} + #[test] fn filesystem() -> Result<()> { let filename = "foo.txt"; diff --git a/test-generator/src/lib.rs b/test-generator/src/lib.rs index a60423f..b58a191 100644 --- a/test-generator/src/lib.rs +++ b/test-generator/src/lib.rs @@ -875,6 +875,7 @@ static TESTER: Lazy> = Lazy::new(|| {{ GUEST_CODE, &[], &[], + None, *SEED ).unwrap() }});