From 42f9d823cc28f3d6d9fa5b5de5dd45d0098b5b0a Mon Sep 17 00:00:00 2001 From: itowlson Date: Tue, 23 Jul 2024 11:18:26 +1200 Subject: [PATCH 01/11] Validate against target environments during build Signed-off-by: itowlson --- Cargo.lock | 373 +++++++++- Cargo.toml | 1 + crates/build/Cargo.toml | 1 + crates/build/src/deployment.rs | 23 + crates/build/src/lib.rs | 37 +- crates/build/src/manifest.rs | 79 +- crates/compose/Cargo.toml | 2 + crates/compose/src/lib.rs | 89 ++- crates/environments/Cargo.toml | 31 + .../src/environment_definition.rs | 38 + crates/environments/src/lib.rs | 225 ++++++ crates/environments/src/loader.rs | 179 +++++ crates/loader/src/lib.rs | 2 + crates/loader/src/local.rs | 390 +++++----- crates/manifest/src/compat.rs | 1 + crates/manifest/src/schema/v2.rs | 3 + crates/trigger/src/loader.rs | 16 +- examples/spin-timer/Cargo.lock | 679 +++++++++--------- 18 files changed, 1586 insertions(+), 583 deletions(-) create mode 100644 crates/build/src/deployment.rs create mode 100644 crates/environments/Cargo.toml create mode 100644 crates/environments/src/environment_definition.rs create mode 100644 crates/environments/src/lib.rs create mode 100644 crates/environments/src/loader.rs diff --git a/Cargo.lock b/Cargo.lock index caaf4ad993..956ae9b81e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -474,9 +474,9 @@ checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" [[package]] name = "async-trait" -version = "0.1.82" +version = "0.1.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a27b8a3a6e1a44fa4c8baf1f653e4172e81486d4941f2237e20dc2d0cf4ddff1" +checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", @@ -3384,6 +3384,7 @@ dependencies = [ "tokio", "tokio-rustls 0.26.0", "tower-service", + "webpki-roots 0.26.6", ] [[package]] @@ -3430,9 +3431,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da62f120a8a37763efb0cf8fdf264b884c7b8b9ac8660b900c8661030c00e6ba" +checksum = "41296eb09f183ac68eec06e03cdbea2e759633d4067b2f6552fc2e009bcad08b" dependencies = [ "bytes", "futures-channel", @@ -3443,7 +3444,6 @@ dependencies = [ "pin-project-lite", "socket2 0.5.7", "tokio", - "tower 0.4.13", "tower-service", "tracing", ] @@ -3872,9 +3872,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.158" +version = "0.2.159" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" +checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" [[package]] name = "libflate" @@ -3920,7 +3920,7 @@ checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ "bitflags 2.6.0", "libc", - "redox_syscall 0.5.4", + "redox_syscall 0.5.5", ] [[package]] @@ -4301,6 +4301,7 @@ checksum = "4edc8853320c2a0dab800fbda86253c8938f6ea88510dc92c5f1ed20e794afc1" dependencies = [ "cfg-if", "miette-derive 7.2.0", + "serde 1.0.210", "thiserror", "unicode-width", ] @@ -5079,7 +5080,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.4", + "redox_syscall 0.5.5", "smallvec", "windows-targets 0.52.6", ] @@ -5470,7 +5471,7 @@ version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b" dependencies = [ - "toml_edit 0.22.21", + "toml_edit 0.22.22", ] [[package]] @@ -5669,6 +5670,54 @@ dependencies = [ "reborrow", ] +[[package]] +name = "quinn" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c7c5fdde3cdae7203427dc4f0a68fe0ed09833edc525a03456b153b79828684" +dependencies = [ + "bytes", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash 2.0.0", + "rustls 0.23.13", + "socket2 0.5.7", + "thiserror", + "tokio", + "tracing", +] + +[[package]] +name = "quinn-proto" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" +dependencies = [ + "bytes", + "rand 0.8.5", + "ring", + "rustc-hash 2.0.0", + "rustls 0.23.13", + "slab", + "thiserror", + "tinyvec", + "tracing", +] + +[[package]] +name = "quinn-udp" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fe68c2e9e1a1234e218683dbdf9f9dfcb094113c5ac2b938dfcb9bab4c4140b" +dependencies = [ + "libc", + "once_cell", + "socket2 0.5.7", + "tracing", + "windows-sys 0.59.0", +] + [[package]] name = "quote" version = "1.0.37" @@ -5873,9 +5922,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0884ad60e090bf1345b93da0a5de8923c93884cd03f40dfcfddd3b4bee661853" +checksum = "62871f2d65009c0256aed1b9cfeeb8ac272833c404e13d53d400cd0dad7a2ac0" dependencies = [ "bitflags 2.6.0", ] @@ -6023,7 +6072,10 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", + "quinn", + "rustls 0.23.13", "rustls-pemfile 2.1.3", + "rustls-pki-types", "serde 1.0.210", "serde_json", "serde_urlencoded", @@ -6031,6 +6083,7 @@ dependencies = [ "system-configuration 0.6.1", "tokio", "tokio-native-tls", + "tokio-rustls 0.26.0", "tokio-socks", "tokio-util", "tower-service", @@ -6039,6 +6092,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", + "webpki-roots 0.26.6", "windows-registry", ] @@ -6917,6 +6971,7 @@ dependencies = [ "futures", "serde 1.0.210", "spin-common", + "spin-environments", "spin-manifest", "subprocess", "terminal", @@ -6972,6 +7027,7 @@ dependencies = [ "spin-build", "spin-common", "spin-doctor", + "spin-environments", "spin-expressions", "spin-factor-outbound-networking", "spin-http", @@ -7051,9 +7107,11 @@ dependencies = [ "indexmap 2.5.0", "semver", "spin-app", + "spin-common", "spin-componentize", "spin-serde", "thiserror", + "tokio", "wac-graph", ] @@ -7091,11 +7149,38 @@ dependencies = [ "terminal", "tokio", "toml 0.8.19", - "toml_edit 0.22.21", + "toml_edit 0.22.22", "tracing", "ui-testing", ] +[[package]] +name = "spin-environments" +version = "2.8.0-pre0" +dependencies = [ + "anyhow", + "async-trait", + "futures", + "indexmap 2.5.0", + "miette 7.2.0", + "oci-distribution 0.11.0 (git+https://github.com/fermyon/oci-distribution?rev=7e4ce9be9bcd22e78a28f06204931f10c44402ba)", + "semver", + "serde 1.0.210", + "serde_json", + "spin-common", + "spin-componentize", + "spin-compose", + "spin-loader", + "spin-manifest", + "spin-serde", + "tokio", + "tracing", + "wac-parser", + "wac-resolver", + "wac-types", + "wasm-pkg-loader", +] + [[package]] name = "spin-expressions" version = "2.8.0-pre0" @@ -7764,7 +7849,7 @@ dependencies = [ "tempfile", "tokio", "toml 0.8.19", - "toml_edit 0.22.21", + "toml_edit 0.22.22", "url", "walkdir", ] @@ -8537,7 +8622,7 @@ dependencies = [ "serde 1.0.210", "serde_spanned", "toml_datetime", - "toml_edit 0.22.21", + "toml_edit 0.22.22", ] [[package]] @@ -8562,15 +8647,15 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.21" +version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b072cee73c449a636ffd6f32bd8de3a9f7119139aff882f44943ce2986dc5cf" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ "indexmap 2.5.0", "serde 1.0.210", "serde_spanned", "toml_datetime", - "winnow 0.6.18", + "winnow 0.6.19", ] [[package]] @@ -9020,6 +9105,49 @@ dependencies = [ "wasmparser 0.202.0", ] +[[package]] +name = "wac-parser" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83eb40c28590dd36f789f49ac8c1118f00a975d54672c1abce6dc5f0f8783a03" +dependencies = [ + "anyhow", + "id-arena", + "indexmap 2.5.0", + "log", + "logos", + "miette 7.2.0", + "semver", + "serde 1.0.210", + "thiserror", + "wac-graph", + "wasm-encoder 0.202.0", + "wasm-metadata 0.202.0", + "wasmparser 0.202.0", +] + +[[package]] +name = "wac-resolver" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdb3a216294d59cd676f5928187f10aec5d7ff9a9e1e1a9a0ad0ac4835151b7" +dependencies = [ + "anyhow", + "futures", + "indexmap 2.5.0", + "log", + "miette 7.2.0", + "semver", + "thiserror", + "tokio", + "wac-parser", + "wac-types", + "warg-client 0.9.0", + "warg-crypto 0.9.0", + "warg-protocol 0.9.0", + "wit-component 0.202.0", +] + [[package]] name = "wac-types" version = "0.6.0" @@ -9070,8 +9198,23 @@ dependencies = [ "serde 1.0.210", "serde_with", "thiserror", - "warg-crypto", - "warg-protocol", + "warg-crypto 0.7.0", + "warg-protocol 0.7.0", +] + +[[package]] +name = "warg-api" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44b422328c3a86be288f569694aa97df958ade0cd9514ed00bc562952c6778e" +dependencies = [ + "indexmap 2.5.0", + "itertools 0.12.1", + "serde 1.0.210", + "serde_with", + "thiserror", + "warg-crypto 0.9.0", + "warg-protocol 0.9.0", ] [[package]] @@ -9109,10 +9252,56 @@ dependencies = [ "tracing", "url", "walkdir", - "warg-api", - "warg-crypto", - "warg-protocol", - "warg-transparency", + "warg-api 0.7.0", + "warg-crypto 0.7.0", + "warg-protocol 0.7.0", + "warg-transparency 0.7.0", + "wasm-compose", + "wasm-encoder 0.41.2", + "wasmparser 0.121.2", + "wasmprinter 0.2.80", + "windows-sys 0.52.0", +] + +[[package]] +name = "warg-client" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd1af3c0a73c56613c152fb99048af889a57063756299460c8fc26fea52ddccc" +dependencies = [ + "anyhow", + "async-recursion", + "async-trait", + "bytes", + "clap 4.5.18", + "dialoguer", + "dirs 5.0.1", + "futures-util", + "indexmap 2.5.0", + "itertools 0.12.1", + "keyring", + "libc", + "normpath", + "once_cell", + "pathdiff", + "ptree", + "reqwest 0.12.7", + "secrecy", + "semver", + "serde 1.0.210", + "serde_json", + "sha256", + "tempfile", + "thiserror", + "tokio", + "tokio-util", + "tracing", + "url", + "walkdir", + "warg-api 0.9.0", + "warg-crypto 0.9.0", + "warg-protocol 0.9.0", + "warg-transparency 0.9.0", "wasm-compose", "wasm-encoder 0.41.2", "wasmparser 0.121.2", @@ -9141,6 +9330,27 @@ dependencies = [ "thiserror", ] +[[package]] +name = "warg-crypto" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52fb6f3a64e3fef5425a0ab2b4354f1e49a699b76d58dd91d632483634f10474" +dependencies = [ + "anyhow", + "base64 0.21.7", + "digest", + "hex", + "leb128", + "once_cell", + "p256", + "rand_core 0.6.4", + "secrecy", + "serde 1.0.210", + "sha2", + "signature", + "thiserror", +] + [[package]] name = "warg-protobuf" version = "0.7.0" @@ -9157,7 +9367,26 @@ dependencies = [ "protox", "regex", "serde 1.0.210", - "warg-crypto", + "warg-crypto 0.7.0", +] + +[[package]] +name = "warg-protobuf" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bbb2d353497af3f6334bce11bfe69d638eedbb6d5059992acb2a05dd0beef5b" +dependencies = [ + "anyhow", + "pbjson", + "pbjson-build", + "pbjson-types", + "prost 0.12.6", + "prost-build", + "prost-types", + "protox", + "regex", + "serde 1.0.210", + "warg-crypto 0.9.0", ] [[package]] @@ -9177,9 +9406,32 @@ dependencies = [ "serde 1.0.210", "serde_with", "thiserror", - "warg-crypto", - "warg-protobuf", - "warg-transparency", + "warg-crypto 0.7.0", + "warg-protobuf 0.7.0", + "warg-transparency 0.7.0", + "wasmparser 0.121.2", +] + +[[package]] +name = "warg-protocol" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a710c66d8b5f2a7b046ecd2d45121530f561fd5c699b9c85ee49d332cfe773" +dependencies = [ + "anyhow", + "base64 0.21.7", + "hex", + "indexmap 2.5.0", + "pbjson-types", + "prost 0.12.6", + "prost-types", + "semver", + "serde 1.0.210", + "serde_with", + "thiserror", + "warg-crypto 0.9.0", + "warg-protobuf 0.9.0", + "warg-transparency 0.9.0", "wasmparser 0.121.2", ] @@ -9193,8 +9445,22 @@ dependencies = [ "indexmap 2.5.0", "prost 0.12.6", "thiserror", - "warg-crypto", - "warg-protobuf", + "warg-crypto 0.7.0", + "warg-protobuf 0.7.0", +] + +[[package]] +name = "warg-transparency" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b950a71a544b7ac8f5a5e95f43886ac97c3fe5c7080b955b1b534037596d7be" +dependencies = [ + "anyhow", + "indexmap 2.5.0", + "prost 0.12.6", + "thiserror", + "warg-crypto 0.9.0", + "warg-protobuf 0.9.0", ] [[package]] @@ -9433,8 +9699,8 @@ dependencies = [ "tracing", "tracing-subscriber", "url", - "warg-client", - "warg-protocol", + "warg-client 0.7.0", + "warg-protocol 0.7.0", "wasm-pkg-common", ] @@ -9997,7 +10263,7 @@ version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "372d5b87f58ec45c384ba03563b03544dc5fadc3983e434b286913f5b4a9bb6d" dependencies = [ - "redox_syscall 0.5.4", + "redox_syscall 0.5.5", "wasite", "web-sys", ] @@ -10371,9 +10637,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.6.18" +version = "0.6.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f" +checksum = "c52ac009d615e79296318c1bcce2d422aaca15ad08515e344feeda07df67a587" dependencies = [ "memchr", ] @@ -10404,6 +10670,25 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "wit-component" +version = "0.202.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c836b1fd9932de0431c1758d8be08212071b6bba0151f7bac826dbc4312a2a9" +dependencies = [ + "anyhow", + "bitflags 2.6.0", + "indexmap 2.5.0", + "log", + "serde 1.0.210", + "serde_derive", + "serde_json", + "wasm-encoder 0.202.0", + "wasm-metadata 0.202.0", + "wasmparser 0.202.0", + "wit-parser 0.202.0", +] + [[package]] name = "wit-component" version = "0.209.1" @@ -10442,6 +10727,24 @@ dependencies = [ "wit-parser 0.217.0", ] +[[package]] +name = "wit-parser" +version = "0.202.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "744237b488352f4f27bca05a10acb79474415951c450e52ebd0da784c1df2bcc" +dependencies = [ + "anyhow", + "id-arena", + "indexmap 2.5.0", + "log", + "semver", + "serde 1.0.210", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser 0.202.0", +] + [[package]] name = "wit-parser" version = "0.209.1" diff --git a/Cargo.toml b/Cargo.toml index f6b19eae5f..a131afa6f7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -59,6 +59,7 @@ spin-app = { path = "crates/app" } spin-build = { path = "crates/build" } spin-common = { path = "crates/common" } spin-doctor = { path = "crates/doctor" } +spin-environments = { path = "crates/environments" } spin-expressions = { path = "crates/expressions" } spin-factor-outbound-networking = { path = "crates/factor-outbound-networking" } spin-http = { path = "crates/http" } diff --git a/crates/build/Cargo.toml b/crates/build/Cargo.toml index 2bc9204228..662f0c5190 100644 --- a/crates/build/Cargo.toml +++ b/crates/build/Cargo.toml @@ -9,6 +9,7 @@ anyhow = { workspace = true } futures = { workspace = true } serde = { workspace = true } spin-common = { path = "../common" } +spin-environments = { path = "../environments" } spin-manifest = { path = "../manifest" } subprocess = "0.2" terminal = { path = "../terminal" } diff --git a/crates/build/src/deployment.rs b/crates/build/src/deployment.rs new file mode 100644 index 0000000000..bfd40bed5e --- /dev/null +++ b/crates/build/src/deployment.rs @@ -0,0 +1,23 @@ +#[derive(Default)] +pub struct DeploymentTargets { + target_environments: Vec, +} +pub type DeploymentTarget = String; + +impl DeploymentTargets { + pub fn new(envs: Vec) -> Self { + Self { + target_environments: envs, + } + } + + pub fn iter(&self) -> impl Iterator { + self.target_environments.iter().map(|s| s.as_str()) + } + + pub fn is_empty(&self) -> bool { + // TODO: it would be nice to let "no-op" behaviour fall out organically, + // but currently we do some stuff eagerly, so... + self.target_environments.is_empty() + } +} diff --git a/crates/build/src/lib.rs b/crates/build/src/lib.rs index 861cf12e8d..845bebb441 100644 --- a/crates/build/src/lib.rs +++ b/crates/build/src/lib.rs @@ -2,6 +2,7 @@ //! A library for building Spin components. +mod deployment; mod manifest; use anyhow::{anyhow, bail, Context, Result}; @@ -17,24 +18,38 @@ use crate::manifest::component_build_configs; /// If present, run the build command of each component. pub async fn build(manifest_file: &Path, component_ids: &[String]) -> Result<()> { - let (components, manifest_err) = - component_build_configs(manifest_file) - .await - .with_context(|| { - format!( - "Cannot read manifest file from {}", - quoted_path(manifest_file) - ) - })?; + let (components, deployment_targets, manifest) = component_build_configs(manifest_file) + .await + .with_context(|| { + format!( + "Cannot read manifest file from {}", + quoted_path(manifest_file) + ) + })?; let app_dir = parent_dir(manifest_file)?; let build_result = build_components(component_ids, components, app_dir); - if let Some(e) = manifest_err { + if let Err(e) = &manifest { terminal::warn!("The manifest has errors not related to the Wasm component build. Error details:\n{e:#}"); } - build_result + build_result?; + + if let Ok(manifest) = &manifest { + if !deployment_targets.is_empty() { + let resolution_context = + spin_environments::ResolutionContext::new(manifest_file.parent().unwrap()).await?; + spin_environments::validate_application_against_environment_ids( + deployment_targets.iter(), + manifest, + &resolution_context, + ) + .await?; + } + } + + Ok(()) } fn build_components( diff --git a/crates/build/src/manifest.rs b/crates/build/src/manifest.rs index 2fcd68dadb..bd17c2e250 100644 --- a/crates/build/src/manifest.rs +++ b/crates/build/src/manifest.rs @@ -4,37 +4,60 @@ use std::{collections::BTreeMap, path::Path}; use spin_manifest::{schema::v2, ManifestVersion}; +use crate::deployment::DeploymentTargets; + /// Returns a map of component IDs to [`v2::ComponentBuildConfig`]s for the /// given (v1 or v2) manifest path. If the manifest cannot be loaded, the /// function attempts fallback: if fallback succeeds, result is Ok but the load error /// is also returned via the second part of the return value tuple. pub async fn component_build_configs( manifest_file: impl AsRef, -) -> Result<(Vec, Option)> { +) -> Result<( + Vec, + DeploymentTargets, + Result, +)> { let manifest = spin_manifest::manifest_from_file(&manifest_file); match manifest { - Ok(manifest) => Ok((build_configs_from_manifest(manifest), None)), - Err(e) => fallback_load_build_configs(&manifest_file) - .await - .map(|bc| (bc, Some(e))), + Ok(mut manifest) => { + spin_manifest::normalize::normalize_manifest(&mut manifest); + let bc = build_configs_from_manifest(&manifest); + let dt = deployment_targets_from_manifest(&manifest); + Ok((bc, dt, Ok(manifest))) + } + Err(e) => { + let bc = fallback_load_build_configs(&manifest_file).await?; + let dt = fallback_load_deployment_targets(&manifest_file).await?; + Ok((bc, dt, Err(e))) + } } } fn build_configs_from_manifest( - mut manifest: spin_manifest::schema::v2::AppManifest, + manifest: &spin_manifest::schema::v2::AppManifest, ) -> Vec { - spin_manifest::normalize::normalize_manifest(&mut manifest); - manifest .components - .into_iter() + .iter() .map(|(id, c)| ComponentBuildInfo { id: id.to_string(), - build: c.build, + build: c.build.clone(), }) .collect() } +fn deployment_targets_from_manifest( + manifest: &spin_manifest::schema::v2::AppManifest, +) -> DeploymentTargets { + let target_environments = manifest.application.targets.clone(); + // let components = manifest + // .components + // .iter() + // .map(|(id, c)| (id.to_string(), c.source.clone())) + // .collect(); + DeploymentTargets::new(target_environments) +} + async fn fallback_load_build_configs( manifest_file: impl AsRef, ) -> Result> { @@ -57,6 +80,42 @@ async fn fallback_load_build_configs( }) } +async fn fallback_load_deployment_targets( + manifest_file: impl AsRef, +) -> Result { + // fn try_parse_component_source(c: (&String, &toml::Value)) -> Option<(String, spin_manifest::schema::v2::ComponentSource)> { + // let (id, ctab) = c; + // let cs = ctab.as_table() + // .and_then(|c| c.get("source")) + // .and_then(|cs| spin_manifest::schema::v2::ComponentSource::deserialize(cs.clone()).ok()); + // cs.map(|cs| (id.to_string(), cs)) + // } + let manifest_text = tokio::fs::read_to_string(manifest_file).await?; + Ok(match ManifestVersion::detect(&manifest_text)? { + ManifestVersion::V1 => Default::default(), + ManifestVersion::V2 => { + let table: toml::value::Table = toml::from_str(&manifest_text)?; + let target_environments = table + .get("application") + .and_then(|a| a.as_table()) + .and_then(|t| t.get("targets")) + .and_then(|arr| arr.as_array()) + .map(|v| v.as_slice()) + .unwrap_or_default() + .iter() + .filter_map(|t| t.as_str()) + .map(|s| s.to_owned()) + .collect(); + // let components = table + // .get("component") + // .and_then(|cs| cs.as_table()) + // .map(|table| table.iter().filter_map(try_parse_component_source).collect()) + // .unwrap_or_default(); + DeploymentTargets::new(target_environments) + } + }) +} + #[derive(Deserialize)] pub struct ComponentBuildInfo { #[serde(default)] diff --git a/crates/compose/Cargo.toml b/crates/compose/Cargo.toml index 38beda3e2d..fe803162a2 100644 --- a/crates/compose/Cargo.toml +++ b/crates/compose/Cargo.toml @@ -14,9 +14,11 @@ async-trait = { workspace = true } indexmap = "2" semver = "1" spin-app = { path = "../app" } +spin-common = { path = "../common" } spin-componentize = { workspace = true } spin-serde = { path = "../serde" } thiserror = { workspace = true } +tokio = { version = "1.23", features = ["fs"] } wac-graph = "0.6" [lints] diff --git a/crates/compose/src/lib.rs b/crates/compose/src/lib.rs index a9ee43fce7..efaa8163f1 100644 --- a/crates/compose/src/lib.rs +++ b/crates/compose/src/lib.rs @@ -1,7 +1,7 @@ use anyhow::Context; use indexmap::IndexMap; use semver::Version; -use spin_app::locked::{self, InheritConfiguration, LockedComponent, LockedComponentDependency}; +use spin_app::locked::InheritConfiguration as LockedInheritConfiguration; use spin_serde::{DependencyName, KebabId}; use std::collections::BTreeMap; use thiserror::Error; @@ -28,18 +28,68 @@ use wac_graph::{CompositionGraph, NodeId}; /// composition graph into a byte array and return it. pub async fn compose<'a, L: ComponentSourceLoader>( loader: &'a L, - component: &LockedComponent, + component: &L::Component, ) -> Result, ComposeError> { Composer::new(loader).compose(component).await } +#[async_trait::async_trait] +pub trait DependencyLike { + fn inherit(&self) -> InheritConfiguration; + fn export(&self) -> &Option; +} + +pub enum InheritConfiguration { + All, + Some(Vec), +} + +#[async_trait::async_trait] +pub trait ComponentLike { + type Dependency: DependencyLike; + + fn dependencies( + &self, + ) -> impl std::iter::ExactSizeIterator; + fn id(&self) -> &str; +} + +#[async_trait::async_trait] +impl ComponentLike for spin_app::locked::LockedComponent { + type Dependency = spin_app::locked::LockedComponentDependency; + + fn dependencies( + &self, + ) -> impl std::iter::ExactSizeIterator { + self.dependencies.iter() + } + + fn id(&self) -> &str { + &self.id + } +} + +#[async_trait::async_trait] +impl DependencyLike for spin_app::locked::LockedComponentDependency { + fn inherit(&self) -> InheritConfiguration { + match &self.inherit { + LockedInheritConfiguration::All => InheritConfiguration::All, + LockedInheritConfiguration::Some(cfgs) => InheritConfiguration::Some(cfgs.clone()), + } + } + + fn export(&self) -> &Option { + &self.export + } +} + /// This trait is used to load component source code from a locked component source across various embdeddings. #[async_trait::async_trait] pub trait ComponentSourceLoader { - async fn load_component_source( - &self, - source: &locked::LockedComponentSource, - ) -> anyhow::Result>; + type Component: ComponentLike; + type Dependency: DependencyLike; + async fn load_component_source(&self, source: &Self::Component) -> anyhow::Result>; + async fn load_dependency_source(&self, source: &Self::Dependency) -> anyhow::Result>; } /// Represents an error that can occur when composing dependencies. @@ -98,19 +148,19 @@ struct Composer<'a, L> { } impl<'a, L: ComponentSourceLoader> Composer<'a, L> { - async fn compose(mut self, component: &LockedComponent) -> Result, ComposeError> { + async fn compose(mut self, component: &L::Component) -> Result, ComposeError> { let source = self .loader - .load_component_source(&component.source) + .load_component_source(component) .await .map_err(ComposeError::PrepareError)?; - if component.dependencies.is_empty() { + if component.dependencies().len() == 0 { return Ok(source); } let (world_id, instantiation_id) = self - .register_package(&component.id, None, source) + .register_package(component.id(), None, source) .map_err(ComposeError::PrepareError)?; let prepared = self.prepare_dependencies(world_id, component).await?; @@ -150,7 +200,7 @@ impl<'a, L: ComponentSourceLoader> Composer<'a, L> { async fn prepare_dependencies( &mut self, world_id: WorldId, - component: &LockedComponent, + component: &L::Component, ) -> Result, ComposeError> { let imports = self.graph.types()[world_id].imports.clone(); @@ -158,7 +208,7 @@ impl<'a, L: ComponentSourceLoader> Composer<'a, L> { let mut mappings: BTreeMap> = BTreeMap::new(); - for (dependency_name, dependency) in &component.dependencies { + for (dependency_name, dependency) in component.dependencies() { let mut matched = Vec::new(); for import_name in &import_keys { @@ -171,7 +221,7 @@ impl<'a, L: ComponentSourceLoader> Composer<'a, L> { if matched.is_empty() { return Err(ComposeError::UnmatchedDependencyName { - component_id: component.id.clone(), + component_id: component.id().to_owned(), dependency_name: dependency_name.clone(), }); } @@ -195,7 +245,7 @@ impl<'a, L: ComponentSourceLoader> Composer<'a, L> { if !conflicts.is_empty() { return Err(ComposeError::DependencyConflicts { - component_id: component.id.clone(), + component_id: component.id().to_owned(), conflicts: conflicts .into_iter() .map(|(import_name, infos)| { @@ -300,19 +350,16 @@ impl<'a, L: ComponentSourceLoader> Composer<'a, L> { async fn register_dependency( &mut self, dependency_name: DependencyName, - dependency: &LockedComponentDependency, + dependency: &L::Dependency, ) -> anyhow::Result { - let mut dependency_source = self - .loader - .load_component_source(&dependency.source) - .await?; + let mut dependency_source = self.loader.load_dependency_source(dependency).await?; let package_name = match &dependency_name { DependencyName::Package(name) => name.package.to_string(), DependencyName::Plain(name) => name.to_string(), }; - match &dependency.inherit { + match dependency.inherit() { InheritConfiguration::Some(configurations) => { if configurations.is_empty() { // Configuration inheritance is disabled, apply deny_all adapter @@ -333,7 +380,7 @@ impl<'a, L: ComponentSourceLoader> Composer<'a, L> { manifest_name: dependency_name, instantiation_id, world_id, - export_name: dependency.export.clone(), + export_name: dependency.export().clone(), }) } diff --git a/crates/environments/Cargo.toml b/crates/environments/Cargo.toml new file mode 100644 index 0000000000..350599bda4 --- /dev/null +++ b/crates/environments/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "spin-environments" +version = { workspace = true } +authors = { workspace = true } +edition = { workspace = true } + +[dependencies] +anyhow = { workspace = true } +async-trait = "0.1" +futures = "0.3" +indexmap = "2.2.6" +miette = "7.2.0" +oci-distribution = { git = "https://github.com/fermyon/oci-distribution", rev = "7e4ce9be9bcd22e78a28f06204931f10c44402ba" } +semver = "1.0" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +spin-common = { path = "../common" } +spin-componentize = { path = "../componentize" } +spin-compose = { path = "../compose" } +spin-loader = { path = "../loader" } +spin-manifest = { path = "../manifest" } +spin-serde = { path = "../serde" } +tokio = { version = "1.23", features = ["fs"] } +tracing = { workspace = true } +wac-parser = "0.6.0" +wac-resolver = "0.6.0" +wac-types = "0.6.0" +wasm-pkg-loader = "0.4.1" + +[lints] +workspace = true diff --git a/crates/environments/src/environment_definition.rs b/crates/environments/src/environment_definition.rs new file mode 100644 index 0000000000..79813ecac3 --- /dev/null +++ b/crates/environments/src/environment_definition.rs @@ -0,0 +1,38 @@ +use wasm_pkg_loader::PackageRef; + +#[derive(Debug, serde::Deserialize)] +pub struct TargetEnvironment { + pub name: String, + pub environments: std::collections::HashMap, +} + +#[derive(Debug, Eq, Hash, PartialEq, serde::Deserialize)] +pub struct TargetWorld { + wit_package: PackageRef, + package_ver: String, // TODO: tidy to semver::Version + world_name: WorldNames, +} + +#[derive(Debug, Eq, Hash, PartialEq, serde::Deserialize)] +#[serde(untagged)] +enum WorldNames { + Exactly(String), + AnyOf(Vec), +} + +impl TargetWorld { + fn versioned_name(&self, world_name: &str) -> String { + format!("{}/{}@{}", self.wit_package, world_name, self.package_ver) + } + + pub fn versioned_names(&self) -> Vec { + match &self.world_name { + WorldNames::Exactly(name) => vec![self.versioned_name(name)], + WorldNames::AnyOf(names) => { + names.iter().map(|name| self.versioned_name(name)).collect() + } + } + } +} + +pub type TriggerType = String; diff --git a/crates/environments/src/lib.rs b/crates/environments/src/lib.rs new file mode 100644 index 0000000000..2578c17583 --- /dev/null +++ b/crates/environments/src/lib.rs @@ -0,0 +1,225 @@ +use anyhow::{anyhow, Context}; + +mod environment_definition; +mod loader; + +use environment_definition::{TargetEnvironment, TargetWorld, TriggerType}; +pub use loader::ResolutionContext; +use loader::{component_source, ComponentSourceLoader, ComponentToValidate}; + +pub async fn validate_application_against_environment_ids( + env_ids: impl Iterator, + app: &spin_manifest::schema::v2::AppManifest, + resolution_context: &ResolutionContext, +) -> anyhow::Result<()> { + let envs = futures::future::join_all(env_ids.map(resolve_environment_id)).await; + let envs: Vec<_> = envs.into_iter().collect::>()?; + validate_application_against_environments(&envs, app, resolution_context).await +} + +async fn resolve_environment_id(id: &str) -> anyhow::Result { + let (name, ver) = id.split_once('@').ok_or(anyhow!( + "Target environment '{id}' does not specify a version" + ))?; + let client = oci_distribution::Client::default(); + let auth = oci_distribution::secrets::RegistryAuth::Anonymous; + let env_def_ref = + oci_distribution::Reference::try_from(format!("ghcr.io/itowlson/spinenvs/{name}:{ver}"))?; + let (man, _digest) = client + .pull_manifest(&env_def_ref, &auth) + .await + .with_context(|| format!("Failed to find environment '{id}' in registry"))?; + let im = match man { + oci_distribution::manifest::OciManifest::Image(im) => im, + oci_distribution::manifest::OciManifest::ImageIndex(_ind) => { + anyhow::bail!("Environment '{id}' definition is unusable - stored in registry in incorrect format") + } + }; + let the_layer = &im.layers[0]; + let mut out = Vec::with_capacity(the_layer.size.try_into().unwrap_or_default()); + client + .pull_blob(&env_def_ref, the_layer, &mut out) + .await + .with_context(|| { + format!("Failed to download environment '{id}' definition from registry") + })?; + let te = serde_json::from_slice(&out).with_context(|| { + format!("Failed to load environment '{id}' definition - invalid JSON schema") + })?; + Ok(te) +} + +pub async fn validate_application_against_environments( + envs: &[TargetEnvironment], + app: &spin_manifest::schema::v2::AppManifest, + resolution_context: &ResolutionContext, +) -> anyhow::Result<()> { + for trigger_type in app.triggers.keys() { + if let Some(env) = envs + .iter() + .find(|e| !e.environments.contains_key(trigger_type)) + { + anyhow::bail!( + "Environment {} does not support trigger type {trigger_type}", + env.name + ); + } + } + + let components_by_trigger_type = app + .triggers + .iter() + .map(|(ty, ts)| { + ts.iter() + .map(|t| component_source(app, t)) + .collect::, _>>() + .map(|css| (ty, css)) + }) + .collect::, _>>()?; + + for (trigger_type, component) in components_by_trigger_type { + for component in &component { + validate_component_against_environments( + envs, + trigger_type, + component, + resolution_context, + ) + .await?; + } + } + + Ok(()) +} + +async fn validate_component_against_environments( + envs: &[TargetEnvironment], + trigger_type: &TriggerType, + component: &ComponentToValidate<'_>, + resolution_context: &ResolutionContext, +) -> anyhow::Result<()> { + let worlds = envs + .iter() + .map(|e| { + e.environments + .get(trigger_type) + .ok_or(anyhow!( + "Environment '{}' doesn't support trigger type {trigger_type}", + e.name + )) + .map(|w| (e.name.as_str(), w)) + }) + .collect::, _>>()?; + validate_component_against_worlds(worlds.into_iter(), component, resolution_context).await?; + Ok(()) +} + +async fn validate_component_against_worlds( + target_worlds: impl Iterator, + component: &ComponentToValidate<'_>, + resolution_context: &ResolutionContext, +) -> anyhow::Result<()> { + let loader = ComponentSourceLoader::new(resolution_context.wasm_loader()); + let wasm_bytes = spin_compose::compose(&loader, component).await?; + + for (env_name, target_world) in target_worlds { + validate_wasm_against_any_world(env_name, target_world, component, wasm_bytes.as_ref()) + .await?; + } + + tracing::info!( + "Validated component {} {} against all target worlds", + component.id(), + component.source_description() + ); + Ok(()) +} + +async fn validate_wasm_against_any_world( + env_name: &str, + target_world: &TargetWorld, + component: &ComponentToValidate<'_>, + wasm: &[u8], +) -> anyhow::Result<()> { + let mut result = Ok(()); + for target_str in target_world.versioned_names() { + tracing::info!( + "Trying component {} {} against target world {target_str}", + component.id(), + component.source_description(), + ); + match validate_wasm_against_world(env_name, &target_str, component, wasm).await { + Ok(()) => { + tracing::info!( + "Validated component {} {} against target world {target_str}", + component.id(), + component.source_description(), + ); + return Ok(()); + } + Err(e) => { + // Record the error, but continue in case a different world succeeds + tracing::info!( + "Rejecting component {} {} for target world {target_str} because {e:?}", + component.id(), + component.source_description(), + ); + result = Err(e); + } + } + } + result +} + +async fn validate_wasm_against_world( + env_name: &str, + target_str: &str, + component: &ComponentToValidate<'_>, + wasm: &[u8], +) -> anyhow::Result<()> { + let comp_name = "root:component"; + + let wac_text = format!( + r#" + package validate:component@1.0.0 targets {target_str}; + let c = new {comp_name} {{ ... }}; + export c...; + "# + ); + + let doc = wac_parser::Document::parse(&wac_text)?; + + let compkey = wac_types::BorrowedPackageKey::from_name_and_version(comp_name, None); + + let mut refpkgs = wac_resolver::packages(&doc)?; + refpkgs.retain(|k, _| k != &compkey); + + let reg_resolver = wac_resolver::RegistryPackageResolver::new(Some("wa.dev"), None).await?; + let mut packages = reg_resolver + .resolve(&refpkgs) + .await + .context("reg_resolver.resolve failed")?; + + packages.insert(compkey, wasm.to_vec()); + + match doc.resolve(packages) { + Ok(_) => Ok(()), + Err(wac_parser::resolution::Error::TargetMismatch { kind, name, world, .. }) => { + // This one doesn't seem to get hit at the moment - we get MissingTargetExport or ImportNotInTarget instead + Err(anyhow!("Component {} ({}) can't run in environment {env_name} because world {world} expects an {} named {name}", component.id(), component.source_description(), kind.to_string().to_lowercase())) + } + Err(wac_parser::resolution::Error::MissingTargetExport { name, world, .. }) => { + Err(anyhow!("Component {} ({}) can't run in environment {env_name} because world {world} requires an export named {name}, which the component does not provide", component.id(), component.source_description())) + } + Err(wac_parser::resolution::Error::PackageMissingExport { export, .. }) => { + // TODO: The export here seems wrong - it seems to contain the world name rather than the interface name + Err(anyhow!("Component {} ({}) can't run in environment {env_name} because world {target_str} requires an export named {export}, which the component does not provide", component.id(), component.source_description())) + } + Err(wac_parser::resolution::Error::ImportNotInTarget { name, world, .. }) => { + Err(anyhow!("Component {} ({}) can't run in environment {env_name} because world {world} does not provide an import named {name}, which the component requires", component.id(), component.source_description())) + } + Err(e) => { + Err(anyhow!(e)) + }, + } +} diff --git a/crates/environments/src/loader.rs b/crates/environments/src/loader.rs new file mode 100644 index 0000000000..13e4c6d38b --- /dev/null +++ b/crates/environments/src/loader.rs @@ -0,0 +1,179 @@ +use std::path::Path; + +use anyhow::anyhow; +use spin_common::ui::quoted_path; + +pub(crate) struct ComponentToValidate<'a> { + id: &'a str, + source: &'a spin_manifest::schema::v2::ComponentSource, + dependencies: WrappedComponentDependencies, +} + +impl<'a> ComponentToValidate<'a> { + pub fn id(&self) -> &str { + self.id + } + + pub fn source_description(&self) -> String { + match self.source { + spin_manifest::schema::v2::ComponentSource::Local(path) => { + format!("file {}", quoted_path(path)) + } + spin_manifest::schema::v2::ComponentSource::Remote { url, .. } => format!("URL {url}"), + spin_manifest::schema::v2::ComponentSource::Registry { package, .. } => { + format!("package {package}") + } + } + } +} + +pub fn component_source<'a>( + app: &'a spin_manifest::schema::v2::AppManifest, + trigger: &'a spin_manifest::schema::v2::Trigger, +) -> anyhow::Result> { + let component_spec = trigger + .component + .as_ref() + .ok_or_else(|| anyhow!("No component specified for trigger {}", trigger.id))?; + let (id, source, dependencies) = match component_spec { + spin_manifest::schema::v2::ComponentSpec::Inline(c) => { + (trigger.id.as_str(), &c.source, &c.dependencies) + } + spin_manifest::schema::v2::ComponentSpec::Reference(r) => { + let id = r.as_ref(); + let Some(component) = app.components.get(r) else { + anyhow::bail!( + "Component {id} specified for trigger {} does not exist", + trigger.id + ); + }; + (id, &component.source, &component.dependencies) + } + }; + Ok(ComponentToValidate { + id, + source, + dependencies: WrappedComponentDependencies::new(dependencies), + }) +} + +pub struct ResolutionContext { + wasm_loader: spin_loader::WasmLoader, +} + +impl ResolutionContext { + pub async fn new(base_dir: impl AsRef) -> anyhow::Result { + let wasm_loader = + spin_loader::WasmLoader::new(base_dir.as_ref().to_owned(), None, None).await?; + Ok(Self { wasm_loader }) + } + + pub(crate) fn wasm_loader(&self) -> &spin_loader::WasmLoader { + &self.wasm_loader + } +} + +pub(crate) struct ComponentSourceLoader<'a> { + wasm_loader: &'a spin_loader::WasmLoader, + _phantom: std::marker::PhantomData<&'a usize>, +} + +impl<'a> ComponentSourceLoader<'a> { + pub fn new(wasm_loader: &'a spin_loader::WasmLoader) -> Self { + Self { + wasm_loader, + _phantom: std::marker::PhantomData, + } + } +} + +#[async_trait::async_trait] +impl<'a> spin_compose::ComponentSourceLoader for ComponentSourceLoader<'a> { + type Component = ComponentToValidate<'a>; + type Dependency = WrappedComponentDependency; + async fn load_component_source(&self, source: &Self::Component) -> anyhow::Result> { + let path = self + .wasm_loader + .load_component_source(source.id(), source.source) + .await?; + let bytes = tokio::fs::read(&path).await?; + let component = spin_componentize::componentize_if_necessary(&bytes)?; + Ok(component.into()) + } + + async fn load_dependency_source(&self, source: &Self::Dependency) -> anyhow::Result> { + let (path, _) = self + .wasm_loader + .load_component_dependency(&source.name, &source.dependency) + .await?; + let bytes = tokio::fs::read(&path).await?; + let component = spin_componentize::componentize_if_necessary(&bytes)?; + Ok(component.into()) + } +} + +// This exists only to thwart the orphan rule +pub(crate) struct WrappedComponentDependency { + name: spin_serde::DependencyName, + dependency: spin_manifest::schema::v2::ComponentDependency, +} + +// To manage lifetimes around the thwarting of the orphan rule +struct WrappedComponentDependencies { + dependencies: indexmap::IndexMap, +} + +impl WrappedComponentDependencies { + fn new(deps: &spin_manifest::schema::v2::ComponentDependencies) -> Self { + let dependencies = deps + .inner + .clone() + .into_iter() + .map(|(k, v)| { + ( + k.clone(), + WrappedComponentDependency { + name: k, + dependency: v, + }, + ) + }) + .collect(); + Self { dependencies } + } +} + +#[async_trait::async_trait] +impl<'a> spin_compose::ComponentLike for ComponentToValidate<'a> { + type Dependency = WrappedComponentDependency; + + fn dependencies( + &self, + ) -> impl std::iter::ExactSizeIterator + { + self.dependencies.dependencies.iter() + } + + fn id(&self) -> &str { + self.id + } +} + +#[async_trait::async_trait] +impl spin_compose::DependencyLike for WrappedComponentDependency { + fn inherit(&self) -> spin_compose::InheritConfiguration { + // We don't care because this never runs - it is only used to + // verify import satisfaction. Choosing All avoids the compose + // algorithm meddling with it using the deny adapter. + spin_compose::InheritConfiguration::All + } + + fn export(&self) -> &Option { + match &self.dependency { + spin_manifest::schema::v2::ComponentDependency::Version(_) => &None, + spin_manifest::schema::v2::ComponentDependency::Package { export, .. } => export, + spin_manifest::schema::v2::ComponentDependency::Local { export, .. } => export, + spin_manifest::schema::v2::ComponentDependency::HTTP { export, .. } => export, + } + } +} diff --git a/crates/loader/src/lib.rs b/crates/loader/src/lib.rs index ea64bac40d..0142425bfd 100644 --- a/crates/loader/src/lib.rs +++ b/crates/loader/src/lib.rs @@ -23,6 +23,8 @@ mod fs; mod http; mod local; +pub use local::WasmLoader; + /// Maximum number of files to copy (or download) concurrently pub(crate) const MAX_FILE_LOADING_CONCURRENCY: usize = 16; diff --git a/crates/loader/src/local.rs b/crates/loader/src/local.rs index bc4e5ca2b1..7533783218 100644 --- a/crates/loader/src/local.rs +++ b/crates/loader/src/local.rs @@ -23,8 +23,8 @@ use crate::{cache::Cache, FilesMountStrategy}; pub struct LocalLoader { app_root: PathBuf, files_mount_strategy: FilesMountStrategy, - cache: Cache, - file_loading_permits: Semaphore, + file_loading_permits: std::sync::Arc, + wasm_loader: WasmLoader, } impl LocalLoader { @@ -35,12 +35,14 @@ impl LocalLoader { ) -> Result { let app_root = safe_canonicalize(app_root) .with_context(|| format!("Invalid manifest dir `{}`", app_root.display()))?; + let file_loading_permits = + std::sync::Arc::new(Semaphore::new(crate::MAX_FILE_LOADING_CONCURRENCY)); Ok(Self { - app_root, + app_root: app_root.clone(), files_mount_strategy, - cache: Cache::new(cache_root).await?, // Limit concurrency to avoid hitting system resource limits - file_loading_permits: Semaphore::new(crate::MAX_FILE_LOADING_CONCURRENCY), + file_loading_permits: file_loading_permits.clone(), + wasm_loader: WasmLoader::new(app_root, cache_root, Some(file_loading_permits)).await?, }) } @@ -259,74 +261,15 @@ impl LocalLoader { dependency_name: DependencyName, dependency: v2::ComponentDependency, ) -> Result { - let (content, export) = match dependency { - v2::ComponentDependency::Version(version) => { - let version = semver::VersionReq::parse(&version).with_context(|| format!("Component dependency {dependency_name:?} specifies an invalid semantic version requirement ({version:?}) for its package version"))?; - - // This `unwrap()` should be OK because we've already validated - // this form of dependency requires a package name, i.e. the - // dependency name is not a kebab id. - let package = dependency_name.package().unwrap(); - - let content = self.load_registry_source(None, package, &version).await?; - (content, None) - } - v2::ComponentDependency::Package { - version, - registry, - package, - export, - } => { - let version = semver::VersionReq::parse(&version).with_context(|| format!("Component dependency {dependency_name:?} specifies an invalid semantic version requirement ({version:?}) for its package version"))?; - - let package = match package { - Some(package) => { - package.parse().with_context(|| format!("Component dependency {dependency_name:?} specifies an invalid package name ({package:?})"))? - } - None => { - // This `unwrap()` should be OK because we've already validated - // this form of dependency requires a package name, i.e. the - // dependency name is not a kebab id. - dependency_name - .package() - .cloned() - .unwrap() - } - }; - - let registry = match registry { - Some(registry) => { - registry - .parse() - .map(Some) - .with_context(|| format!("Component dependency {dependency_name:?} specifies an invalid registry name ({registry:?})"))? - } - None => None, - }; - - let content = self - .load_registry_source(registry.as_ref(), &package, &version) - .await?; - (content, export) - } - v2::ComponentDependency::Local { path, export } => { - let content = file_content_ref(self.app_root.join(path))?; - (content, export) - } - v2::ComponentDependency::HTTP { - url, - digest, - export, - } => { - let content = self.load_http_source(&url, &digest).await?; - (content, export) - } - }; + let (content, export) = self + .wasm_loader + .load_component_dependency(&dependency_name, &dependency) + .await?; Ok(LockedComponentDependency { source: LockedComponentSource { content_type: "application/wasm".into(), - content, + content: file_content_ref(content)?, }, export, inherit: if inherit_configuration { @@ -344,110 +287,16 @@ impl LocalLoader { component_id: &KebabId, source: v2::ComponentSource, ) -> Result { - let content = match source { - v2::ComponentSource::Local(path) => file_content_ref(self.app_root.join(path))?, - v2::ComponentSource::Remote { url, digest } => { - self.load_http_source(&url, &digest).await? - } - v2::ComponentSource::Registry { - registry, - package, - version, - } => { - let version = semver::Version::parse(&version).with_context(|| format!("Component {component_id} specifies an invalid semantic version ({version:?}) for its package version"))?; - let version_req = format!("={version}").parse().expect("version"); - - self.load_registry_source(registry.as_ref(), &package, &version_req) - .await? - } - }; + let path = self + .wasm_loader + .load_component_source(component_id.as_ref(), &source) + .await?; Ok(LockedComponentSource { content_type: "application/wasm".into(), - content, + content: file_content_ref(path)?, }) } - // Load a Wasm source from the given HTTP ContentRef source URL and - // return a ContentRef an absolute path to the local copy. - async fn load_http_source(&self, url: &str, digest: &str) -> Result { - ensure!( - digest.starts_with("sha256:"), - "invalid `digest` {digest:?}; must start with 'sha256:'" - ); - let path = if let Ok(cached_path) = self.cache.wasm_file(digest) { - cached_path - } else { - let _loading_permit = self.file_loading_permits.acquire().await?; - - self.cache.ensure_dirs().await?; - let dest = self.cache.wasm_path(digest); - verified_download(url, digest, &dest) - .await - .with_context(|| format!("Error fetching source URL {url:?}"))?; - dest - }; - file_content_ref(path) - } - - async fn load_registry_source( - &self, - registry: Option<&wasm_pkg_loader::Registry>, - package: &wasm_pkg_loader::PackageRef, - version: &semver::VersionReq, - ) -> Result { - let mut client_config = wasm_pkg_loader::Config::global_defaults()?; - - if let Some(registry) = registry.cloned() { - client_config.set_package_registry_override(package.clone(), registry); - } - let mut pkg_loader = wasm_pkg_loader::Client::new(client_config); - - let mut releases = pkg_loader.list_all_versions(package).await.map_err(|e| { - if matches!(e, wasm_pkg_loader::Error::NoRegistryForNamespace(_)) && registry.is_none() { - anyhow!("No default registry specified for wasm-pkg-loader. Create a default config, or set `registry` for package {package:?}") - } else { - e.into() - } - })?; - - releases.sort(); - - let release_version = releases - .iter() - .rev() - .find(|release| version.matches(&release.version) && !release.yanked) - .with_context(|| format!("No matching version found for {package} {version}",))?; - - let release = pkg_loader - .get_release(package, &release_version.version) - .await?; - - let digest = match &release.content_digest { - wasm_pkg_loader::ContentDigest::Sha256 { hex } => format!("sha256:{hex}"), - }; - - let path = if let Ok(cached_path) = self.cache.wasm_file(&digest) { - cached_path - } else { - let mut stm = pkg_loader.stream_content(package, &release).await?; - - self.cache.ensure_dirs().await?; - let dest = self.cache.wasm_path(&digest); - - let mut file = tokio::fs::File::create(&dest).await?; - while let Some(block) = stm.next().await { - let bytes = block.context("Failed to get content from registry")?; - file.write_all(&bytes) - .await - .context("Failed to save registry content to cache")?; - } - - dest - }; - - file_content_ref(path) - } - // Copy content(s) from the given `mount` async fn copy_file_mounts( &self, @@ -764,6 +613,211 @@ fn locked_trigger(trigger_type: String, trigger: v2::Trigger) -> Result, +} + +impl WasmLoader { + /// Create a new instance of WasmLoader. + pub async fn new( + app_root: PathBuf, + cache_root: Option, + file_loading_permits: Option>, + ) -> Result { + let file_loading_permits = file_loading_permits.unwrap_or_else(|| { + std::sync::Arc::new(Semaphore::new(crate::MAX_FILE_LOADING_CONCURRENCY)) + }); + Ok(Self { + app_root, + cache: Cache::new(cache_root).await?, + file_loading_permits, + }) + } + + /// Load a Wasm source from the given ComponentSource and return a path + /// to a file location from where it can be read. + pub async fn load_component_source( + &self, + component_id: &str, + source: &v2::ComponentSource, + ) -> Result { + let content = match source { + v2::ComponentSource::Local(path) => self.app_root.join(path), + v2::ComponentSource::Remote { url, digest } => { + self.load_http_source(url, digest).await? + } + v2::ComponentSource::Registry { + registry, + package, + version, + } => { + let version = semver::Version::parse(version).with_context(|| format!("Component {component_id} specifies an invalid semantic version ({version:?}) for its package version"))?; + let version_req = format!("={version}").parse().expect("version"); + + self.load_registry_source(registry.as_ref(), package, &version_req) + .await? + } + }; + Ok(content) + } + + // Load a Wasm source from the given HTTP ContentRef source URL and + // return a ContentRef an absolute path to the local copy. + async fn load_http_source(&self, url: &str, digest: &str) -> Result { + ensure!( + digest.starts_with("sha256:"), + "invalid `digest` {digest:?}; must start with 'sha256:'" + ); + let path = if let Ok(cached_path) = self.cache.wasm_file(digest) { + cached_path + } else { + let _loading_permit = self.file_loading_permits.acquire().await?; + + self.cache.ensure_dirs().await?; + let dest = self.cache.wasm_path(digest); + verified_download(url, digest, &dest) + .await + .with_context(|| format!("Error fetching source URL {url:?}"))?; + dest + }; + Ok(path) + } + + async fn load_registry_source( + &self, + registry: Option<&wasm_pkg_loader::Registry>, + package: &wasm_pkg_loader::PackageRef, + version: &semver::VersionReq, + ) -> Result { + let mut client_config = wasm_pkg_loader::Config::global_defaults()?; + + if let Some(registry) = registry.cloned() { + client_config.set_package_registry_override(package.clone(), registry); + } + let mut pkg_loader = wasm_pkg_loader::Client::new(client_config); + + let mut releases = pkg_loader.list_all_versions(package).await.map_err(|e| { + if matches!(e, wasm_pkg_loader::Error::NoRegistryForNamespace(_)) && registry.is_none() { + anyhow!("No default registry specified for wasm-pkg-loader. Create a default config, or set `registry` for package {package:?}") + } else { + e.into() + } + })?; + + releases.sort(); + + let release_version = releases + .iter() + .rev() + .find(|release| version.matches(&release.version) && !release.yanked) + .with_context(|| format!("No matching version found for {package} {version}",))?; + + let release = pkg_loader + .get_release(package, &release_version.version) + .await?; + + let digest = match &release.content_digest { + wasm_pkg_loader::ContentDigest::Sha256 { hex } => format!("sha256:{hex}"), + }; + + let path = if let Ok(cached_path) = self.cache.wasm_file(&digest) { + cached_path + } else { + let mut stm = pkg_loader.stream_content(package, &release).await?; + + self.cache.ensure_dirs().await?; + let dest = self.cache.wasm_path(&digest); + + let mut file = tokio::fs::File::create(&dest).await?; + while let Some(block) = stm.next().await { + let bytes = block.context("Failed to get content from registry")?; + file.write_all(&bytes) + .await + .context("Failed to save registry content to cache")?; + } + + dest + }; + + Ok(path) + } + + /// Loads a dependency + pub async fn load_component_dependency( + &self, + dependency_name: &DependencyName, + dependency: &v2::ComponentDependency, + ) -> Result<(PathBuf, Option)> { + match dependency.clone() { + v2::ComponentDependency::Version(version) => { + let version = semver::VersionReq::parse(&version).with_context(|| format!("Component dependency {dependency_name:?} specifies an invalid semantic version requirement ({version:?}) for its package version"))?; + + // This `unwrap()` should be OK because we've already validated + // this form of dependency requires a package name, i.e. the + // dependency name is not a kebab id. + let package = dependency_name.package().unwrap(); + + let content = self.load_registry_source(None, package, &version).await?; + Ok((content, None)) + } + v2::ComponentDependency::Package { + version, + registry, + package, + export, + } => { + let version = semver::VersionReq::parse(&version).with_context(|| format!("Component dependency {dependency_name:?} specifies an invalid semantic version requirement ({version:?}) for its package version"))?; + + let package = match package { + Some(package) => { + package.parse().with_context(|| format!("Component dependency {dependency_name:?} specifies an invalid package name ({package:?})"))? + } + None => { + // This `unwrap()` should be OK because we've already validated + // this form of dependency requires a package name, i.e. the + // dependency name is not a kebab id. + dependency_name + .package() + .cloned() + .unwrap() + } + }; + + let registry = match registry { + Some(registry) => { + registry + .parse() + .map(Some) + .with_context(|| format!("Component dependency {dependency_name:?} specifies an invalid registry name ({registry:?})"))? + } + None => None, + }; + + let content = self + .load_registry_source(registry.as_ref(), &package, &version) + .await?; + Ok((content, export)) + } + v2::ComponentDependency::Local { path, export } => { + let content = self.app_root.join(path); + Ok((content, export)) + } + v2::ComponentDependency::HTTP { + url, + digest, + export, + } => { + let content = self.load_http_source(&url, &digest).await?; + Ok((content, export)) + } + } + } +} + fn looks_like_glob_pattern(s: impl AsRef) -> bool { let s = s.as_ref(); glob::Pattern::escape(s) != s diff --git a/crates/manifest/src/compat.rs b/crates/manifest/src/compat.rs index 638fbf3d01..b1c17107e3 100644 --- a/crates/manifest/src/compat.rs +++ b/crates/manifest/src/compat.rs @@ -20,6 +20,7 @@ pub fn v1_to_v2_app(manifest: v1::AppManifestV1) -> Result, + /// `targets = ["spin-2.5", "fermyon-cloud", "spinkube-0.4"]` + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub targets: Vec, /// `[application.triggers.]` #[serde(rename = "trigger", default, skip_serializing_if = "Map::is_empty")] pub trigger_global_configs: Map, diff --git a/crates/trigger/src/loader.rs b/crates/trigger/src/loader.rs index 8479fa5e8c..0d131ab975 100644 --- a/crates/trigger/src/loader.rs +++ b/crates/trigger/src/loader.rs @@ -106,8 +106,20 @@ struct ComponentSourceLoader; #[async_trait] impl spin_compose::ComponentSourceLoader for ComponentSourceLoader { - async fn load_component_source( - &self, + type Component = spin_app::locked::LockedComponent; + type Dependency = spin_app::locked::LockedComponentDependency; + + async fn load_component_source(&self, source: &Self::Component) -> anyhow::Result> { + Self::load_from_locked_source(&source.source).await + } + + async fn load_dependency_source(&self, source: &Self::Dependency) -> anyhow::Result> { + Self::load_from_locked_source(&source.source).await + } +} + +impl ComponentSourceLoader { + async fn load_from_locked_source( source: &spin_app::locked::LockedComponentSource, ) -> anyhow::Result> { let source = source diff --git a/examples/spin-timer/Cargo.lock b/examples/spin-timer/Cargo.lock index 41c4e14a63..de40e6018d 100644 --- a/examples/spin-timer/Cargo.lock +++ b/examples/spin-timer/Cargo.lock @@ -2,15 +2,6 @@ # It is not intended for manual editing. version = 3 -[[package]] -name = "addr2line" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" -dependencies = [ - "gimli 0.28.1", -] - [[package]] name = "addr2line" version = "0.22.0" @@ -21,10 +12,13 @@ dependencies = [ ] [[package]] -name = "adler" -version = "1.0.2" +name = "addr2line" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +checksum = "f5fb1d8e4442bd405fdfd1dacb42792696b0cf9cb15882e5d097b742a676d375" +dependencies = [ + "gimli 0.31.0", +] [[package]] name = "adler2" @@ -82,9 +76,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.86" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" +checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6" [[package]] name = "arbitrary" @@ -166,9 +160,9 @@ dependencies = [ [[package]] name = "async-process" -version = "2.2.4" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8a07789659a4d385b79b18b9127fc27e1a59e1e89117c78c5ea3b806f016374" +checksum = "63255f1dc2381611000436537bbedfe83183faa303a5a0edaf191edef06526bb" dependencies = [ "async-channel 2.3.1", "async-io", @@ -181,7 +175,6 @@ dependencies = [ "futures-lite 2.3.0", "rustix", "tracing", - "windows-sys 0.59.0", ] [[package]] @@ -221,7 +214,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -232,13 +225,13 @@ checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" [[package]] name = "async-trait" -version = "0.1.81" +version = "0.1.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" +checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -260,15 +253,15 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.1.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "axum" -version = "0.7.5" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" +checksum = "8f43644eed690f5374f1af436ecd6aea01cd201f6fbdf0178adaf6907afb2cec" dependencies = [ "async-trait", "axum-core", @@ -286,16 +279,16 @@ dependencies = [ "rustversion", "serde", "sync_wrapper 1.0.1", - "tower", + "tower 0.5.1", "tower-layer", "tower-service", ] [[package]] name = "axum-core" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a15c63fd72d41492dc4f497196f5da1fb04fb7529e631d73630d1b491e47a2e3" +checksum = "5e6b8ba012a258d63c9adfa28b9ddcf66149da6f986c5b5452e629d5ee64bf00" dependencies = [ "async-trait", "bytes", @@ -306,7 +299,7 @@ dependencies = [ "mime", "pin-project-lite", "rustversion", - "sync_wrapper 0.1.2", + "sync_wrapper 1.0.1", "tower-layer", "tower-service", ] @@ -314,14 +307,14 @@ dependencies = [ [[package]] name = "azure_core" version = "0.20.0" -source = "git+https://github.com/azure/azure-sdk-for-rust.git?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c#8c4caa251c3903d5eae848b41bb1d02a4d65231c" +source = "git+https://github.com/azure/azure-sdk-for-rust?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c#8c4caa251c3903d5eae848b41bb1d02a4d65231c" dependencies = [ "async-trait", "base64 0.22.1", "bytes", "dyn-clone", "futures", - "getrandom 0.2.12", + "getrandom 0.2.15", "hmac", "http-types", "once_cell", @@ -342,7 +335,7 @@ dependencies = [ [[package]] name = "azure_data_cosmos" version = "0.20.0" -source = "git+https://github.com/azure/azure-sdk-for-rust.git?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c#8c4caa251c3903d5eae848b41bb1d02a4d65231c" +source = "git+https://github.com/azure/azure-sdk-for-rust?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c#8c4caa251c3903d5eae848b41bb1d02a4d65231c" dependencies = [ "async-trait", "azure_core", @@ -360,7 +353,7 @@ dependencies = [ [[package]] name = "azure_identity" version = "0.20.0" -source = "git+https://github.com/azure/azure-sdk-for-rust.git?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c#8c4caa251c3903d5eae848b41bb1d02a4d65231c" +source = "git+https://github.com/azure/azure-sdk-for-rust?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c#8c4caa251c3903d5eae848b41bb1d02a4d65231c" dependencies = [ "async-lock", "async-process", @@ -380,7 +373,7 @@ dependencies = [ [[package]] name = "azure_security_keyvault" version = "0.20.0" -source = "git+https://github.com/azure/azure-sdk-for-rust.git?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c#8c4caa251c3903d5eae848b41bb1d02a4d65231c" +source = "git+https://github.com/azure/azure-sdk-for-rust?rev=8c4caa251c3903d5eae848b41bb1d02a4d65231c#8c4caa251c3903d5eae848b41bb1d02a4d65231c" dependencies = [ "async-trait", "azure_core", @@ -392,17 +385,17 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.71" +version = "0.3.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" dependencies = [ - "addr2line 0.21.0", - "cc", + "addr2line 0.24.1", "cfg-if", "libc", - "miniz_oxide 0.7.2", - "object 0.32.2", + "miniz_oxide", + "object", "rustc-demangle", + "windows-targets 0.52.6", ] [[package]] @@ -432,13 +425,13 @@ dependencies = [ "bitflags 2.6.0", "cexpr", "clang-sys", - "itertools", + "itertools 0.13.0", "proc-macro2", "quote", "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -486,9 +479,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.14.0" +version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "byteorder" @@ -498,9 +491,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" +checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" dependencies = [ "serde", ] @@ -610,9 +603,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cfg_aliases" -version = "0.1.1" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" @@ -741,18 +734,18 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpp_demangle" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e8227005286ec39567949b33df9896bcadfa6051bccca2488129f108ca23119" +checksum = "96e58d342ad113c2b878f16d5d034c03be492ae460cdbc02b7f0f2284d310c7d" dependencies = [ "cfg-if", ] [[package]] name = "cpufeatures" -version = "0.2.12" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" dependencies = [ "libc", ] @@ -791,7 +784,7 @@ dependencies = [ "cranelift-entity", "cranelift-isle", "gimli 0.29.0", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "log", "regalloc2", "rustc-hash 2.0.0", @@ -872,7 +865,7 @@ dependencies = [ "cranelift-codegen", "cranelift-entity", "cranelift-frontend", - "itertools", + "itertools 0.12.1", "log", "smallvec", "wasmparser 0.217.0", @@ -881,9 +874,9 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" dependencies = [ "cfg-if", ] @@ -940,9 +933,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.19" +version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" [[package]] name = "crypto-common" @@ -956,12 +949,12 @@ dependencies = [ [[package]] name = "ctrlc" -version = "3.4.4" +version = "3.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "672465ae37dc1bc6380a6547a8883d5dd397b0f1faaad4f265726cc7042a5345" +checksum = "90eeab0aa92f3f9b4e87f258c72b139c207d251f9cbc1080a0086b86a8870dd3" dependencies = [ "nix", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -1136,9 +1129,9 @@ checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125" [[package]] name = "either" -version = "1.10.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" [[package]] name = "embedded-io" @@ -1146,11 +1139,17 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + [[package]] name = "encoding_rs" -version = "0.8.33" +version = "0.8.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" +checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" dependencies = [ "cfg-if", ] @@ -1163,9 +1162,9 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" dependencies = [ "libc", "windows-sys 0.52.0", @@ -1255,7 +1254,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "324a1be68054ef05ad64b861cc9eaf1d623d2d8cb25b4bf2cb9cdd902b4bf253" dependencies = [ "crc32fast", - "miniz_oxide 0.8.0", + "miniz_oxide", ] [[package]] @@ -1394,7 +1393,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -1472,9 +1471,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.12" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", "js-sys", @@ -1483,12 +1482,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "gimli" -version = "0.28.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" - [[package]] name = "gimli" version = "0.29.0" @@ -1496,10 +1489,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" dependencies = [ "fallible-iterator 0.3.0", - "indexmap 2.2.6", + "indexmap 2.5.0", "stable_deref_trait", ] +[[package]] +name = "gimli" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32085ea23f3234fc7846555e85283ba4de91e21016dc0455a16286d87a292d64" + [[package]] name = "glob" version = "0.3.1" @@ -1518,7 +1517,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.2.6", + "indexmap 2.5.0", "slab", "tokio", "tokio-util", @@ -1537,7 +1536,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.1.0", - "indexmap 2.2.6", + "indexmap 2.5.0", "slab", "tokio", "tokio-util", @@ -1552,9 +1551,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hashbrown" -version = "0.14.3" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ "ahash", "allocator-api2", @@ -1567,7 +1566,7 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" dependencies = [ - "hashbrown 0.14.3", + "hashbrown 0.14.5", ] [[package]] @@ -1768,20 +1767,20 @@ dependencies = [ "rustls-pki-types", "tokio", "tokio-rustls 0.25.0", - "webpki-roots 0.26.3", + "webpki-roots 0.26.6", ] [[package]] name = "hyper-rustls" -version = "0.27.2" +version = "0.27.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" +checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" dependencies = [ "futures-util", "http 1.1.0", "hyper 1.4.1", "hyper-util", - "rustls 0.23.12", + "rustls 0.23.13", "rustls-pki-types", "tokio", "tokio-rustls 0.26.0", @@ -1819,9 +1818,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.7" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde7055719c54e36e95e8719f95883f22072a48ede39db7fc17a4e1d5281e9b9" +checksum = "41296eb09f183ac68eec06e03cdbea2e759633d4067b2f6552fc2e009bcad08b" dependencies = [ "bytes", "futures-channel", @@ -1832,16 +1831,15 @@ dependencies = [ "pin-project-lite", "socket2", "tokio", - "tower", "tower-service", "tracing", ] [[package]] name = "iana-time-zone" -version = "0.1.60" +version = "0.1.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -1894,12 +1892,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.2.6" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" dependencies = [ "equivalent", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "serde", ] @@ -1942,9 +1940,9 @@ checksum = "aa2f047c0a98b2f299aa5d6d7088443570faae494e9ae1305e48be000c9e0eb1" [[package]] name = "ipnet" -version = "2.9.0" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" +checksum = "187674a687eed5fe42285b40c6291f9a01517d415fad1c3cbc6a9f778af7fcd4" [[package]] name = "itertools" @@ -1955,11 +1953,20 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "ittapi" @@ -1983,18 +1990,18 @@ dependencies = [ [[package]] name = "jobserver" -version = "0.1.31" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" dependencies = [ "libc", ] [[package]] name = "js-sys" -version = "0.3.69" +version = "0.3.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" dependencies = [ "wasm-bindgen", ] @@ -2005,7 +2012,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ee7893dab2e44ae5f9d0173f26ff4aa327c10b01b06a72b52dd9405b628640d" dependencies = [ - "indexmap 2.2.6", + "indexmap 2.5.0", ] [[package]] @@ -2022,9 +2029,9 @@ checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" [[package]] name = "libc" -version = "0.2.158" +version = "0.2.159" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" +checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" [[package]] name = "libloading" @@ -2075,7 +2082,7 @@ dependencies = [ "thiserror", "tokio", "tokio-util", - "tower", + "tower 0.4.13", "tracing", ] @@ -2100,7 +2107,7 @@ dependencies = [ "bitflags 2.6.0", "cc", "fallible-iterator 0.3.0", - "indexmap 2.2.6", + "indexmap 2.5.0", "log", "memchr", "phf", @@ -2128,9 +2135,9 @@ checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" [[package]] name = "lock_api" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" dependencies = [ "autocfg", "scopeguard", @@ -2148,7 +2155,7 @@ version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37ee39891760e7d94734f6f63fedc29a2e4a152f836120753a72503f09fcf904" dependencies = [ - "hashbrown 0.14.3", + "hashbrown 0.14.5", ] [[package]] @@ -2193,9 +2200,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.1" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memfd" @@ -2218,15 +2225,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" -[[package]] -name = "miniz_oxide" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" -dependencies = [ - "adler", -] - [[package]] name = "miniz_oxide" version = "0.8.0" @@ -2330,9 +2328,9 @@ dependencies = [ [[package]] name = "nix" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ "bitflags 2.6.0", "cfg-if", @@ -2421,7 +2419,7 @@ checksum = "c38841cdd844847e3e7c8d29cef9dcfed8877f8f56f9071f77843ecf3baf937f" dependencies = [ "base64 0.13.1", "chrono", - "getrandom 0.2.12", + "getrandom 0.2.15", "http 0.2.12", "rand 0.8.5", "serde", @@ -2434,22 +2432,13 @@ dependencies = [ [[package]] name = "object" -version = "0.32.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" -dependencies = [ - "memchr", -] - -[[package]] -name = "object" -version = "0.36.0" +version = "0.36.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "576dfe1fc8f9df304abb159d767a29d0476f7750fbf8aa7ad07816004a207434" +checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" dependencies = [ "crc32fast", - "hashbrown 0.14.3", - "indexmap 2.2.6", + "hashbrown 0.14.5", + "indexmap 2.5.0", "memchr", ] @@ -2482,7 +2471,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -2543,7 +2532,7 @@ dependencies = [ "opentelemetry-http", "opentelemetry-proto", "opentelemetry_sdk", - "prost 0.13.2", + "prost 0.13.3", "reqwest 0.12.7", "thiserror", "tokio", @@ -2558,7 +2547,7 @@ checksum = "2c43620e8f93359eb7e627a3b16ee92d8585774986f24f2ab010817426c5ce61" dependencies = [ "opentelemetry", "opentelemetry_sdk", - "prost 0.13.2", + "prost 0.13.3", "tonic", ] @@ -2609,15 +2598,15 @@ checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" [[package]] name = "parking" -version = "2.2.0" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" [[package]] name = "parking_lot" -version = "0.12.1" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" dependencies = [ "lock_api", "parking_lot_core", @@ -2625,22 +2614,22 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.9" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", "redox_syscall", "smallvec", - "windows-targets 0.48.5", + "windows-targets 0.52.6", ] [[package]] name = "paste" -version = "1.0.14" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" [[package]] name = "pem" @@ -2665,7 +2654,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", - "indexmap 2.2.6", + "indexmap 2.5.0", ] [[package]] @@ -2724,14 +2713,14 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] name = "pin-project-lite" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" [[package]] name = "pin-utils" @@ -2752,9 +2741,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.29" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2900ede94e305130c13ddd391e0ab7cbaeb783945ae07a279c268cb05109c6cb" +checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" [[package]] name = "polling" @@ -2773,12 +2762,13 @@ dependencies = [ [[package]] name = "postcard" -version = "1.0.8" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a55c51ee6c0db07e68448e336cf8ea4131a620edefebf9893e759b2d793420f8" +checksum = "5f7f0a8d620d71c457dd1d47df76bb18960378da56af4527aaa10f515eee732e" dependencies = [ "cobs", - "embedded-io", + "embedded-io 0.4.0", + "embedded-io 0.6.1", "serde", ] @@ -2815,9 +2805,9 @@ dependencies = [ [[package]] name = "postgres-types" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02048d9e032fb3cc3413bbf7b83a15d84a5d419778e2628751896d856498eee9" +checksum = "f66ea23a2d0e5734297357705193335e0a957696f34bed2f2faefacb2fec336f" dependencies = [ "bytes", "fallible-iterator 0.2.0", @@ -2884,12 +2874,12 @@ dependencies = [ [[package]] name = "prost" -version = "0.13.2" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b2ecbe40f08db5c006b5764a2645f7f3f141ce756412ac9e1dd6087e6d32995" +checksum = "7b0487d90e047de87f984913713b85c601c05609aad5b0df4b4573fbf69aa13f" dependencies = [ "bytes", - "prost-derive 0.13.2", + "prost-derive 0.13.3", ] [[package]] @@ -2899,39 +2889,39 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" dependencies = [ "anyhow", - "itertools", + "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] name = "prost-derive" -version = "0.13.2" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acf0c195eebb4af52c752bec4f52f645da98b6e92077a04110c7f349477ae5ac" +checksum = "e9552f850d5f0964a4e4d0bf306459ac29323ddfbae05e35a7c0d35cb0803cc5" dependencies = [ "anyhow", - "itertools", + "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] name = "psm" -version = "0.1.21" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5787f7cda34e3033a72192c018bc5883100330f362ef279a8cbccfce8bb4e874" +checksum = "aa37f80ca58604976033fae9515a8a2989fc13797d953f7c04fb8fa36a11f205" dependencies = [ "cc", ] [[package]] name = "quote" -version = "1.0.35" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" dependencies = [ "proc-macro2", ] @@ -2995,7 +2985,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.12", + "getrandom 0.2.15", ] [[package]] @@ -3009,9 +2999,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.8.1" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7237101a77a10773db45d62004a272517633fbcc3df19d96455ede1122e051" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" dependencies = [ "either", "rayon-core", @@ -3077,20 +3067,20 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.4.1" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +checksum = "62871f2d65009c0256aed1b9cfeeb8ac272833c404e13d53d400cd0dad7a2ac0" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.6.0", ] [[package]] name = "redox_users" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ - "getrandom 0.2.12", + "getrandom 0.2.15", "libredox", "thiserror", ] @@ -3101,7 +3091,7 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12908dbeb234370af84d0579b9f68258a0f67e201412dd9a2814e6f45b2fc0f0" dependencies = [ - "hashbrown 0.14.3", + "hashbrown 0.14.5", "log", "rustc-hash 2.0.0", "slice-group-by", @@ -3211,7 +3201,7 @@ dependencies = [ "http-body 1.0.1", "http-body-util", "hyper 1.4.1", - "hyper-rustls 0.27.2", + "hyper-rustls 0.27.3", "hyper-tls", "hyper-util", "ipnet", @@ -3227,7 +3217,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "sync_wrapper 1.0.1", - "system-configuration 0.6.0", + "system-configuration 0.6.1", "tokio", "tokio-native-tls", "tokio-util", @@ -3248,7 +3238,7 @@ checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", "cfg-if", - "getrandom 0.2.12", + "getrandom 0.2.15", "libc", "spin", "untrusted", @@ -3267,7 +3257,7 @@ dependencies = [ "log", "rustls-native-certs", "rustls-pemfile 2.1.3", - "rustls-webpki 0.102.7", + "rustls-webpki 0.102.8", "thiserror", "tokio", "tokio-rustls 0.25.0", @@ -3290,9 +3280,9 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] name = "rustc-hash" @@ -3351,9 +3341,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.35" +version = "0.38.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a85d50532239da68e9addb745ba38ff4612a242c1c7ceea689c4bc7c2f43c36f" +checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" dependencies = [ "bitflags 2.6.0", "errno", @@ -3385,22 +3375,22 @@ dependencies = [ "log", "ring", "rustls-pki-types", - "rustls-webpki 0.102.7", + "rustls-webpki 0.102.8", "subtle", "zeroize", ] [[package]] name = "rustls" -version = "0.23.12" +version = "0.23.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" +checksum = "f2dabaac7466917e566adb06783a81ca48944c6898a1b08b9374106dd671f4c8" dependencies = [ "log", "once_cell", "ring", "rustls-pki-types", - "rustls-webpki 0.102.7", + "rustls-webpki 0.102.8", "subtle", "zeroize", ] @@ -3455,9 +3445,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.102.7" +version = "0.102.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84678086bd54edf2b415183ed7a94d0efb049f1b646a33e22a36f3794be6ae56" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" dependencies = [ "ring", "rustls-pki-types", @@ -3472,9 +3462,9 @@ checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" [[package]] name = "ryu" -version = "1.0.16" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "sanitize-filename" @@ -3494,11 +3484,11 @@ checksum = "ece8e78b2f38ec51c51f5d475df0a7187ba5111b2a28bdc761ee05b075d40a71" [[package]] name = "schannel" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +checksum = "e9aaafd5a2b6e3d657ff009d82fbd630b6bd54dd4eb06f21693925cdf80f9b8b" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -3519,11 +3509,11 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.10.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "770452e37cad93e0a50d5abc3990d2bc351c36d0328f86cefec2f2fb206eaef6" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.6.0", "core-foundation", "core-foundation-sys", "libc", @@ -3532,9 +3522,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.11.1" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75da29fe9b9b08fe9d6b22b5b4bcbc75d8db3aa31e639aa56bb62e9d46bfceaf" +checksum = "ea4a292869320c0272d7bc55a5a6aafaff59b4f63404a003887b679a2e05b4b6" dependencies = [ "core-foundation-sys", "libc", @@ -3551,31 +3541,32 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.203" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" +checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.203" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" +checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] name = "serde_json" -version = "1.0.117" +version = "1.0.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" +checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" dependencies = [ "itoa", + "memchr", "ryu", "serde", ] @@ -3603,9 +3594,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.6" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0" +checksum = "eb5b1b31579f3811bf615c144393417496f152e12ac8b7663bf664f4a815306d" dependencies = [ "serde", ] @@ -3676,9 +3667,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "signal-hook-registry" -version = "1.4.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" dependencies = [ "libc", ] @@ -3706,21 +3697,21 @@ checksum = "826167069c09b99d56f31e9ae5c99049e932a98c9dc2dac47645b08dbbf76ba7" [[package]] name = "smallvec" -version = "1.13.1" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" dependencies = [ "serde", ] [[package]] name = "socket2" -version = "0.5.5" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" +checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" dependencies = [ "libc", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -3784,12 +3775,14 @@ version = "2.8.0-pre0" dependencies = [ "anyhow", "async-trait", - "indexmap 2.2.6", + "indexmap 2.5.0", "semver", "spin-app", + "spin-common", "spin-componentize", "spin-serde", "thiserror", + "tokio", "wac-graph", ] @@ -3858,7 +3851,7 @@ dependencies = [ "hyper 1.4.1", "ip_network", "reqwest 0.12.7", - "rustls 0.23.12", + "rustls 0.23.13", "spin-factor-outbound-networking", "spin-factors", "spin-telemetry", @@ -3915,7 +3908,7 @@ dependencies = [ "futures-util", "http 1.1.0", "ipnet", - "rustls 0.23.12", + "rustls 0.23.13", "rustls-pemfile 2.1.3", "rustls-pki-types", "serde", @@ -3930,7 +3923,7 @@ dependencies = [ "tracing", "url", "urlencoding", - "webpki-roots 0.26.3", + "webpki-roots 0.26.6", ] [[package]] @@ -4031,7 +4024,7 @@ version = "2.8.0-pre0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -4116,7 +4109,7 @@ name = "spin-manifest" version = "2.8.0-pre0" dependencies = [ "anyhow", - "indexmap 2.2.6", + "indexmap 2.5.0", "semver", "serde", "spin-serde", @@ -4362,9 +4355,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.76" +version = "2.0.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578e081a14e0cefc3279b0472138c513f37b41a08d5a3cca9b6e4e8ceb6cd525" +checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" dependencies = [ "proc-macro2", "quote", @@ -4411,9 +4404,9 @@ dependencies = [ [[package]] name = "system-configuration" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "658bc6ee10a9b4fcf576e9b0819d95ec16f4d2c02d39fd83ac1c8789785c4a42" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" dependencies = [ "bitflags 2.6.0", "core-foundation", @@ -4468,15 +4461,15 @@ checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" [[package]] name = "tempfile" -version = "3.11.0" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fcd239983515c23a32fb82099f97d0b11b8c72f654ed659363a95c3dad7a53" +checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64" dependencies = [ "cfg-if", "fastrand 2.1.1", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -4492,34 +4485,33 @@ dependencies = [ name = "terminal" version = "2.8.0-pre0" dependencies = [ - "atty", "termcolor", ] [[package]] name = "textwrap" -version = "0.16.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" +checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" [[package]] name = "thiserror" -version = "1.0.63" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" +checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.63" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" +checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -4608,7 +4600,7 @@ checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -4623,9 +4615,9 @@ dependencies = [ [[package]] name = "tokio-postgres" -version = "0.7.11" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03adcf0147e203b6032c0b2d30be1415ba03bc348901f3ff1cc0df6a733e60c3" +checksum = "3b5d3742945bc7d7f210693b0c58ae542c6fd47b17adbbda0885f3dcb34a6bdb" dependencies = [ "async-trait", "byteorder", @@ -4674,7 +4666,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls 0.23.12", + "rustls 0.23.13", "rustls-pki-types", "tokio", ] @@ -4691,9 +4683,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.14" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" +checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" dependencies = [ "futures-core", "pin-project-lite", @@ -4702,9 +4694,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.11" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" +checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" dependencies = [ "bytes", "futures-core", @@ -4715,11 +4707,11 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.14" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f49eb2ab21d2f26bd6db7bf383edc527a7ebaee412d17af4d40fdccd442f335" +checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" dependencies = [ - "indexmap 2.2.6", + "indexmap 2.5.0", "serde", "serde_spanned", "toml_datetime", @@ -4728,20 +4720,20 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.6" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" dependencies = [ "serde", ] [[package]] name = "toml_edit" -version = "0.22.14" +version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f21c7aaf97f1bd9ca9d4f9e73b0a6c74bd5afef56f2bc931943a6e1c37e04e38" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ - "indexmap 2.2.6", + "indexmap 2.5.0", "serde", "serde_spanned", "toml_datetime", @@ -4768,11 +4760,11 @@ dependencies = [ "hyper-util", "percent-encoding", "pin-project", - "prost 0.13.2", + "prost 0.13.3", "socket2", "tokio", "tokio-stream", - "tower", + "tower 0.4.13", "tower-layer", "tower-service", "tracing", @@ -4798,6 +4790,20 @@ dependencies = [ "tracing", ] +[[package]] +name = "tower" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2873938d487c3cfb9aed7546dc9f2711d867c9f90c46b889989a2cb84eba6b4f" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper 0.1.2", + "tower-layer", + "tower-service", +] + [[package]] name = "tower-layer" version = "0.3.3" @@ -4842,7 +4848,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -4967,15 +4973,15 @@ checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" [[package]] name = "unicode-normalization" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" dependencies = [ "tinyvec", ] @@ -4988,15 +4994,15 @@ checksum = "52ea75f83c0137a9b98608359a5f1af8144876eb67bcb1ce837368e906a9f524" [[package]] name = "unicode-width" -version = "0.1.11" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" [[package]] name = "unicode-xid" -version = "0.2.4" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" [[package]] name = "untrusted" @@ -5024,11 +5030,11 @@ checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" [[package]] name = "uuid" -version = "1.7.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f00cc9702ca12d3c81455259621e676d0f7251cec66a21e98fe2e9a37db93b2a" +checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" dependencies = [ - "getrandom 0.2.12", + "getrandom 0.2.15", ] [[package]] @@ -5065,9 +5071,9 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "version_check" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "wac-graph" @@ -5077,7 +5083,7 @@ checksum = "86f708c892ce0ebc06de9915f3da2da9b4e482a8b7d417fa447263b110d0a244" dependencies = [ "anyhow", "id-arena", - "indexmap 2.2.6", + "indexmap 2.5.0", "log", "petgraph", "semver", @@ -5096,7 +5102,7 @@ checksum = "b96fe715180f72ab776d90e8c4f47f8e4297e0e61ab263567a31f73c77d45d8d" dependencies = [ "anyhow", "id-arena", - "indexmap 2.2.6", + "indexmap 2.5.0", "semver", "wasm-encoder 0.202.0", "wasmparser 0.202.0", @@ -5137,34 +5143,35 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" dependencies = [ "cfg-if", + "once_cell", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.42" +version = "0.4.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" +checksum = "61e9300f63a621e96ed275155c108eb6f843b6a26d053f122ab69724559dc8ed" dependencies = [ "cfg-if", "js-sys", @@ -5174,9 +5181,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -5184,22 +5191,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" [[package]] name = "wasm-encoder" @@ -5227,7 +5234,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "094aea3cb90e09f16ee25a4c0e324b3e8c934e7fd838bfa039aef5352f44a917" dependencies = [ "anyhow", - "indexmap 2.2.6", + "indexmap 2.5.0", "serde", "serde_derive", "serde_json", @@ -5243,7 +5250,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65a146bf9a60e9264f0548a2599aa9656dba9a641eff9ab88299dc2a637e483c" dependencies = [ "anyhow", - "indexmap 2.2.6", + "indexmap 2.5.0", "serde", "serde_derive", "serde_json", @@ -5289,7 +5296,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6998515d3cf3f8b980ef7c11b29a9b1017d4cf86b99ae93b546992df9931413" dependencies = [ "bitflags 2.6.0", - "indexmap 2.2.6", + "indexmap 2.5.0", "semver", ] @@ -5301,8 +5308,8 @@ checksum = "ca917a21307d3adf2b9857b94dd05ebf8496bdcff4437a9b9fb3899d3e6c74e7" dependencies = [ "ahash", "bitflags 2.6.0", - "hashbrown 0.14.3", - "indexmap 2.2.6", + "hashbrown 0.14.5", + "indexmap 2.5.0", "semver", "serde", ] @@ -5334,15 +5341,15 @@ dependencies = [ "encoding_rs", "fxprof-processed-profile", "gimli 0.29.0", - "hashbrown 0.14.3", - "indexmap 2.2.6", + "hashbrown 0.14.5", + "indexmap 2.5.0", "ittapi", "libc", "libm", "log", "mach2", "memfd", - "object 0.36.0", + "object", "once_cell", "paste", "postcard", @@ -5412,7 +5419,7 @@ dependencies = [ "anyhow", "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", "wasmtime-component-util", "wasmtime-wit-bindgen", "wit-parser", @@ -5440,7 +5447,7 @@ dependencies = [ "cranelift-wasm", "gimli 0.29.0", "log", - "object 0.36.0", + "object", "smallvec", "target-lexicon", "thiserror", @@ -5460,9 +5467,9 @@ dependencies = [ "cranelift-bitset", "cranelift-entity", "gimli 0.29.0", - "indexmap 2.2.6", + "indexmap 2.5.0", "log", - "object 0.36.0", + "object", "postcard", "rustc-demangle", "semver", @@ -5497,7 +5504,7 @@ version = "25.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "109dcbe0367eeda5467ea2950ff81899dab3ee362220eadcae0691d336122d29" dependencies = [ - "object 0.36.0", + "object", "once_cell", "rustix", "wasmtime-versioned-export-macros", @@ -5543,7 +5550,7 @@ checksum = "467bf568f44048477d865a7bb42a1876acd1e2d3de77b42307f5d8e0126fc241" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -5597,7 +5604,7 @@ dependencies = [ "tracing", "wasmtime", "wasmtime-wasi", - "webpki-roots 0.26.3", + "webpki-roots 0.26.6", ] [[package]] @@ -5609,7 +5616,7 @@ dependencies = [ "anyhow", "cranelift-codegen", "gimli 0.29.0", - "object 0.36.0", + "object", "target-lexicon", "wasmparser 0.217.0", "wasmtime-cranelift", @@ -5625,7 +5632,7 @@ checksum = "eb8a4c5f38371e9dc1718421b03bc8737696587af5e1b233ea515ba5a111d106" dependencies = [ "anyhow", "heck", - "indexmap 2.2.6", + "indexmap 2.5.0", "wit-parser", ] @@ -5662,9 +5669,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.69" +version = "0.3.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" +checksum = "26fdeaafd9bd129f65e7c031593c24d62186301e0c72c8978fa1678be7d532c0" dependencies = [ "js-sys", "wasm-bindgen", @@ -5688,18 +5695,18 @@ checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" [[package]] name = "webpki-roots" -version = "0.26.3" +version = "0.26.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd7c23921eeb1713a4e851530e9b9756e4fb0e89978582942612524cf09f01cd" +checksum = "841c67bff177718f1d4dfefde8d8f0e78f9b6589319ba88312f567fc5841a958" dependencies = [ "rustls-pki-types", ] [[package]] name = "whoami" -version = "1.5.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44ab49fad634e88f55bf8f9bb3abd2f27d7204172a112c7c9987e01c1c94ea9" +checksum = "372d5b87f58ec45c384ba03563b03544dc5fadc3983e434b286913f5b4a9bb6d" dependencies = [ "redox_syscall", "wasite", @@ -5732,7 +5739,7 @@ dependencies = [ "proc-macro2", "quote", "shellexpand", - "syn 2.0.76", + "syn 2.0.77", "witx", ] @@ -5744,7 +5751,7 @@ checksum = "6d8565ac65a40335305bce35a2cf48bd3bddc244637008d493f63d6a6685be26" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", "wiggle-generate", ] @@ -5766,11 +5773,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.6" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "winapi", + "windows-sys 0.59.0", ] [[package]] @@ -5985,9 +5992,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.6.13" +version = "0.6.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59b5e5f6c299a3c7890b876a2a587f3115162487e704907d9b6cd29473052ba1" +checksum = "c52ac009d615e79296318c1bcce2d422aaca15ad08515e344feeda07df67a587" dependencies = [ "memchr", ] @@ -6020,7 +6027,7 @@ checksum = "d7117809905e49db716d81e794f79590c052bf2fdbbcda1731ca0fb28f6f3ddf" dependencies = [ "anyhow", "bitflags 2.6.0", - "indexmap 2.2.6", + "indexmap 2.5.0", "log", "serde", "serde_derive", @@ -6039,7 +6046,7 @@ checksum = "fb893dcd6d370cfdf19a0d9adfcd403efb8e544e1a0ea3a8b81a21fe392eaa78" dependencies = [ "anyhow", "id-arena", - "indexmap 2.2.6", + "indexmap 2.5.0", "log", "semver", "serde", @@ -6063,9 +6070,9 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.7.32" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ "byteorder", "zerocopy-derive", @@ -6073,13 +6080,13 @@ dependencies = [ [[package]] name = "zerocopy-derive" -version = "0.7.32" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -6090,27 +6097,27 @@ checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" [[package]] name = "zstd" -version = "0.13.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d789b1514203a1120ad2429eae43a7bd32b90976a7bb8a05f7ec02fa88cc23a" +checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "7.1.0" +version = "7.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd99b45c6bc03a018c8b8a86025678c87e55526064e38f9df301989dce7ec0a" +checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.10+zstd.1.5.6" +version = "2.0.13+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa" +checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" dependencies = [ "cc", "pkg-config", From 3baee56951b74910adb41a9e19516fe703391ed6 Mon Sep 17 00:00:00 2001 From: itowlson Date: Mon, 9 Sep 2024 15:26:41 +1200 Subject: [PATCH 02/11] Load and compose components once at start of validation Signed-off-by: itowlson --- crates/environments/src/lib.rs | 58 +++++++++++------------ crates/environments/src/loader.rs | 77 ++++++++++++++++++++++--------- 2 files changed, 81 insertions(+), 54 deletions(-) diff --git a/crates/environments/src/lib.rs b/crates/environments/src/lib.rs index 2578c17583..09e2ef0757 100644 --- a/crates/environments/src/lib.rs +++ b/crates/environments/src/lib.rs @@ -5,15 +5,14 @@ mod loader; use environment_definition::{TargetEnvironment, TargetWorld, TriggerType}; pub use loader::ResolutionContext; -use loader::{component_source, ComponentSourceLoader, ComponentToValidate}; +use loader::{load_and_resolve_all, ComponentToValidate}; pub async fn validate_application_against_environment_ids( env_ids: impl Iterator, app: &spin_manifest::schema::v2::AppManifest, resolution_context: &ResolutionContext, ) -> anyhow::Result<()> { - let envs = futures::future::join_all(env_ids.map(resolve_environment_id)).await; - let envs: Vec<_> = envs.into_iter().collect::>()?; + let envs = join_all_result(env_ids.map(resolve_environment_id)).await?; validate_application_against_environments(&envs, app, resolution_context).await } @@ -54,6 +53,8 @@ pub async fn validate_application_against_environments( app: &spin_manifest::schema::v2::AppManifest, resolution_context: &ResolutionContext, ) -> anyhow::Result<()> { + use futures::FutureExt; + for trigger_type in app.triggers.keys() { if let Some(env) = envs .iter() @@ -66,26 +67,15 @@ pub async fn validate_application_against_environments( } } - let components_by_trigger_type = app - .triggers - .iter() - .map(|(ty, ts)| { - ts.iter() - .map(|t| component_source(app, t)) - .collect::, _>>() - .map(|css| (ty, css)) - }) - .collect::, _>>()?; + let components_by_trigger_type_futs = app.triggers.iter().map(|(ty, ts)| { + load_and_resolve_all(app, ts, resolution_context) + .map(|css| css.map(|css| (ty.to_owned(), css))) + }); + let components_by_trigger_type = join_all_result(components_by_trigger_type_futs).await?; for (trigger_type, component) in components_by_trigger_type { for component in &component { - validate_component_against_environments( - envs, - trigger_type, - component, - resolution_context, - ) - .await?; + validate_component_against_environments(envs, &trigger_type, component).await?; } } @@ -96,7 +86,6 @@ async fn validate_component_against_environments( envs: &[TargetEnvironment], trigger_type: &TriggerType, component: &ComponentToValidate<'_>, - resolution_context: &ResolutionContext, ) -> anyhow::Result<()> { let worlds = envs .iter() @@ -110,21 +99,16 @@ async fn validate_component_against_environments( .map(|w| (e.name.as_str(), w)) }) .collect::, _>>()?; - validate_component_against_worlds(worlds.into_iter(), component, resolution_context).await?; + validate_component_against_worlds(worlds.into_iter(), component).await?; Ok(()) } async fn validate_component_against_worlds( target_worlds: impl Iterator, component: &ComponentToValidate<'_>, - resolution_context: &ResolutionContext, ) -> anyhow::Result<()> { - let loader = ComponentSourceLoader::new(resolution_context.wasm_loader()); - let wasm_bytes = spin_compose::compose(&loader, component).await?; - for (env_name, target_world) in target_worlds { - validate_wasm_against_any_world(env_name, target_world, component, wasm_bytes.as_ref()) - .await?; + validate_wasm_against_any_world(env_name, target_world, component).await?; } tracing::info!( @@ -139,7 +123,6 @@ async fn validate_wasm_against_any_world( env_name: &str, target_world: &TargetWorld, component: &ComponentToValidate<'_>, - wasm: &[u8], ) -> anyhow::Result<()> { let mut result = Ok(()); for target_str in target_world.versioned_names() { @@ -148,7 +131,7 @@ async fn validate_wasm_against_any_world( component.id(), component.source_description(), ); - match validate_wasm_against_world(env_name, &target_str, component, wasm).await { + match validate_wasm_against_world(env_name, &target_str, component).await { Ok(()) => { tracing::info!( "Validated component {} {} against target world {target_str}", @@ -175,7 +158,6 @@ async fn validate_wasm_against_world( env_name: &str, target_str: &str, component: &ComponentToValidate<'_>, - wasm: &[u8], ) -> anyhow::Result<()> { let comp_name = "root:component"; @@ -200,7 +182,7 @@ async fn validate_wasm_against_world( .await .context("reg_resolver.resolve failed")?; - packages.insert(compkey, wasm.to_vec()); + packages.insert(compkey, component.wasm_bytes().to_vec()); match doc.resolve(packages) { Ok(_) => Ok(()), @@ -223,3 +205,15 @@ async fn validate_wasm_against_world( }, } } + +/// Equivalent to futures::future::join_all, but specialised for iterators of +/// fallible futures. It returns a Result> instead of a Vec> - +/// this just moves the transposition boilerplate out of the main flow. +async fn join_all_result(iter: I) -> anyhow::Result> +where + I: IntoIterator, + I::Item: std::future::Future>, +{ + let vec_result = futures::future::join_all(iter).await; + vec_result.into_iter().collect() +} diff --git a/crates/environments/src/loader.rs b/crates/environments/src/loader.rs index 13e4c6d38b..087994f53b 100644 --- a/crates/environments/src/loader.rs +++ b/crates/environments/src/loader.rs @@ -4,6 +4,12 @@ use anyhow::anyhow; use spin_common::ui::quoted_path; pub(crate) struct ComponentToValidate<'a> { + id: &'a str, + source_description: String, + wasm: Vec, +} + +struct ComponentSource<'a> { id: &'a str, source: &'a spin_manifest::schema::v2::ComponentSource, dependencies: WrappedComponentDependencies, @@ -14,22 +20,30 @@ impl<'a> ComponentToValidate<'a> { self.id } - pub fn source_description(&self) -> String { - match self.source { - spin_manifest::schema::v2::ComponentSource::Local(path) => { - format!("file {}", quoted_path(path)) - } - spin_manifest::schema::v2::ComponentSource::Remote { url, .. } => format!("URL {url}"), - spin_manifest::schema::v2::ComponentSource::Registry { package, .. } => { - format!("package {package}") - } - } + pub fn source_description(&self) -> &str { + &self.source_description + } + + pub fn wasm_bytes(&self) -> &[u8] { + &self.wasm } } -pub fn component_source<'a>( +pub async fn load_and_resolve_all<'a>( + app: &'a spin_manifest::schema::v2::AppManifest, + triggers: &'a [spin_manifest::schema::v2::Trigger], + resolution_context: &'a ResolutionContext, +) -> anyhow::Result>> { + let component_futures = triggers + .iter() + .map(|t| load_and_resolve_one(app, t, resolution_context)); + crate::join_all_result(component_futures).await +} + +async fn load_and_resolve_one<'a>( app: &'a spin_manifest::schema::v2::AppManifest, trigger: &'a spin_manifest::schema::v2::Trigger, + resolution_context: &'a ResolutionContext, ) -> anyhow::Result> { let component_spec = trigger .component @@ -50,10 +64,21 @@ pub fn component_source<'a>( (id, &component.source, &component.dependencies) } }; - Ok(ComponentToValidate { + + let component = ComponentSource { id, source, dependencies: WrappedComponentDependencies::new(dependencies), + }; + + let loader = ComponentSourceLoader::new(resolution_context.wasm_loader()); + + let wasm = spin_compose::compose(&loader, &component).await?; + + Ok(ComponentToValidate { + id, + source_description: source_description(component.source), + wasm, }) } @@ -68,33 +93,29 @@ impl ResolutionContext { Ok(Self { wasm_loader }) } - pub(crate) fn wasm_loader(&self) -> &spin_loader::WasmLoader { + fn wasm_loader(&self) -> &spin_loader::WasmLoader { &self.wasm_loader } } -pub(crate) struct ComponentSourceLoader<'a> { +struct ComponentSourceLoader<'a> { wasm_loader: &'a spin_loader::WasmLoader, - _phantom: std::marker::PhantomData<&'a usize>, } impl<'a> ComponentSourceLoader<'a> { pub fn new(wasm_loader: &'a spin_loader::WasmLoader) -> Self { - Self { - wasm_loader, - _phantom: std::marker::PhantomData, - } + Self { wasm_loader } } } #[async_trait::async_trait] impl<'a> spin_compose::ComponentSourceLoader for ComponentSourceLoader<'a> { - type Component = ComponentToValidate<'a>; + type Component = ComponentSource<'a>; type Dependency = WrappedComponentDependency; async fn load_component_source(&self, source: &Self::Component) -> anyhow::Result> { let path = self .wasm_loader - .load_component_source(source.id(), source.source) + .load_component_source(source.id, source.source) .await?; let bytes = tokio::fs::read(&path).await?; let component = spin_componentize::componentize_if_necessary(&bytes)?; @@ -144,7 +165,7 @@ impl WrappedComponentDependencies { } #[async_trait::async_trait] -impl<'a> spin_compose::ComponentLike for ComponentToValidate<'a> { +impl<'a> spin_compose::ComponentLike for ComponentSource<'a> { type Dependency = WrappedComponentDependency; fn dependencies( @@ -177,3 +198,15 @@ impl spin_compose::DependencyLike for WrappedComponentDependency { } } } + +fn source_description(source: &spin_manifest::schema::v2::ComponentSource) -> String { + match source { + spin_manifest::schema::v2::ComponentSource::Local(path) => { + format!("file {}", quoted_path(path)) + } + spin_manifest::schema::v2::ComponentSource::Remote { url, .. } => format!("URL {url}"), + spin_manifest::schema::v2::ComponentSource::Registry { package, .. } => { + format!("package {package}") + } + } +} From fd45f2fdc2a6634fb8dd20dae7019170f3c6e32e Mon Sep 17 00:00:00 2001 From: itowlson Date: Thu, 12 Sep 2024 15:39:18 +1200 Subject: [PATCH 03/11] WIT-based environment syntax Signed-off-by: itowlson --- Cargo.lock | 5 + crates/environments/Cargo.toml | 5 + .../src/environment_definition.rs | 127 ++++++++++++---- crates/environments/src/lib.rs | 136 +++++++----------- 4 files changed, 161 insertions(+), 112 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 956ae9b81e..d39a58a3ea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7160,7 +7160,10 @@ version = "2.8.0-pre0" dependencies = [ "anyhow", "async-trait", + "bytes", "futures", + "futures-util", + "id-arena", "indexmap 2.5.0", "miette 7.2.0", "oci-distribution 0.11.0 (git+https://github.com/fermyon/oci-distribution?rev=7e4ce9be9bcd22e78a28f06204931f10c44402ba)", @@ -7179,6 +7182,8 @@ dependencies = [ "wac-resolver", "wac-types", "wasm-pkg-loader", + "wit-component 0.217.0", + "wit-parser 0.217.0", ] [[package]] diff --git a/crates/environments/Cargo.toml b/crates/environments/Cargo.toml index 350599bda4..daea8209ed 100644 --- a/crates/environments/Cargo.toml +++ b/crates/environments/Cargo.toml @@ -7,7 +7,10 @@ edition = { workspace = true } [dependencies] anyhow = { workspace = true } async-trait = "0.1" +bytes = "1.1" futures = "0.3" +futures-util = "0.3" +id-arena = "2" indexmap = "2.2.6" miette = "7.2.0" oci-distribution = { git = "https://github.com/fermyon/oci-distribution", rev = "7e4ce9be9bcd22e78a28f06204931f10c44402ba" } @@ -26,6 +29,8 @@ wac-parser = "0.6.0" wac-resolver = "0.6.0" wac-types = "0.6.0" wasm-pkg-loader = "0.4.1" +wit-component = "0.217.0" +wit-parser = "0.217.0" [lints] workspace = true diff --git a/crates/environments/src/environment_definition.rs b/crates/environments/src/environment_definition.rs index 79813ecac3..5e576b2cdd 100644 --- a/crates/environments/src/environment_definition.rs +++ b/crates/environments/src/environment_definition.rs @@ -1,37 +1,112 @@ -use wasm_pkg_loader::PackageRef; +use anyhow::Context; -#[derive(Debug, serde::Deserialize)] -pub struct TargetEnvironment { - pub name: String, - pub environments: std::collections::HashMap, -} +pub async fn load_environment(env_id: &str) -> anyhow::Result { + use futures_util::TryStreamExt; + + let (pkg_name, pkg_ver) = env_id.split_once('@').unwrap(); + + let mut client = wasm_pkg_loader::Client::with_global_defaults()?; + + let package = pkg_name.to_owned().try_into().context("pkg ref parse")?; + let version = wasm_pkg_loader::Version::parse(pkg_ver).context("pkg ver parse")?; + + let release = client + .get_release(&package, &version) + .await + .context("get release")?; + let stm = client + .stream_content(&package, &release) + .await + .context("stream content")?; + let bytes = stm + .try_collect::() + .await + .context("collect stm")? + .to_vec(); -#[derive(Debug, Eq, Hash, PartialEq, serde::Deserialize)] -pub struct TargetWorld { - wit_package: PackageRef, - package_ver: String, // TODO: tidy to semver::Version - world_name: WorldNames, + TargetEnvironment::new(env_id.to_owned(), bytes) } -#[derive(Debug, Eq, Hash, PartialEq, serde::Deserialize)] -#[serde(untagged)] -enum WorldNames { - Exactly(String), - AnyOf(Vec), +pub struct TargetEnvironment { + name: String, + decoded: wit_parser::decoding::DecodedWasm, + package: wit_parser::Package, // saves unwrapping it every time + package_id: id_arena::Id, + package_bytes: Vec, } -impl TargetWorld { - fn versioned_name(&self, world_name: &str) -> String { - format!("{}/{}@{}", self.wit_package, world_name, self.package_ver) +impl TargetEnvironment { + fn new(name: String, bytes: Vec) -> anyhow::Result { + let decoded = wit_component::decode(&bytes).context("decode wasm")?; + let package_id = decoded.package(); + let package = decoded + .resolve() + .packages + .get(package_id) + .context("should had a package")? + .clone(); + + Ok(Self { + name, + decoded, + package, + package_id, + package_bytes: bytes, + }) + } + + pub fn is_world_for(&self, trigger_type: &TriggerType, world: &wit_parser::World) -> bool { + world.name.starts_with(&format!("trigger-{trigger_type}")) + && world.package.is_some_and(|p| p == self.package_id) + } + + pub fn supports_trigger_type(&self, trigger_type: &TriggerType) -> bool { + self.decoded + .resolve() + .worlds + .iter() + .any(|(_, world)| self.is_world_for(trigger_type, world)) + } + + pub fn worlds(&self, trigger_type: &TriggerType) -> Vec { + self.decoded + .resolve() + .worlds + .iter() + .filter(|(_, world)| self.is_world_for(trigger_type, world)) + .map(|(_, world)| self.world_qname(world)) + .collect() + } + + /// Fully qualified world name (e.g. fermyon:spin/http-trigger@2.0.0) + fn world_qname(&self, world: &wit_parser::World) -> String { + let version_suffix = match self.package_version() { + Some(version) => format!("@{version}"), + None => "".to_owned(), + }; + format!( + "{}/{}{version_suffix}", + self.package_namespaced_name(), + world.name, + ) + } + + /// The environment name for UI purposes + pub fn name(&self) -> &str { + &self.name + } + + /// Namespaced but unversioned package name (e.g. spin:cli) + pub fn package_namespaced_name(&self) -> String { + format!("{}:{}", self.package.name.namespace, self.package.name.name) + } + + pub fn package_version(&self) -> Option<&semver::Version> { + self.package.name.version.as_ref() } - pub fn versioned_names(&self) -> Vec { - match &self.world_name { - WorldNames::Exactly(name) => vec![self.versioned_name(name)], - WorldNames::AnyOf(names) => { - names.iter().map(|name| self.versioned_name(name)).collect() - } - } + pub fn package_bytes(&self) -> &[u8] { + &self.package_bytes } } diff --git a/crates/environments/src/lib.rs b/crates/environments/src/lib.rs index 09e2ef0757..2997d96417 100644 --- a/crates/environments/src/lib.rs +++ b/crates/environments/src/lib.rs @@ -1,9 +1,9 @@ -use anyhow::{anyhow, Context}; +use anyhow::anyhow; mod environment_definition; mod loader; -use environment_definition::{TargetEnvironment, TargetWorld, TriggerType}; +use environment_definition::{load_environment, TargetEnvironment, TriggerType}; pub use loader::ResolutionContext; use loader::{load_and_resolve_all, ComponentToValidate}; @@ -12,43 +12,11 @@ pub async fn validate_application_against_environment_ids( app: &spin_manifest::schema::v2::AppManifest, resolution_context: &ResolutionContext, ) -> anyhow::Result<()> { - let envs = join_all_result(env_ids.map(resolve_environment_id)).await?; + let envs = join_all_result(env_ids.map(load_environment)).await?; validate_application_against_environments(&envs, app, resolution_context).await } -async fn resolve_environment_id(id: &str) -> anyhow::Result { - let (name, ver) = id.split_once('@').ok_or(anyhow!( - "Target environment '{id}' does not specify a version" - ))?; - let client = oci_distribution::Client::default(); - let auth = oci_distribution::secrets::RegistryAuth::Anonymous; - let env_def_ref = - oci_distribution::Reference::try_from(format!("ghcr.io/itowlson/spinenvs/{name}:{ver}"))?; - let (man, _digest) = client - .pull_manifest(&env_def_ref, &auth) - .await - .with_context(|| format!("Failed to find environment '{id}' in registry"))?; - let im = match man { - oci_distribution::manifest::OciManifest::Image(im) => im, - oci_distribution::manifest::OciManifest::ImageIndex(_ind) => { - anyhow::bail!("Environment '{id}' definition is unusable - stored in registry in incorrect format") - } - }; - let the_layer = &im.layers[0]; - let mut out = Vec::with_capacity(the_layer.size.try_into().unwrap_or_default()); - client - .pull_blob(&env_def_ref, the_layer, &mut out) - .await - .with_context(|| { - format!("Failed to download environment '{id}' definition from registry") - })?; - let te = serde_json::from_slice(&out).with_context(|| { - format!("Failed to load environment '{id}' definition - invalid JSON schema") - })?; - Ok(te) -} - -pub async fn validate_application_against_environments( +async fn validate_application_against_environments( envs: &[TargetEnvironment], app: &spin_manifest::schema::v2::AppManifest, resolution_context: &ResolutionContext, @@ -56,13 +24,10 @@ pub async fn validate_application_against_environments( use futures::FutureExt; for trigger_type in app.triggers.keys() { - if let Some(env) = envs - .iter() - .find(|e| !e.environments.contains_key(trigger_type)) - { + if let Some(env) = envs.iter().find(|e| !e.supports_trigger_type(trigger_type)) { anyhow::bail!( "Environment {} does not support trigger type {trigger_type}", - env.name + env.name() ); } } @@ -87,28 +52,9 @@ async fn validate_component_against_environments( trigger_type: &TriggerType, component: &ComponentToValidate<'_>, ) -> anyhow::Result<()> { - let worlds = envs - .iter() - .map(|e| { - e.environments - .get(trigger_type) - .ok_or(anyhow!( - "Environment '{}' doesn't support trigger type {trigger_type}", - e.name - )) - .map(|w| (e.name.as_str(), w)) - }) - .collect::, _>>()?; - validate_component_against_worlds(worlds.into_iter(), component).await?; - Ok(()) -} - -async fn validate_component_against_worlds( - target_worlds: impl Iterator, - component: &ComponentToValidate<'_>, -) -> anyhow::Result<()> { - for (env_name, target_world) in target_worlds { - validate_wasm_against_any_world(env_name, target_world, component).await?; + for env in envs { + let worlds = env.worlds(trigger_type); + validate_wasm_against_any_world(env, &worlds, component).await?; } tracing::info!( @@ -120,21 +66,21 @@ async fn validate_component_against_worlds( } async fn validate_wasm_against_any_world( - env_name: &str, - target_world: &TargetWorld, + env: &TargetEnvironment, + world_names: &[String], component: &ComponentToValidate<'_>, ) -> anyhow::Result<()> { let mut result = Ok(()); - for target_str in target_world.versioned_names() { - tracing::info!( - "Trying component {} {} against target world {target_str}", + for target_world in world_names { + tracing::debug!( + "Trying component {} {} against target world {target_world}", component.id(), component.source_description(), ); - match validate_wasm_against_world(env_name, &target_str, component).await { + match validate_wasm_against_world(env, target_world, component).await { Ok(()) => { tracing::info!( - "Validated component {} {} against target world {target_str}", + "Validated component {} {} against target world {target_world}", component.id(), component.source_description(), ); @@ -143,7 +89,7 @@ async fn validate_wasm_against_any_world( Err(e) => { // Record the error, but continue in case a different world succeeds tracing::info!( - "Rejecting component {} {} for target world {target_str} because {e:?}", + "Rejecting component {} {} for target world {target_world} because {e:?}", component.id(), component.source_description(), ); @@ -155,34 +101,52 @@ async fn validate_wasm_against_any_world( } async fn validate_wasm_against_world( - env_name: &str, - target_str: &str, + env: &TargetEnvironment, + target_world: &str, component: &ComponentToValidate<'_>, ) -> anyhow::Result<()> { - let comp_name = "root:component"; + // Because we are abusing a composition tool to do validation, we have to + // provide a name by which to refer to the component in the dummy composition. + let component_name = "root:component"; + let component_key = wac_types::BorrowedPackageKey::from_name_and_version(component_name, None); + + // wac is going to get the world from the environment package bytes. + // This constructs a key for that mapping. + let env_pkg_name = env.package_namespaced_name(); + let env_pkg_key = + wac_types::BorrowedPackageKey::from_name_and_version(&env_pkg_name, env.package_version()); + + let env_name = env.name(); let wac_text = format!( r#" - package validate:component@1.0.0 targets {target_str}; - let c = new {comp_name} {{ ... }}; + package validate:component@1.0.0 targets {target_world}; + let c = new {component_name} {{ ... }}; export c...; "# ); let doc = wac_parser::Document::parse(&wac_text)?; - let compkey = wac_types::BorrowedPackageKey::from_name_and_version(comp_name, None); + // TODO: if we end up needing the registry, we need to do this dance + // for things we are providing separately, or the registry will try to + // hoover them up and will fail. + // let mut refpkgs = wac_resolver::packages(&doc)?; + // refpkgs.shift_remove(&env_pkg_key); + // refpkgs.shift_remove(&component_key); - let mut refpkgs = wac_resolver::packages(&doc)?; - refpkgs.retain(|k, _| k != &compkey); + // TODO: determine if this is needed in circumstances other than the simple test + // let reg_resolver = wac_resolver::RegistryPackageResolver::new(Some("wa.dev"), None).await?; + // let mut packages = reg_resolver + // .resolve(&refpkgs) + // .await + // .context("reg_resolver.resolve failed")?; - let reg_resolver = wac_resolver::RegistryPackageResolver::new(Some("wa.dev"), None).await?; - let mut packages = reg_resolver - .resolve(&refpkgs) - .await - .context("reg_resolver.resolve failed")?; + let mut packages: indexmap::IndexMap> = + Default::default(); - packages.insert(compkey, component.wasm_bytes().to_vec()); + packages.insert(env_pkg_key, env.package_bytes().to_vec()); + packages.insert(component_key, component.wasm_bytes().to_vec()); match doc.resolve(packages) { Ok(_) => Ok(()), @@ -195,7 +159,7 @@ async fn validate_wasm_against_world( } Err(wac_parser::resolution::Error::PackageMissingExport { export, .. }) => { // TODO: The export here seems wrong - it seems to contain the world name rather than the interface name - Err(anyhow!("Component {} ({}) can't run in environment {env_name} because world {target_str} requires an export named {export}, which the component does not provide", component.id(), component.source_description())) + Err(anyhow!("Component {} ({}) can't run in environment {env_name} because world {target_world} requires an export named {export}, which the component does not provide", component.id(), component.source_description())) } Err(wac_parser::resolution::Error::ImportNotInTarget { name, world, .. }) => { Err(anyhow!("Component {} ({}) can't run in environment {env_name} because world {world} does not provide an import named {name}, which the component requires", component.id(), component.source_description())) From cadbf91db2104971402002a06715f5cc135bbe06 Mon Sep 17 00:00:00 2001 From: itowlson Date: Tue, 17 Sep 2024 13:29:02 +1200 Subject: [PATCH 04/11] Less meaningless error messages Signed-off-by: itowlson --- .../src/environment_definition.rs | 29 +++++++++++++------ crates/environments/src/lib.rs | 9 ++++-- crates/environments/src/loader.rs | 4 +-- 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/crates/environments/src/environment_definition.rs b/crates/environments/src/environment_definition.rs index 5e576b2cdd..49ea4f91b9 100644 --- a/crates/environments/src/environment_definition.rs +++ b/crates/environments/src/environment_definition.rs @@ -3,25 +3,33 @@ use anyhow::Context; pub async fn load_environment(env_id: &str) -> anyhow::Result { use futures_util::TryStreamExt; - let (pkg_name, pkg_ver) = env_id.split_once('@').unwrap(); + let (pkg_name, pkg_ver) = env_id.split_once('@').with_context(|| format!("Failed to parse target environment {env_id} as package reference - is the target correct?"))?; - let mut client = wasm_pkg_loader::Client::with_global_defaults()?; + // TODO: this requires wkg configuration which shouldn't be on users: + // is there a better way to handle it? + let mut client = wasm_pkg_loader::Client::with_global_defaults() + .context("Failed to create a package loader from your global settings")?; - let package = pkg_name.to_owned().try_into().context("pkg ref parse")?; - let version = wasm_pkg_loader::Version::parse(pkg_ver).context("pkg ver parse")?; + let package = pkg_name + .to_owned() + .try_into() + .with_context(|| format!("Failed to parse environment name {pkg_name} as package name"))?; + let version = wasm_pkg_loader::Version::parse(pkg_ver).with_context(|| { + format!("Failed to parse environment version {pkg_ver} as package version") + })?; let release = client .get_release(&package, &version) .await - .context("get release")?; + .with_context(|| format!("Failed to get {env_id} release from registry"))?; let stm = client .stream_content(&package, &release) .await - .context("stream content")?; + .with_context(|| format!("Failed to get {env_id} package from registry"))?; let bytes = stm .try_collect::() .await - .context("collect stm")? + .with_context(|| format!("Failed to get {env_id} package data from registry"))? .to_vec(); TargetEnvironment::new(env_id.to_owned(), bytes) @@ -37,13 +45,16 @@ pub struct TargetEnvironment { impl TargetEnvironment { fn new(name: String, bytes: Vec) -> anyhow::Result { - let decoded = wit_component::decode(&bytes).context("decode wasm")?; + let decoded = wit_component::decode(&bytes) + .with_context(|| format!("Failed to decode package for environment {name}"))?; let package_id = decoded.package(); let package = decoded .resolve() .packages .get(package_id) - .context("should had a package")? + .with_context(|| { + format!("The {name} environment is invalid (no package for decoded package ID)") + })? .clone(); Ok(Self { diff --git a/crates/environments/src/lib.rs b/crates/environments/src/lib.rs index 2997d96417..6771c3a6ec 100644 --- a/crates/environments/src/lib.rs +++ b/crates/environments/src/lib.rs @@ -1,4 +1,4 @@ -use anyhow::anyhow; +use anyhow::{anyhow, Context}; mod environment_definition; mod loader; @@ -36,7 +36,9 @@ async fn validate_application_against_environments( load_and_resolve_all(app, ts, resolution_context) .map(|css| css.map(|css| (ty.to_owned(), css))) }); - let components_by_trigger_type = join_all_result(components_by_trigger_type_futs).await?; + let components_by_trigger_type = join_all_result(components_by_trigger_type_futs) + .await + .context("Failed to prepare components for target environment checking")?; for (trigger_type, component) in components_by_trigger_type { for component in &component { @@ -126,7 +128,8 @@ async fn validate_wasm_against_world( "# ); - let doc = wac_parser::Document::parse(&wac_text)?; + let doc = wac_parser::Document::parse(&wac_text) + .context("Internal error constructing WAC document for target checking")?; // TODO: if we end up needing the registry, we need to do this dance // for things we are providing separately, or the registry will try to diff --git a/crates/environments/src/loader.rs b/crates/environments/src/loader.rs index 087994f53b..0dc55c31e0 100644 --- a/crates/environments/src/loader.rs +++ b/crates/environments/src/loader.rs @@ -1,6 +1,6 @@ use std::path::Path; -use anyhow::anyhow; +use anyhow::{anyhow, Context}; use spin_common::ui::quoted_path; pub(crate) struct ComponentToValidate<'a> { @@ -73,7 +73,7 @@ async fn load_and_resolve_one<'a>( let loader = ComponentSourceLoader::new(resolution_context.wasm_loader()); - let wasm = spin_compose::compose(&loader, &component).await?; + let wasm = spin_compose::compose(&loader, &component).await.with_context(|| format!("Spin needed to compose dependencies for {id} as part of target checking, but composition failed"))?; Ok(ComponentToValidate { id, From cd1c3ccf27ad3a7f6963fcc9faf77d95855478d4 Mon Sep 17 00:00:00 2001 From: itowlson Date: Thu, 19 Sep 2024 10:25:25 +1200 Subject: [PATCH 05/11] Enable skipping target validation for offline or to bypass delays Signed-off-by: itowlson --- crates/build/src/lib.rs | 7 ++++--- src/commands/build.rs | 8 +++++++- src/commands/registry.rs | 4 ++-- src/commands/up.rs | 2 +- src/commands/up/app_source.rs | 2 +- 5 files changed, 15 insertions(+), 8 deletions(-) diff --git a/crates/build/src/lib.rs b/crates/build/src/lib.rs index 845bebb441..a40bcdb4ef 100644 --- a/crates/build/src/lib.rs +++ b/crates/build/src/lib.rs @@ -17,7 +17,7 @@ use subprocess::{Exec, Redirection}; use crate::manifest::component_build_configs; /// If present, run the build command of each component. -pub async fn build(manifest_file: &Path, component_ids: &[String]) -> Result<()> { +pub async fn build(manifest_file: &Path, component_ids: &[String], skip_target_checks: bool) -> Result<()> { let (components, deployment_targets, manifest) = component_build_configs(manifest_file) .await .with_context(|| { @@ -37,7 +37,8 @@ pub async fn build(manifest_file: &Path, component_ids: &[String]) -> Result<()> build_result?; if let Ok(manifest) = &manifest { - if !deployment_targets.is_empty() { + let should_check_targets = !skip_target_checks && !deployment_targets.is_empty(); + if should_check_targets { let resolution_context = spin_environments::ResolutionContext::new(manifest_file.parent().unwrap()).await?; spin_environments::validate_application_against_environment_ids( @@ -163,6 +164,6 @@ mod tests { #[tokio::test] async fn can_load_even_if_trigger_invalid() { let bad_trigger_file = test_data_root().join("bad_trigger.toml"); - build(&bad_trigger_file, &[]).await.unwrap(); + build(&bad_trigger_file, &[], true).await.unwrap(); } } diff --git a/src/commands/build.rs b/src/commands/build.rs index 16270640e9..2937043e07 100644 --- a/src/commands/build.rs +++ b/src/commands/build.rs @@ -29,6 +29,12 @@ pub struct BuildCommand { #[clap(short = 'c', long, multiple = true)] pub component_id: Vec, + /// By default, if the application manifest specifies one or more deployment targets, Spin + /// checks that all components are compatible with those deployment targets. Specify + /// this option to bypass those target checks. + #[clap(long = "skip-target-checks", alias = "skip-target-check", takes_value = false)] + skip_target_checks: bool, + /// Run the application after building. #[clap(name = BUILD_UP_OPT, short = 'u', long = "up")] pub up: bool, @@ -43,7 +49,7 @@ impl BuildCommand { spin_common::paths::find_manifest_file_path(self.app_source.as_ref())?; notify_if_nondefault_rel(&manifest_file, distance); - spin_build::build(&manifest_file, &self.component_id).await?; + spin_build::build(&manifest_file, &self.component_id, self.skip_target_checks).await?; if self.up { let mut cmd = UpCommand::parse_from( diff --git a/src/commands/registry.rs b/src/commands/registry.rs index a044ba6526..a9ddf21fac 100644 --- a/src/commands/registry.rs +++ b/src/commands/registry.rs @@ -49,7 +49,7 @@ pub struct Push { )] pub insecure: bool, - /// Specifies to perform `spin build` before pushing the application. + /// Specifies to perform `spin build` (with the default options) before pushing the application. #[clap(long, takes_value = false, env = ALWAYS_BUILD_ENV)] pub build: bool, @@ -75,7 +75,7 @@ impl Push { notify_if_nondefault_rel(&app_file, distance); if self.build { - spin_build::build(&app_file, &[]).await?; + spin_build::build(&app_file, &[], false).await?; } let annotations = if self.annotations.is_empty() { diff --git a/src/commands/up.rs b/src/commands/up.rs index 275b54a266..e372f24cd8 100644 --- a/src/commands/up.rs +++ b/src/commands/up.rs @@ -108,7 +108,7 @@ pub struct UpCommand { #[clap(long, takes_value = false)] pub direct_mounts: bool, - /// For local apps, specifies to perform `spin build` before running the application. + /// For local apps, specifies to perform `spin build` (with the default options) before running the application. /// /// This is ignored on remote applications, as they are already built. #[clap(long, takes_value = false, env = ALWAYS_BUILD_ENV)] diff --git a/src/commands/up/app_source.rs b/src/commands/up/app_source.rs index 7969106992..b1934efe13 100644 --- a/src/commands/up/app_source.rs +++ b/src/commands/up/app_source.rs @@ -58,7 +58,7 @@ impl AppSource { pub async fn build(&self) -> anyhow::Result<()> { match self { - Self::File(path) => spin_build::build(path, &[]).await, + Self::File(path) => spin_build::build(path, &[], false).await, _ => Ok(()), } } From 897e4ddb06ccc1f1251e6164878db75e34eb2815 Mon Sep 17 00:00:00 2001 From: itowlson Date: Thu, 19 Sep 2024 11:18:49 +1200 Subject: [PATCH 06/11] Report all target validation fails not just the first Signed-off-by: itowlson --- crates/build/src/deployment.rs | 23 ---------- crates/build/src/lib.rs | 22 ++++++--- crates/build/src/manifest.rs | 25 ++-------- .../src/environment_definition.rs | 4 +- crates/environments/src/lib.rs | 46 +++++++++++++------ src/commands/build.rs | 6 ++- 6 files changed, 60 insertions(+), 66 deletions(-) delete mode 100644 crates/build/src/deployment.rs diff --git a/crates/build/src/deployment.rs b/crates/build/src/deployment.rs deleted file mode 100644 index bfd40bed5e..0000000000 --- a/crates/build/src/deployment.rs +++ /dev/null @@ -1,23 +0,0 @@ -#[derive(Default)] -pub struct DeploymentTargets { - target_environments: Vec, -} -pub type DeploymentTarget = String; - -impl DeploymentTargets { - pub fn new(envs: Vec) -> Self { - Self { - target_environments: envs, - } - } - - pub fn iter(&self) -> impl Iterator { - self.target_environments.iter().map(|s| s.as_str()) - } - - pub fn is_empty(&self) -> bool { - // TODO: it would be nice to let "no-op" behaviour fall out organically, - // but currently we do some stuff eagerly, so... - self.target_environments.is_empty() - } -} diff --git a/crates/build/src/lib.rs b/crates/build/src/lib.rs index a40bcdb4ef..94d2a7d2da 100644 --- a/crates/build/src/lib.rs +++ b/crates/build/src/lib.rs @@ -2,7 +2,6 @@ //! A library for building Spin components. -mod deployment; mod manifest; use anyhow::{anyhow, bail, Context, Result}; @@ -17,7 +16,11 @@ use subprocess::{Exec, Redirection}; use crate::manifest::component_build_configs; /// If present, run the build command of each component. -pub async fn build(manifest_file: &Path, component_ids: &[String], skip_target_checks: bool) -> Result<()> { +pub async fn build( + manifest_file: &Path, + component_ids: &[String], + skip_target_checks: bool, +) -> Result<()> { let (components, deployment_targets, manifest) = component_build_configs(manifest_file) .await .with_context(|| { @@ -37,16 +40,23 @@ pub async fn build(manifest_file: &Path, component_ids: &[String], skip_target_c build_result?; if let Ok(manifest) = &manifest { - let should_check_targets = !skip_target_checks && !deployment_targets.is_empty(); - if should_check_targets { + if !skip_target_checks { let resolution_context = spin_environments::ResolutionContext::new(manifest_file.parent().unwrap()).await?; - spin_environments::validate_application_against_environment_ids( - deployment_targets.iter(), + let errors = spin_environments::validate_application_against_environment_ids( + &deployment_targets, manifest, &resolution_context, ) .await?; + + for error in &errors { + terminal::error!("{error}"); + } + + if !errors.is_empty() { + anyhow::bail!("All components built successfully, but one or more was incompatible with one or more of the deployment targets."); + } } } diff --git a/crates/build/src/manifest.rs b/crates/build/src/manifest.rs index bd17c2e250..db4f08c056 100644 --- a/crates/build/src/manifest.rs +++ b/crates/build/src/manifest.rs @@ -4,7 +4,8 @@ use std::{collections::BTreeMap, path::Path}; use spin_manifest::{schema::v2, ManifestVersion}; -use crate::deployment::DeploymentTargets; +pub type DeploymentTarget = String; +pub type DeploymentTargets = Vec; /// Returns a map of component IDs to [`v2::ComponentBuildConfig`]s for the /// given (v1 or v2) manifest path. If the manifest cannot be loaded, the @@ -49,13 +50,7 @@ fn build_configs_from_manifest( fn deployment_targets_from_manifest( manifest: &spin_manifest::schema::v2::AppManifest, ) -> DeploymentTargets { - let target_environments = manifest.application.targets.clone(); - // let components = manifest - // .components - // .iter() - // .map(|(id, c)| (id.to_string(), c.source.clone())) - // .collect(); - DeploymentTargets::new(target_environments) + manifest.application.targets.clone() } async fn fallback_load_build_configs( @@ -83,13 +78,6 @@ async fn fallback_load_build_configs( async fn fallback_load_deployment_targets( manifest_file: impl AsRef, ) -> Result { - // fn try_parse_component_source(c: (&String, &toml::Value)) -> Option<(String, spin_manifest::schema::v2::ComponentSource)> { - // let (id, ctab) = c; - // let cs = ctab.as_table() - // .and_then(|c| c.get("source")) - // .and_then(|cs| spin_manifest::schema::v2::ComponentSource::deserialize(cs.clone()).ok()); - // cs.map(|cs| (id.to_string(), cs)) - // } let manifest_text = tokio::fs::read_to_string(manifest_file).await?; Ok(match ManifestVersion::detect(&manifest_text)? { ManifestVersion::V1 => Default::default(), @@ -106,12 +94,7 @@ async fn fallback_load_deployment_targets( .filter_map(|t| t.as_str()) .map(|s| s.to_owned()) .collect(); - // let components = table - // .get("component") - // .and_then(|cs| cs.as_table()) - // .map(|table| table.iter().filter_map(try_parse_component_source).collect()) - // .unwrap_or_default(); - DeploymentTargets::new(target_environments) + target_environments } }) } diff --git a/crates/environments/src/environment_definition.rs b/crates/environments/src/environment_definition.rs index 49ea4f91b9..d1c5ebd226 100644 --- a/crates/environments/src/environment_definition.rs +++ b/crates/environments/src/environment_definition.rs @@ -1,8 +1,10 @@ use anyhow::Context; -pub async fn load_environment(env_id: &str) -> anyhow::Result { +pub async fn load_environment(env_id: impl AsRef) -> anyhow::Result { use futures_util::TryStreamExt; + let env_id = env_id.as_ref(); + let (pkg_name, pkg_ver) = env_id.split_once('@').with_context(|| format!("Failed to parse target environment {env_id} as package reference - is the target correct?"))?; // TODO: this requires wkg configuration which shouldn't be on users: diff --git a/crates/environments/src/lib.rs b/crates/environments/src/lib.rs index 6771c3a6ec..a069fbd69e 100644 --- a/crates/environments/src/lib.rs +++ b/crates/environments/src/lib.rs @@ -8,11 +8,15 @@ pub use loader::ResolutionContext; use loader::{load_and_resolve_all, ComponentToValidate}; pub async fn validate_application_against_environment_ids( - env_ids: impl Iterator, + env_ids: &[impl AsRef], app: &spin_manifest::schema::v2::AppManifest, resolution_context: &ResolutionContext, -) -> anyhow::Result<()> { - let envs = join_all_result(env_ids.map(load_environment)).await?; +) -> anyhow::Result> { + if env_ids.is_empty() { + return Ok(Default::default()); + } + + let envs = join_all_result(env_ids.iter().map(load_environment)).await?; validate_application_against_environments(&envs, app, resolution_context).await } @@ -20,7 +24,7 @@ async fn validate_application_against_environments( envs: &[TargetEnvironment], app: &spin_manifest::schema::v2::AppManifest, resolution_context: &ResolutionContext, -) -> anyhow::Result<()> { +) -> anyhow::Result> { use futures::FutureExt; for trigger_type in app.triggers.keys() { @@ -40,31 +44,45 @@ async fn validate_application_against_environments( .await .context("Failed to prepare components for target environment checking")?; + let mut errs = vec![]; + for (trigger_type, component) in components_by_trigger_type { for component in &component { - validate_component_against_environments(envs, &trigger_type, component).await?; + errs.extend( + validate_component_against_environments(envs, &trigger_type, component).await?, + ); } } - Ok(()) + Ok(errs) } async fn validate_component_against_environments( envs: &[TargetEnvironment], trigger_type: &TriggerType, component: &ComponentToValidate<'_>, -) -> anyhow::Result<()> { +) -> anyhow::Result> { + let mut errs = vec![]; + for env in envs { let worlds = env.worlds(trigger_type); - validate_wasm_against_any_world(env, &worlds, component).await?; + if let Some(e) = validate_wasm_against_any_world(env, &worlds, component) + .await + .err() + { + errs.push(e); + } } - tracing::info!( - "Validated component {} {} against all target worlds", - component.id(), - component.source_description() - ); - Ok(()) + if errs.is_empty() { + tracing::info!( + "Validated component {} {} against all target worlds", + component.id(), + component.source_description() + ); + } + + Ok(errs) } async fn validate_wasm_against_any_world( diff --git a/src/commands/build.rs b/src/commands/build.rs index 2937043e07..336dc3287a 100644 --- a/src/commands/build.rs +++ b/src/commands/build.rs @@ -32,7 +32,11 @@ pub struct BuildCommand { /// By default, if the application manifest specifies one or more deployment targets, Spin /// checks that all components are compatible with those deployment targets. Specify /// this option to bypass those target checks. - #[clap(long = "skip-target-checks", alias = "skip-target-check", takes_value = false)] + #[clap( + long = "skip-target-checks", + alias = "skip-target-check", + takes_value = false + )] skip_target_checks: bool, /// Run the application after building. From bc2cc175ba0e11178e01bd7a1eccc6eb79059169 Mon Sep 17 00:00:00 2001 From: itowlson Date: Mon, 23 Sep 2024 14:55:22 +1200 Subject: [PATCH 07/11] Updated after initial review Signed-off-by: itowlson --- Cargo.lock | 1 - crates/build/src/lib.rs | 65 ++++++---- crates/build/src/manifest.rs | 116 +++++++++++++----- crates/compose/src/lib.rs | 4 + crates/environments/Cargo.toml | 9 +- .../src/environment_definition.rs | 27 +++- crates/environments/src/lib.rs | 17 +-- crates/environments/src/loader.rs | 3 +- 8 files changed, 162 insertions(+), 80 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d39a58a3ea..cd55a8108c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7165,7 +7165,6 @@ dependencies = [ "futures-util", "id-arena", "indexmap 2.5.0", - "miette 7.2.0", "oci-distribution 0.11.0 (git+https://github.com/fermyon/oci-distribution?rev=7e4ce9be9bcd22e78a28f06204931f10c44402ba)", "semver", "serde 1.0.210", diff --git a/crates/build/src/lib.rs b/crates/build/src/lib.rs index 94d2a7d2da..9e07d79ff3 100644 --- a/crates/build/src/lib.rs +++ b/crates/build/src/lib.rs @@ -21,42 +21,59 @@ pub async fn build( component_ids: &[String], skip_target_checks: bool, ) -> Result<()> { - let (components, deployment_targets, manifest) = component_build_configs(manifest_file) + let build_info = component_build_configs(manifest_file) .await .with_context(|| { - format!( - "Cannot read manifest file from {}", - quoted_path(manifest_file) - ) - })?; + format!( + "Cannot read manifest file from {}", + quoted_path(manifest_file) + ) + })?; let app_dir = parent_dir(manifest_file)?; - let build_result = build_components(component_ids, components, app_dir); + let build_result = build_components(component_ids, build_info.components(), app_dir); - if let Err(e) = &manifest { + // Emit any required warnings now, so that they don't bury any errors. + if let Some(e) = build_info.load_error() { + // The manifest had errors. We managed to attempt a build anyway, but we want to + // let the user know about them. terminal::warn!("The manifest has errors not related to the Wasm component build. Error details:\n{e:#}"); + // Checking deployment targets requires a healthy manifest (because trigger types etc.), + // if any of these were specified, warn they are being skipped. + let should_have_checked_targets = + !skip_target_checks && build_info.has_deployment_targets(); + if should_have_checked_targets { + terminal::warn!( + "The manifest error(s) prevented Spin from checking the deployment targets." + ); + } } + // If the build failed, exit with an error at this point. build_result?; - if let Ok(manifest) = &manifest { - if !skip_target_checks { - let resolution_context = - spin_environments::ResolutionContext::new(manifest_file.parent().unwrap()).await?; - let errors = spin_environments::validate_application_against_environment_ids( - &deployment_targets, - manifest, - &resolution_context, - ) - .await?; + let Some(manifest) = build_info.manifest() else { + // We can't proceed to checking (because that needs a full healthy manifest), and we've + // already emitted any necessary warning, so quit. + return Ok(()); + }; - for error in &errors { - terminal::error!("{error}"); - } + if !skip_target_checks { + let resolution_context = + spin_environments::ResolutionContext::new(manifest_file.parent().unwrap()).await?; + let errors = spin_environments::validate_application_against_environment_ids( + build_info.deployment_targets(), + manifest, + &resolution_context, + ) + .await?; - if !errors.is_empty() { - anyhow::bail!("All components built successfully, but one or more was incompatible with one or more of the deployment targets."); - } + for error in &errors { + terminal::error!("{error}"); + } + + if !errors.is_empty() { + anyhow::bail!("All components built successfully, but one or more was incompatible with one or more of the deployment targets."); } } diff --git a/crates/build/src/manifest.rs b/crates/build/src/manifest.rs index db4f08c056..a4c295b0e5 100644 --- a/crates/build/src/manifest.rs +++ b/crates/build/src/manifest.rs @@ -4,32 +4,96 @@ use std::{collections::BTreeMap, path::Path}; use spin_manifest::{schema::v2, ManifestVersion}; -pub type DeploymentTarget = String; -pub type DeploymentTargets = Vec; +pub enum ManifestBuildInfo { + Loadable { + components: Vec, + deployment_targets: Vec, + manifest: spin_manifest::schema::v2::AppManifest, + }, + Unloadable { + components: Vec, + has_deployment_targets: bool, + load_error: spin_manifest::Error, + }, +} + +impl ManifestBuildInfo { + pub fn components(&self) -> Vec { + match self { + Self::Loadable { components, .. } => components.clone(), + Self::Unloadable { components, .. } => components.clone(), + } + } + + pub fn load_error(&self) -> Option<&spin_manifest::Error> { + match self { + Self::Loadable { .. } => None, + Self::Unloadable { load_error, .. } => Some(load_error), + } + } + + pub fn deployment_targets(&self) -> &[String] { + match self { + Self::Loadable { + deployment_targets, .. + } => deployment_targets, + Self::Unloadable { .. } => &[], + } + } + + pub fn has_deployment_targets(&self) -> bool { + match self { + Self::Loadable { + deployment_targets, .. + } => !deployment_targets.is_empty(), + Self::Unloadable { + has_deployment_targets, + .. + } => *has_deployment_targets, + } + } + + pub fn manifest(&self) -> Option<&spin_manifest::schema::v2::AppManifest> { + match self { + Self::Loadable { manifest, .. } => Some(manifest), + Self::Unloadable { .. } => None, + } + } +} /// Returns a map of component IDs to [`v2::ComponentBuildConfig`]s for the /// given (v1 or v2) manifest path. If the manifest cannot be loaded, the /// function attempts fallback: if fallback succeeds, result is Ok but the load error /// is also returned via the second part of the return value tuple. -pub async fn component_build_configs( - manifest_file: impl AsRef, -) -> Result<( - Vec, - DeploymentTargets, - Result, -)> { +pub async fn component_build_configs(manifest_file: impl AsRef) -> Result { let manifest = spin_manifest::manifest_from_file(&manifest_file); match manifest { Ok(mut manifest) => { spin_manifest::normalize::normalize_manifest(&mut manifest); - let bc = build_configs_from_manifest(&manifest); - let dt = deployment_targets_from_manifest(&manifest); - Ok((bc, dt, Ok(manifest))) + let components = build_configs_from_manifest(&manifest); + let deployment_targets = deployment_targets_from_manifest(&manifest); + Ok(ManifestBuildInfo::Loadable { + components, + deployment_targets, + manifest, + }) } - Err(e) => { - let bc = fallback_load_build_configs(&manifest_file).await?; - let dt = fallback_load_deployment_targets(&manifest_file).await?; - Ok((bc, dt, Err(e))) + Err(load_error) => { + // The manifest didn't load, but the problem might not be build-affecting. + // Try to fall back by parsing out only the bits we need. And if something + // goes wrong with the fallback, give up and return the original manifest load + // error. + let Ok(components) = fallback_load_build_configs(&manifest_file).await else { + return Err(load_error.into()); + }; + let Ok(has_deployment_targets) = has_deployment_targets(&manifest_file).await else { + return Err(load_error.into()); + }; + Ok(ManifestBuildInfo::Unloadable { + components, + has_deployment_targets, + load_error, + }) } } } @@ -49,7 +113,7 @@ fn build_configs_from_manifest( fn deployment_targets_from_manifest( manifest: &spin_manifest::schema::v2::AppManifest, -) -> DeploymentTargets { +) -> Vec { manifest.application.targets.clone() } @@ -75,31 +139,23 @@ async fn fallback_load_build_configs( }) } -async fn fallback_load_deployment_targets( - manifest_file: impl AsRef, -) -> Result { +async fn has_deployment_targets(manifest_file: impl AsRef) -> Result { let manifest_text = tokio::fs::read_to_string(manifest_file).await?; Ok(match ManifestVersion::detect(&manifest_text)? { - ManifestVersion::V1 => Default::default(), + ManifestVersion::V1 => false, ManifestVersion::V2 => { let table: toml::value::Table = toml::from_str(&manifest_text)?; - let target_environments = table + table .get("application") .and_then(|a| a.as_table()) .and_then(|t| t.get("targets")) .and_then(|arr| arr.as_array()) - .map(|v| v.as_slice()) - .unwrap_or_default() - .iter() - .filter_map(|t| t.as_str()) - .map(|s| s.to_owned()) - .collect(); - target_environments + .is_some_and(|arr| !arr.is_empty()) } }) } -#[derive(Deserialize)] +#[derive(Clone, Deserialize)] pub struct ComponentBuildInfo { #[serde(default)] pub id: String, diff --git a/crates/compose/src/lib.rs b/crates/compose/src/lib.rs index efaa8163f1..18d3f79520 100644 --- a/crates/compose/src/lib.rs +++ b/crates/compose/src/lib.rs @@ -33,6 +33,8 @@ pub async fn compose<'a, L: ComponentSourceLoader>( Composer::new(loader).compose(component).await } +/// A Spin component dependency. This abstracts over the metadata associated with the +/// dependency. The abstraction allows both manifest and lockfile types to participate in composition. #[async_trait::async_trait] pub trait DependencyLike { fn inherit(&self) -> InheritConfiguration; @@ -44,6 +46,8 @@ pub enum InheritConfiguration { Some(Vec), } +/// A Spin component. This abstracts over the list of dependencies for the component. +/// The abstraction allows both manifest and lockfile types to participate in composition. #[async_trait::async_trait] pub trait ComponentLike { type Dependency: DependencyLike; diff --git a/crates/environments/Cargo.toml b/crates/environments/Cargo.toml index daea8209ed..90e2078e97 100644 --- a/crates/environments/Cargo.toml +++ b/crates/environments/Cargo.toml @@ -11,12 +11,11 @@ bytes = "1.1" futures = "0.3" futures-util = "0.3" id-arena = "2" -indexmap = "2.2.6" -miette = "7.2.0" +indexmap = "2" oci-distribution = { git = "https://github.com/fermyon/oci-distribution", rev = "7e4ce9be9bcd22e78a28f06204931f10c44402ba" } -semver = "1.0" -serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" +semver = "1" +serde = { version = "1", features = ["derive"] } +serde_json = "1" spin-common = { path = "../common" } spin-componentize = { path = "../componentize" } spin-compose = { path = "../compose" } diff --git a/crates/environments/src/environment_definition.rs b/crates/environments/src/environment_definition.rs index d1c5ebd226..60915ee225 100644 --- a/crates/environments/src/environment_definition.rs +++ b/crates/environments/src/environment_definition.rs @@ -1,5 +1,6 @@ use anyhow::Context; +/// Loads the given `TargetEnvironment` from a registry. pub async fn load_environment(env_id: impl AsRef) -> anyhow::Result { use futures_util::TryStreamExt; @@ -37,10 +38,20 @@ pub async fn load_environment(env_id: impl AsRef) -> anyhow::Result, package_bytes: Vec, } @@ -68,11 +79,14 @@ impl TargetEnvironment { }) } + /// Returns true if the given trigger type provides the world identified by + /// `world` in this environment. pub fn is_world_for(&self, trigger_type: &TriggerType, world: &wit_parser::World) -> bool { world.name.starts_with(&format!("trigger-{trigger_type}")) && world.package.is_some_and(|p| p == self.package_id) } + /// Returns true if the given trigger type can run in this environment. pub fn supports_trigger_type(&self, trigger_type: &TriggerType) -> bool { self.decoded .resolve() @@ -81,6 +95,7 @@ impl TargetEnvironment { .any(|(_, world)| self.is_world_for(trigger_type, world)) } + /// Lists all worlds supported for the given trigger type in this environment. pub fn worlds(&self, trigger_type: &TriggerType) -> Vec { self.decoded .resolve() @@ -93,10 +108,10 @@ impl TargetEnvironment { /// Fully qualified world name (e.g. fermyon:spin/http-trigger@2.0.0) fn world_qname(&self, world: &wit_parser::World) -> String { - let version_suffix = match self.package_version() { - Some(version) => format!("@{version}"), - None => "".to_owned(), - }; + let version_suffix = self + .package_version() + .map(|version| format!("@{version}")) + .unwrap_or_default(); format!( "{}/{}{version_suffix}", self.package_namespaced_name(), @@ -114,10 +129,12 @@ impl TargetEnvironment { format!("{}:{}", self.package.name.namespace, self.package.name.name) } + /// The package version for the environment package. pub fn package_version(&self) -> Option<&semver::Version> { self.package.name.version.as_ref() } + /// The Wasm-encoded bytes of the environment package. pub fn package_bytes(&self) -> &[u8] { &self.package_bytes } diff --git a/crates/environments/src/lib.rs b/crates/environments/src/lib.rs index a069fbd69e..c5a232a01c 100644 --- a/crates/environments/src/lib.rs +++ b/crates/environments/src/lib.rs @@ -4,6 +4,7 @@ mod environment_definition; mod loader; use environment_definition::{load_environment, TargetEnvironment, TriggerType}; +use futures::future::try_join_all; pub use loader::ResolutionContext; use loader::{load_and_resolve_all, ComponentToValidate}; @@ -16,7 +17,7 @@ pub async fn validate_application_against_environment_ids( return Ok(Default::default()); } - let envs = join_all_result(env_ids.iter().map(load_environment)).await?; + let envs = try_join_all(env_ids.iter().map(load_environment)).await?; validate_application_against_environments(&envs, app, resolution_context).await } @@ -40,7 +41,7 @@ async fn validate_application_against_environments( load_and_resolve_all(app, ts, resolution_context) .map(|css| css.map(|css| (ty.to_owned(), css))) }); - let components_by_trigger_type = join_all_result(components_by_trigger_type_futs) + let components_by_trigger_type = try_join_all(components_by_trigger_type_futs) .await .context("Failed to prepare components for target environment checking")?; @@ -190,15 +191,3 @@ async fn validate_wasm_against_world( }, } } - -/// Equivalent to futures::future::join_all, but specialised for iterators of -/// fallible futures. It returns a Result> instead of a Vec> - -/// this just moves the transposition boilerplate out of the main flow. -async fn join_all_result(iter: I) -> anyhow::Result> -where - I: IntoIterator, - I::Item: std::future::Future>, -{ - let vec_result = futures::future::join_all(iter).await; - vec_result.into_iter().collect() -} diff --git a/crates/environments/src/loader.rs b/crates/environments/src/loader.rs index 0dc55c31e0..458f8f4ca1 100644 --- a/crates/environments/src/loader.rs +++ b/crates/environments/src/loader.rs @@ -1,6 +1,7 @@ use std::path::Path; use anyhow::{anyhow, Context}; +use futures::future::try_join_all; use spin_common::ui::quoted_path; pub(crate) struct ComponentToValidate<'a> { @@ -37,7 +38,7 @@ pub async fn load_and_resolve_all<'a>( let component_futures = triggers .iter() .map(|t| load_and_resolve_one(app, t, resolution_context)); - crate::join_all_result(component_futures).await + try_join_all(component_futures).await } async fn load_and_resolve_one<'a>( From 68fbd1a2e4c6b4808af6dec56cb97f6ea5209966 Mon Sep 17 00:00:00 2001 From: itowlson Date: Mon, 23 Sep 2024 16:39:40 +1200 Subject: [PATCH 08/11] Encapsulate the application and loader Signed-off-by: itowlson --- crates/build/src/lib.rs | 10 +- crates/environments/src/lib.rs | 24 ++--- crates/environments/src/loader.rs | 159 ++++++++++++++++++------------ 3 files changed, 108 insertions(+), 85 deletions(-) diff --git a/crates/build/src/lib.rs b/crates/build/src/lib.rs index 9e07d79ff3..037a27bd9c 100644 --- a/crates/build/src/lib.rs +++ b/crates/build/src/lib.rs @@ -59,12 +59,14 @@ pub async fn build( }; if !skip_target_checks { - let resolution_context = - spin_environments::ResolutionContext::new(manifest_file.parent().unwrap()).await?; + let application = spin_environments::ApplicationToValidate::new( + manifest.clone(), + manifest_file.parent().unwrap(), + ) + .await?; let errors = spin_environments::validate_application_against_environment_ids( + &application, build_info.deployment_targets(), - manifest, - &resolution_context, ) .await?; diff --git a/crates/environments/src/lib.rs b/crates/environments/src/lib.rs index c5a232a01c..657bf94c0e 100644 --- a/crates/environments/src/lib.rs +++ b/crates/environments/src/lib.rs @@ -5,30 +5,26 @@ mod loader; use environment_definition::{load_environment, TargetEnvironment, TriggerType}; use futures::future::try_join_all; -pub use loader::ResolutionContext; -use loader::{load_and_resolve_all, ComponentToValidate}; +pub use loader::ApplicationToValidate; +use loader::ComponentToValidate; pub async fn validate_application_against_environment_ids( + application: &ApplicationToValidate, env_ids: &[impl AsRef], - app: &spin_manifest::schema::v2::AppManifest, - resolution_context: &ResolutionContext, ) -> anyhow::Result> { if env_ids.is_empty() { return Ok(Default::default()); } let envs = try_join_all(env_ids.iter().map(load_environment)).await?; - validate_application_against_environments(&envs, app, resolution_context).await + validate_application_against_environments(application, &envs).await } async fn validate_application_against_environments( + application: &ApplicationToValidate, envs: &[TargetEnvironment], - app: &spin_manifest::schema::v2::AppManifest, - resolution_context: &ResolutionContext, ) -> anyhow::Result> { - use futures::FutureExt; - - for trigger_type in app.triggers.keys() { + for trigger_type in application.trigger_types() { if let Some(env) = envs.iter().find(|e| !e.supports_trigger_type(trigger_type)) { anyhow::bail!( "Environment {} does not support trigger type {trigger_type}", @@ -37,13 +33,7 @@ async fn validate_application_against_environments( } } - let components_by_trigger_type_futs = app.triggers.iter().map(|(ty, ts)| { - load_and_resolve_all(app, ts, resolution_context) - .map(|css| css.map(|css| (ty.to_owned(), css))) - }); - let components_by_trigger_type = try_join_all(components_by_trigger_type_futs) - .await - .context("Failed to prepare components for target environment checking")?; + let components_by_trigger_type = application.components_by_trigger_type().await?; let mut errs = vec![]; diff --git a/crates/environments/src/loader.rs b/crates/environments/src/loader.rs index 458f8f4ca1..a039ba80b4 100644 --- a/crates/environments/src/loader.rs +++ b/crates/environments/src/loader.rs @@ -10,12 +10,6 @@ pub(crate) struct ComponentToValidate<'a> { wasm: Vec, } -struct ComponentSource<'a> { - id: &'a str, - source: &'a spin_manifest::schema::v2::ComponentSource, - dependencies: WrappedComponentDependencies, -} - impl<'a> ComponentToValidate<'a> { pub fn id(&self) -> &str { self.id @@ -30,75 +24,112 @@ impl<'a> ComponentToValidate<'a> { } } -pub async fn load_and_resolve_all<'a>( - app: &'a spin_manifest::schema::v2::AppManifest, - triggers: &'a [spin_manifest::schema::v2::Trigger], - resolution_context: &'a ResolutionContext, -) -> anyhow::Result>> { - let component_futures = triggers - .iter() - .map(|t| load_and_resolve_one(app, t, resolution_context)); - try_join_all(component_futures).await +pub struct ApplicationToValidate { + manifest: spin_manifest::schema::v2::AppManifest, + wasm_loader: spin_loader::WasmLoader, } -async fn load_and_resolve_one<'a>( - app: &'a spin_manifest::schema::v2::AppManifest, - trigger: &'a spin_manifest::schema::v2::Trigger, - resolution_context: &'a ResolutionContext, -) -> anyhow::Result> { - let component_spec = trigger - .component - .as_ref() - .ok_or_else(|| anyhow!("No component specified for trigger {}", trigger.id))?; - let (id, source, dependencies) = match component_spec { - spin_manifest::schema::v2::ComponentSpec::Inline(c) => { - (trigger.id.as_str(), &c.source, &c.dependencies) - } - spin_manifest::schema::v2::ComponentSpec::Reference(r) => { - let id = r.as_ref(); - let Some(component) = app.components.get(r) else { - anyhow::bail!( - "Component {id} specified for trigger {} does not exist", - trigger.id - ); - }; - (id, &component.source, &component.dependencies) - } - }; - - let component = ComponentSource { - id, - source, - dependencies: WrappedComponentDependencies::new(dependencies), - }; +impl ApplicationToValidate { + pub async fn new( + manifest: spin_manifest::schema::v2::AppManifest, + base_dir: impl AsRef, + ) -> anyhow::Result { + let wasm_loader = + spin_loader::WasmLoader::new(base_dir.as_ref().to_owned(), None, None).await?; + Ok(Self { + manifest, + wasm_loader, + }) + } - let loader = ComponentSourceLoader::new(resolution_context.wasm_loader()); + fn component_source<'a>( + &'a self, + trigger: &'a spin_manifest::schema::v2::Trigger, + ) -> anyhow::Result> { + let component_spec = trigger + .component + .as_ref() + .ok_or_else(|| anyhow!("No component specified for trigger {}", trigger.id))?; + let (id, source, dependencies) = match component_spec { + spin_manifest::schema::v2::ComponentSpec::Inline(c) => { + (trigger.id.as_str(), &c.source, &c.dependencies) + } + spin_manifest::schema::v2::ComponentSpec::Reference(r) => { + let id = r.as_ref(); + let Some(component) = self.manifest.components.get(r) else { + anyhow::bail!( + "Component {id} specified for trigger {} does not exist", + trigger.id + ); + }; + (id, &component.source, &component.dependencies) + } + }; + + Ok(ComponentSource { + id, + source, + dependencies: WrappedComponentDependencies::new(dependencies), + }) + } - let wasm = spin_compose::compose(&loader, &component).await.with_context(|| format!("Spin needed to compose dependencies for {id} as part of target checking, but composition failed"))?; + pub fn trigger_types(&self) -> impl Iterator { + self.manifest.triggers.keys() + } - Ok(ComponentToValidate { - id, - source_description: source_description(component.source), - wasm, - }) -} + pub fn triggers( + &self, + ) -> impl Iterator)> { + self.manifest.triggers.iter() + } -pub struct ResolutionContext { - wasm_loader: spin_loader::WasmLoader, -} + pub(crate) async fn components_by_trigger_type( + &self, + ) -> anyhow::Result>)>> { + use futures::FutureExt; + + let components_by_trigger_type_futs = self.triggers().map(|(ty, ts)| { + self.components_for_trigger(ts) + .map(|css| css.map(|css| (ty.to_owned(), css))) + }); + let components_by_trigger_type = try_join_all(components_by_trigger_type_futs) + .await + .context("Failed to prepare components for target environment checking")?; + Ok(components_by_trigger_type) + } -impl ResolutionContext { - pub async fn new(base_dir: impl AsRef) -> anyhow::Result { - let wasm_loader = - spin_loader::WasmLoader::new(base_dir.as_ref().to_owned(), None, None).await?; - Ok(Self { wasm_loader }) + async fn components_for_trigger<'a>( + &'a self, + triggers: &'a [spin_manifest::schema::v2::Trigger], + ) -> anyhow::Result>> { + let component_futures = triggers.iter().map(|t| self.load_and_resolve_trigger(t)); + try_join_all(component_futures).await } - fn wasm_loader(&self) -> &spin_loader::WasmLoader { - &self.wasm_loader + async fn load_and_resolve_trigger<'a>( + &'a self, + trigger: &'a spin_manifest::schema::v2::Trigger, + ) -> anyhow::Result> { + let component = self.component_source(trigger)?; + + let loader = ComponentSourceLoader::new(&self.wasm_loader); + + let wasm = spin_compose::compose(&loader, &component).await.with_context(|| format!("Spin needed to compose dependencies for {} as part of target checking, but composition failed", component.id))?; + + Ok(ComponentToValidate { + id: component.id, + source_description: source_description(component.source), + wasm, + }) } } +struct ComponentSource<'a> { + id: &'a str, + source: &'a spin_manifest::schema::v2::ComponentSource, + dependencies: WrappedComponentDependencies, +} + struct ComponentSourceLoader<'a> { wasm_loader: &'a spin_loader::WasmLoader, } @@ -135,7 +166,7 @@ impl<'a> spin_compose::ComponentSourceLoader for ComponentSourceLoader<'a> { } // This exists only to thwart the orphan rule -pub(crate) struct WrappedComponentDependency { +struct WrappedComponentDependency { name: spin_serde::DependencyName, dependency: spin_manifest::schema::v2::ComponentDependency, } From 76cb8af41e24c3db0685a1cace5d4841b5f5047b Mon Sep 17 00:00:00 2001 From: itowlson Date: Tue, 24 Sep 2024 15:31:37 +1200 Subject: [PATCH 09/11] Load target env from other registry or local dir Signed-off-by: itowlson --- crates/build/src/manifest.rs | 6 +- .../src/environment_definition.rs | 82 +++++++++++++++++-- crates/environments/src/lib.rs | 3 +- crates/manifest/src/schema/v2.rs | 23 +++++- 4 files changed, 102 insertions(+), 12 deletions(-) diff --git a/crates/build/src/manifest.rs b/crates/build/src/manifest.rs index a4c295b0e5..b28d6547ef 100644 --- a/crates/build/src/manifest.rs +++ b/crates/build/src/manifest.rs @@ -7,7 +7,7 @@ use spin_manifest::{schema::v2, ManifestVersion}; pub enum ManifestBuildInfo { Loadable { components: Vec, - deployment_targets: Vec, + deployment_targets: Vec, manifest: spin_manifest::schema::v2::AppManifest, }, Unloadable { @@ -32,7 +32,7 @@ impl ManifestBuildInfo { } } - pub fn deployment_targets(&self) -> &[String] { + pub fn deployment_targets(&self) -> &[spin_manifest::schema::v2::TargetEnvironmentRef] { match self { Self::Loadable { deployment_targets, .. @@ -113,7 +113,7 @@ fn build_configs_from_manifest( fn deployment_targets_from_manifest( manifest: &spin_manifest::schema::v2::AppManifest, -) -> Vec { +) -> Vec { manifest.application.targets.clone() } diff --git a/crates/environments/src/environment_definition.rs b/crates/environments/src/environment_definition.rs index 60915ee225..613d7a684a 100644 --- a/crates/environments/src/environment_definition.rs +++ b/crates/environments/src/environment_definition.rs @@ -1,17 +1,46 @@ +use std::path::Path; + use anyhow::Context; +use spin_common::ui::quoted_path; +use spin_manifest::schema::v2::TargetEnvironmentRef; + +const DEFAULT_REGISTRY: &str = "fermyon.com"; /// Loads the given `TargetEnvironment` from a registry. -pub async fn load_environment(env_id: impl AsRef) -> anyhow::Result { - use futures_util::TryStreamExt; +pub async fn load_environment(env_id: &TargetEnvironmentRef) -> anyhow::Result { + match env_id { + TargetEnvironmentRef::DefaultRegistry(package) => { + load_environment_from_registry(DEFAULT_REGISTRY, package).await + } + TargetEnvironmentRef::Registry { registry, package } => { + load_environment_from_registry(registry, package).await + } + TargetEnvironmentRef::WitDirectory { path } => load_environment_from_dir(path), + } +} - let env_id = env_id.as_ref(); +async fn load_environment_from_registry( + registry: &str, + env_id: &str, +) -> anyhow::Result { + use futures_util::TryStreamExt; let (pkg_name, pkg_ver) = env_id.split_once('@').with_context(|| format!("Failed to parse target environment {env_id} as package reference - is the target correct?"))?; + let env_pkg_ref: wasm_pkg_loader::PackageRef = pkg_name + .parse() + .with_context(|| format!("Environment {pkg_name} is not a valid package name"))?; + + let registry: wasm_pkg_loader::Registry = registry + .parse() + .with_context(|| format!("Registry {registry} is not a valid registry name"))?; // TODO: this requires wkg configuration which shouldn't be on users: // is there a better way to handle it? - let mut client = wasm_pkg_loader::Client::with_global_defaults() - .context("Failed to create a package loader from your global settings")?; + let mut wkg_config = wasm_pkg_loader::Config::global_defaults() + .unwrap_or_else(|_| wasm_pkg_loader::Config::empty()); + wkg_config.set_package_registry_override(env_pkg_ref, registry); + + let mut client = wasm_pkg_loader::Client::new(wkg_config); let package = pkg_name .to_owned() @@ -35,7 +64,14 @@ pub async fn load_environment(env_id: impl AsRef) -> anyhow::Result anyhow::Result { + let mut resolve = wit_parser::Resolve::default(); + let (pkg_id, _) = resolve.push_dir(path)?; + let decoded = wit_parser::decoding::DecodedWasm::WitPackage(resolve, pkg_id); + TargetEnvironment::from_decoded_wasm(path, decoded) } /// A parsed document representing a deployment environment, e.g. Spin 2.7, @@ -57,7 +93,7 @@ pub struct TargetEnvironment { } impl TargetEnvironment { - fn new(name: String, bytes: Vec) -> anyhow::Result { + fn from_package_bytes(name: String, bytes: Vec) -> anyhow::Result { let decoded = wit_component::decode(&bytes) .with_context(|| format!("Failed to decode package for environment {name}"))?; let package_id = decoded.package(); @@ -79,6 +115,38 @@ impl TargetEnvironment { }) } + fn from_decoded_wasm( + source: &Path, + decoded: wit_parser::decoding::DecodedWasm, + ) -> anyhow::Result { + let package_id = decoded.package(); + let package = decoded + .resolve() + .packages + .get(package_id) + .with_context(|| { + format!( + "The {} environment is invalid (no package for decoded package ID)", + quoted_path(source) + ) + })? + .clone(); + let name = package.name.to_string(); + + // This versionm of wit_component requires a flag for v2 encoding. + // v1 encoding is retired in wit_component main. You can remove the + // flag when this breaks next time we upgrade the crate! + let bytes = wit_component::encode(Some(true), decoded.resolve(), package_id)?; + + Ok(Self { + name, + decoded, + package, + package_id, + package_bytes: bytes, + }) + } + /// Returns true if the given trigger type provides the world identified by /// `world` in this environment. pub fn is_world_for(&self, trigger_type: &TriggerType, world: &wit_parser::World) -> bool { diff --git a/crates/environments/src/lib.rs b/crates/environments/src/lib.rs index 657bf94c0e..0afb033cce 100644 --- a/crates/environments/src/lib.rs +++ b/crates/environments/src/lib.rs @@ -7,10 +7,11 @@ use environment_definition::{load_environment, TargetEnvironment, TriggerType}; use futures::future::try_join_all; pub use loader::ApplicationToValidate; use loader::ComponentToValidate; +use spin_manifest::schema::v2::TargetEnvironmentRef; pub async fn validate_application_against_environment_ids( application: &ApplicationToValidate, - env_ids: &[impl AsRef], + env_ids: &[TargetEnvironmentRef], ) -> anyhow::Result> { if env_ids.is_empty() { return Ok(Default::default()); diff --git a/crates/manifest/src/schema/v2.rs b/crates/manifest/src/schema/v2.rs index ffe510fd32..f7fbdb8b25 100644 --- a/crates/manifest/src/schema/v2.rs +++ b/crates/manifest/src/schema/v2.rs @@ -58,7 +58,7 @@ pub struct AppDetails { pub authors: Vec, /// `targets = ["spin-2.5", "fermyon-cloud", "spinkube-0.4"]` #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub targets: Vec, + pub targets: Vec, /// `[application.triggers.]` #[serde(rename = "trigger", default, skip_serializing_if = "Map::is_empty")] pub trigger_global_configs: Map, @@ -340,6 +340,27 @@ impl ComponentDependencies { } } +/// Identifies a deployment target. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(untagged, deny_unknown_fields)] +pub enum TargetEnvironmentRef { + /// Environment package reference e.g. `spin:cli@3.0`. This is looked up + /// in the default environment registry. + DefaultRegistry(String), + /// A target package in a registry other than the default + Registry { + /// Registry hosting the environment package e.g. `fermyon.com``. + registry: String, + /// Environment package reference e.g. `my:spin-env@1.2`. + package: String, + }, + /// A filesystem directory. This is expected to contain a WIT package. + WitDirectory { + /// The directory containing the environment WIT. + path: PathBuf, + }, +} + mod kebab_or_snake_case { use serde::{Deserialize, Serialize}; pub use spin_serde::{KebabId, SnakeId}; From 2a3a213b8f97860a6442e1bdebc6fe64c257e2c1 Mon Sep 17 00:00:00 2001 From: itowlson Date: Fri, 4 Oct 2024 14:01:16 +1300 Subject: [PATCH 10/11] Cache environment definitions Signed-off-by: itowlson --- crates/build/src/lib.rs | 4 +- .../src/environment_definition.rs | 30 ++++++++--- crates/environments/src/lib.rs | 7 ++- crates/loader/src/cache.rs | 52 +++++++++++++++++++ src/commands/build.rs | 8 ++- src/commands/registry.rs | 2 +- src/commands/up.rs | 2 +- src/commands/up/app_source.rs | 4 +- 8 files changed, 94 insertions(+), 15 deletions(-) diff --git a/crates/build/src/lib.rs b/crates/build/src/lib.rs index 037a27bd9c..5b285ad248 100644 --- a/crates/build/src/lib.rs +++ b/crates/build/src/lib.rs @@ -20,6 +20,7 @@ pub async fn build( manifest_file: &Path, component_ids: &[String], skip_target_checks: bool, + cache_root: Option, ) -> Result<()> { let build_info = component_build_configs(manifest_file) .await @@ -67,6 +68,7 @@ pub async fn build( let errors = spin_environments::validate_application_against_environment_ids( &application, build_info.deployment_targets(), + cache_root.clone(), ) .await?; @@ -193,6 +195,6 @@ mod tests { #[tokio::test] async fn can_load_even_if_trigger_invalid() { let bad_trigger_file = test_data_root().join("bad_trigger.toml"); - build(&bad_trigger_file, &[], true).await.unwrap(); + build(&bad_trigger_file, &[], true, None).await.unwrap(); } } diff --git a/crates/environments/src/environment_definition.rs b/crates/environments/src/environment_definition.rs index 613d7a684a..18b50d7726 100644 --- a/crates/environments/src/environment_definition.rs +++ b/crates/environments/src/environment_definition.rs @@ -7,13 +7,16 @@ use spin_manifest::schema::v2::TargetEnvironmentRef; const DEFAULT_REGISTRY: &str = "fermyon.com"; /// Loads the given `TargetEnvironment` from a registry. -pub async fn load_environment(env_id: &TargetEnvironmentRef) -> anyhow::Result { +pub async fn load_environment( + env_id: &TargetEnvironmentRef, + cache: &spin_loader::cache::Cache, +) -> anyhow::Result { match env_id { TargetEnvironmentRef::DefaultRegistry(package) => { - load_environment_from_registry(DEFAULT_REGISTRY, package).await + load_environment_from_registry(DEFAULT_REGISTRY, package, cache).await } TargetEnvironmentRef::Registry { registry, package } => { - load_environment_from_registry(registry, package).await + load_environment_from_registry(registry, package, cache).await } TargetEnvironmentRef::WitDirectory { path } => load_environment_from_dir(path), } @@ -22,15 +25,24 @@ pub async fn load_environment(env_id: &TargetEnvironmentRef) -> anyhow::Result anyhow::Result { use futures_util::TryStreamExt; + let cache_file = cache.wit_path(registry, env_id); + if cache.wit_path(registry, env_id).exists() { + // Failure to read from cache is not fatal - fall through to origin. + if let Ok(bytes) = tokio::fs::read(cache_file).await { + return TargetEnvironment::from_package_bytes(env_id, bytes); + } + } + let (pkg_name, pkg_ver) = env_id.split_once('@').with_context(|| format!("Failed to parse target environment {env_id} as package reference - is the target correct?"))?; let env_pkg_ref: wasm_pkg_loader::PackageRef = pkg_name .parse() .with_context(|| format!("Environment {pkg_name} is not a valid package name"))?; - let registry: wasm_pkg_loader::Registry = registry + let wkg_registry: wasm_pkg_loader::Registry = registry .parse() .with_context(|| format!("Registry {registry} is not a valid registry name"))?; @@ -38,7 +50,7 @@ async fn load_environment_from_registry( // is there a better way to handle it? let mut wkg_config = wasm_pkg_loader::Config::global_defaults() .unwrap_or_else(|_| wasm_pkg_loader::Config::empty()); - wkg_config.set_package_registry_override(env_pkg_ref, registry); + wkg_config.set_package_registry_override(env_pkg_ref, wkg_registry); let mut client = wasm_pkg_loader::Client::new(wkg_config); @@ -64,7 +76,9 @@ async fn load_environment_from_registry( .with_context(|| format!("Failed to get {env_id} package data from registry"))? .to_vec(); - TargetEnvironment::from_package_bytes(env_id.to_owned(), bytes) + _ = cache.write_wit(&bytes, registry, env_id).await; // Failure to cache is not fatal + + TargetEnvironment::from_package_bytes(env_id, bytes) } fn load_environment_from_dir(path: &Path) -> anyhow::Result { @@ -93,7 +107,7 @@ pub struct TargetEnvironment { } impl TargetEnvironment { - fn from_package_bytes(name: String, bytes: Vec) -> anyhow::Result { + fn from_package_bytes(name: &str, bytes: Vec) -> anyhow::Result { let decoded = wit_component::decode(&bytes) .with_context(|| format!("Failed to decode package for environment {name}"))?; let package_id = decoded.package(); @@ -107,7 +121,7 @@ impl TargetEnvironment { .clone(); Ok(Self { - name, + name: name.to_owned(), decoded, package, package_id, diff --git a/crates/environments/src/lib.rs b/crates/environments/src/lib.rs index 0afb033cce..2c664fb701 100644 --- a/crates/environments/src/lib.rs +++ b/crates/environments/src/lib.rs @@ -12,12 +12,17 @@ use spin_manifest::schema::v2::TargetEnvironmentRef; pub async fn validate_application_against_environment_ids( application: &ApplicationToValidate, env_ids: &[TargetEnvironmentRef], + cache_root: Option, ) -> anyhow::Result> { if env_ids.is_empty() { return Ok(Default::default()); } - let envs = try_join_all(env_ids.iter().map(load_environment)).await?; + let cache = spin_loader::cache::Cache::new(cache_root) + .await + .context("Unable to create cache")?; + + let envs = try_join_all(env_ids.iter().map(|e| load_environment(e, &cache))).await?; validate_application_against_environments(application, &envs).await } diff --git a/crates/loader/src/cache.rs b/crates/loader/src/cache.rs index b5b5662191..31d7113b0d 100644 --- a/crates/loader/src/cache.rs +++ b/crates/loader/src/cache.rs @@ -14,6 +14,7 @@ const REGISTRY_CACHE_DIR: &str = "registry"; const MANIFESTS_DIR: &str = "manifests"; const WASM_DIR: &str = "wasm"; const DATA_DIR: &str = "data"; +const WIT_DIR: &str = "wit"; /// Cache for registry entities. #[derive(Debug)] @@ -57,6 +58,13 @@ impl Cache { self.root.join(DATA_DIR) } + /// The (Wasm-encoded) WIT directory for the current cache. + /// This does not reuse `wasm_dir` because it cannot be addressed + /// by digest. + fn wit_dir(&self) -> PathBuf { + self.root.join(WIT_DIR) + } + /// Return the path to a wasm file given its digest. pub fn wasm_file(&self, digest: impl AsRef) -> Result { // Check the expected wasm directory first; else check the data directory as a fallback. @@ -100,6 +108,18 @@ impl Cache { Ok(()) } + /// Write the contents in the cache's wit directory. + pub async fn write_wit( + &self, + bytes: impl AsRef<[u8]>, + registry: impl AsRef, + name: impl AsRef, + ) -> Result<()> { + self.ensure_wit_dir(registry.as_ref()).await?; + write_file(&self.wit_path(registry, name), bytes.as_ref()).await?; + Ok(()) + } + /// The path of contents in the cache's wasm directory, which may or may not exist. pub fn wasm_path(&self, digest: impl AsRef) -> PathBuf { self.wasm_dir().join(safe_name(digest).as_ref()) @@ -110,12 +130,24 @@ impl Cache { self.data_dir().join(safe_name(digest).as_ref()) } + /// The path of contents in the cache's wit directory, which may or may not exist. + /// This is keyed by registry and package name, not by digest, because digest + /// lookups slow down the `build` command and obviate the value of the cache: + /// therefore, it assumes that WITs are immutable once published, and that users + /// will manage the situation when this assumption is invalid. + pub fn wit_path(&self, registry: impl AsRef, name: impl AsRef) -> PathBuf { + self.wit_dir() + .join(safe_name(registry).as_ref()) + .join(safe_name(name).as_ref()) + } + /// Ensure the expected configuration directories are found in the root. /// └── /// └── registry /// └──manifests /// └──wasm /// └─-data + /// └─-wit pub async fn ensure_dirs(&self) -> Result<()> { tracing::debug!("using cache root directory {}", self.root.display()); @@ -148,10 +180,30 @@ impl Cache { .with_context(|| format!("failed to create assets directory `{}`", p.display()))?; } + let p = root.join(WIT_DIR); + if !p.is_dir() { + create_dir_all(&p) + .await + .with_context(|| format!("failed to create wit directory `{}`", p.display()))?; + } + self.dirs_ensured_once.store(true, Ordering::Relaxed); Ok(()) } + + async fn ensure_wit_dir(&self, registry: impl AsRef) -> Result<()> { + self.ensure_dirs().await?; + + let p = self.wit_dir().join(safe_name(registry).as_ref()); + if !p.is_dir() { + create_dir_all(&p) + .await + .with_context(|| format!("failed to create wit directory `{}`", p.display()))?; + } + + Ok(()) + } } #[cfg(windows)] diff --git a/src/commands/build.rs b/src/commands/build.rs index 336dc3287a..0ca7f4d166 100644 --- a/src/commands/build.rs +++ b/src/commands/build.rs @@ -53,7 +53,13 @@ impl BuildCommand { spin_common::paths::find_manifest_file_path(self.app_source.as_ref())?; notify_if_nondefault_rel(&manifest_file, distance); - spin_build::build(&manifest_file, &self.component_id, self.skip_target_checks).await?; + spin_build::build( + &manifest_file, + &self.component_id, + self.skip_target_checks, + None, + ) + .await?; if self.up { let mut cmd = UpCommand::parse_from( diff --git a/src/commands/registry.rs b/src/commands/registry.rs index a9ddf21fac..b6b50836c6 100644 --- a/src/commands/registry.rs +++ b/src/commands/registry.rs @@ -75,7 +75,7 @@ impl Push { notify_if_nondefault_rel(&app_file, distance); if self.build { - spin_build::build(&app_file, &[], false).await?; + spin_build::build(&app_file, &[], false, self.cache_dir.clone()).await?; } let annotations = if self.annotations.is_empty() { diff --git a/src/commands/up.rs b/src/commands/up.rs index e372f24cd8..55831f90de 100644 --- a/src/commands/up.rs +++ b/src/commands/up.rs @@ -191,7 +191,7 @@ impl UpCommand { } if self.build { - app_source.build().await?; + app_source.build(&self.cache_dir).await?; } let mut locked_app = self .load_resolved_app_source(resolved_app_source, &working_dir) diff --git a/src/commands/up/app_source.rs b/src/commands/up/app_source.rs index b1934efe13..fec5ed5d38 100644 --- a/src/commands/up/app_source.rs +++ b/src/commands/up/app_source.rs @@ -56,9 +56,9 @@ impl AppSource { } } - pub async fn build(&self) -> anyhow::Result<()> { + pub async fn build(&self, cache_root: &Option) -> anyhow::Result<()> { match self { - Self::File(path) => spin_build::build(path, &[], false).await, + Self::File(path) => spin_build::build(path, &[], false, cache_root.clone()).await, _ => Ok(()), } } From 6d068a48f8b855847e07a4b80b324c4d345b4d57 Mon Sep 17 00:00:00 2001 From: itowlson Date: Mon, 7 Oct 2024 15:32:55 +1300 Subject: [PATCH 11/11] Use Till's lockfile Signed-off-by: itowlson --- crates/build/src/lib.rs | 7 +- .../src/environment_definition.rs | 102 ++++++++++++++++-- crates/environments/src/lib.rs | 10 +- crates/loader/src/cache.rs | 52 --------- 4 files changed, 98 insertions(+), 73 deletions(-) diff --git a/crates/build/src/lib.rs b/crates/build/src/lib.rs index 5b285ad248..a3529b21b1 100644 --- a/crates/build/src/lib.rs +++ b/crates/build/src/lib.rs @@ -32,7 +32,7 @@ pub async fn build( })?; let app_dir = parent_dir(manifest_file)?; - let build_result = build_components(component_ids, build_info.components(), app_dir); + let build_result = build_components(component_ids, build_info.components(), &app_dir); // Emit any required warnings now, so that they don't bury any errors. if let Some(e) = build_info.load_error() { @@ -69,6 +69,7 @@ pub async fn build( &application, build_info.deployment_targets(), cache_root.clone(), + &app_dir, ) .await?; @@ -87,7 +88,7 @@ pub async fn build( fn build_components( component_ids: &[String], components: Vec, - app_dir: PathBuf, + app_dir: &Path, ) -> Result<(), anyhow::Error> { let components_to_build = if component_ids.is_empty() { components @@ -117,7 +118,7 @@ fn build_components( components_to_build .into_iter() - .map(|c| build_component(c, &app_dir)) + .map(|c| build_component(c, app_dir)) .collect::, _>>()?; terminal::step!("Finished", "building all Spin components"); diff --git a/crates/environments/src/environment_definition.rs b/crates/environments/src/environment_definition.rs index 18b50d7726..0f8ab8d0ee 100644 --- a/crates/environments/src/environment_definition.rs +++ b/crates/environments/src/environment_definition.rs @@ -1,39 +1,117 @@ -use std::path::Path; +use std::{collections::HashMap, path::Path}; use anyhow::Context; +use futures::future::try_join_all; use spin_common::ui::quoted_path; use spin_manifest::schema::v2::TargetEnvironmentRef; const DEFAULT_REGISTRY: &str = "fermyon.com"; -/// Loads the given `TargetEnvironment` from a registry. -pub async fn load_environment( +/// Serialisation format for the lockfile: registry -> { name -> digest } +#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +struct TargetEnvironmentLockfile(HashMap>); + +impl TargetEnvironmentLockfile { + fn digest(&self, registry: &str, env_id: &str) -> Option<&str> { + self.0 + .get(registry) + .and_then(|m| m.get(env_id)) + .map(|s| s.as_str()) + } + + fn set_digest(&mut self, registry: &str, env_id: &str, digest: &str) { + match self.0.get_mut(registry) { + Some(map) => { + map.insert(env_id.to_string(), digest.to_string()); + } + None => { + let map = vec![(env_id.to_string(), digest.to_string())] + .into_iter() + .collect(); + self.0.insert(registry.to_string(), map); + } + } + } +} + +/// Load all the listed environments from their registries or paths. +/// Registry data will be cached, with a lockfile under `.spin` mapping +/// environment IDs to digests (to allow cache lookup without needing +/// to fetch the digest from the registry). +pub async fn load_environments( + env_ids: &[TargetEnvironmentRef], + cache_root: Option, + app_dir: &std::path::Path, +) -> anyhow::Result> { + if env_ids.is_empty() { + return Ok(Default::default()); + } + + let cache = spin_loader::cache::Cache::new(cache_root) + .await + .context("Unable to create cache")?; + let lockfile_dir = app_dir.join(".spin"); + let lockfile_path = lockfile_dir.join("target-environments.lock"); + + let orig_lockfile: TargetEnvironmentLockfile = tokio::fs::read_to_string(&lockfile_path) + .await + .ok() + .and_then(|s| serde_json::from_str(&s).ok()) + .unwrap_or_default(); + let lockfile = std::sync::Arc::new(tokio::sync::RwLock::new(orig_lockfile.clone())); + + let envs = try_join_all( + env_ids + .iter() + .map(|e| load_environment(e, &cache, &lockfile)), + ) + .await?; + + let final_lockfile = &*lockfile.read().await; + if *final_lockfile != orig_lockfile { + if let Ok(lockfile_json) = serde_json::to_string_pretty(&final_lockfile) { + _ = tokio::fs::create_dir_all(lockfile_dir).await; + _ = tokio::fs::write(&lockfile_path, lockfile_json).await; // failure to update lockfile is not an error + } + } + + Ok(envs) +} + +/// Loads the given `TargetEnvironment` from a registry or directory. +async fn load_environment( env_id: &TargetEnvironmentRef, cache: &spin_loader::cache::Cache, + lockfile: &std::sync::Arc>, ) -> anyhow::Result { match env_id { TargetEnvironmentRef::DefaultRegistry(package) => { - load_environment_from_registry(DEFAULT_REGISTRY, package, cache).await + load_environment_from_registry(DEFAULT_REGISTRY, package, cache, lockfile).await } TargetEnvironmentRef::Registry { registry, package } => { - load_environment_from_registry(registry, package, cache).await + load_environment_from_registry(registry, package, cache, lockfile).await } TargetEnvironmentRef::WitDirectory { path } => load_environment_from_dir(path), } } +/// Loads the given `TargetEnvironment` from the given registry, or +/// from cache if available. If the environment is not in cache, the +/// encoded WIT will be cached, and the in-memory lockfile object +/// updated. async fn load_environment_from_registry( registry: &str, env_id: &str, cache: &spin_loader::cache::Cache, + lockfile: &std::sync::Arc>, ) -> anyhow::Result { use futures_util::TryStreamExt; - let cache_file = cache.wit_path(registry, env_id); - if cache.wit_path(registry, env_id).exists() { - // Failure to read from cache is not fatal - fall through to origin. - if let Ok(bytes) = tokio::fs::read(cache_file).await { - return TargetEnvironment::from_package_bytes(env_id, bytes); + if let Some(digest) = lockfile.read().await.digest(registry, env_id) { + if let Ok(cache_file) = cache.wasm_file(digest) { + if let Ok(bytes) = tokio::fs::read(&cache_file).await { + return TargetEnvironment::from_package_bytes(env_id, bytes); + } } } @@ -76,7 +154,9 @@ async fn load_environment_from_registry( .with_context(|| format!("Failed to get {env_id} package data from registry"))? .to_vec(); - _ = cache.write_wit(&bytes, registry, env_id).await; // Failure to cache is not fatal + let digest = release.content_digest.to_string(); + _ = cache.write_wasm(&bytes, &digest).await; // Failure to cache is not fatal + lockfile.write().await.set_digest(registry, env_id, &digest); TargetEnvironment::from_package_bytes(env_id, bytes) } diff --git a/crates/environments/src/lib.rs b/crates/environments/src/lib.rs index 2c664fb701..95800b82a8 100644 --- a/crates/environments/src/lib.rs +++ b/crates/environments/src/lib.rs @@ -3,8 +3,7 @@ use anyhow::{anyhow, Context}; mod environment_definition; mod loader; -use environment_definition::{load_environment, TargetEnvironment, TriggerType}; -use futures::future::try_join_all; +use environment_definition::{load_environments, TargetEnvironment, TriggerType}; pub use loader::ApplicationToValidate; use loader::ComponentToValidate; use spin_manifest::schema::v2::TargetEnvironmentRef; @@ -13,16 +12,13 @@ pub async fn validate_application_against_environment_ids( application: &ApplicationToValidate, env_ids: &[TargetEnvironmentRef], cache_root: Option, + app_dir: &std::path::Path, ) -> anyhow::Result> { if env_ids.is_empty() { return Ok(Default::default()); } - let cache = spin_loader::cache::Cache::new(cache_root) - .await - .context("Unable to create cache")?; - - let envs = try_join_all(env_ids.iter().map(|e| load_environment(e, &cache))).await?; + let envs = load_environments(env_ids, cache_root, app_dir).await?; validate_application_against_environments(application, &envs).await } diff --git a/crates/loader/src/cache.rs b/crates/loader/src/cache.rs index 31d7113b0d..b5b5662191 100644 --- a/crates/loader/src/cache.rs +++ b/crates/loader/src/cache.rs @@ -14,7 +14,6 @@ const REGISTRY_CACHE_DIR: &str = "registry"; const MANIFESTS_DIR: &str = "manifests"; const WASM_DIR: &str = "wasm"; const DATA_DIR: &str = "data"; -const WIT_DIR: &str = "wit"; /// Cache for registry entities. #[derive(Debug)] @@ -58,13 +57,6 @@ impl Cache { self.root.join(DATA_DIR) } - /// The (Wasm-encoded) WIT directory for the current cache. - /// This does not reuse `wasm_dir` because it cannot be addressed - /// by digest. - fn wit_dir(&self) -> PathBuf { - self.root.join(WIT_DIR) - } - /// Return the path to a wasm file given its digest. pub fn wasm_file(&self, digest: impl AsRef) -> Result { // Check the expected wasm directory first; else check the data directory as a fallback. @@ -108,18 +100,6 @@ impl Cache { Ok(()) } - /// Write the contents in the cache's wit directory. - pub async fn write_wit( - &self, - bytes: impl AsRef<[u8]>, - registry: impl AsRef, - name: impl AsRef, - ) -> Result<()> { - self.ensure_wit_dir(registry.as_ref()).await?; - write_file(&self.wit_path(registry, name), bytes.as_ref()).await?; - Ok(()) - } - /// The path of contents in the cache's wasm directory, which may or may not exist. pub fn wasm_path(&self, digest: impl AsRef) -> PathBuf { self.wasm_dir().join(safe_name(digest).as_ref()) @@ -130,24 +110,12 @@ impl Cache { self.data_dir().join(safe_name(digest).as_ref()) } - /// The path of contents in the cache's wit directory, which may or may not exist. - /// This is keyed by registry and package name, not by digest, because digest - /// lookups slow down the `build` command and obviate the value of the cache: - /// therefore, it assumes that WITs are immutable once published, and that users - /// will manage the situation when this assumption is invalid. - pub fn wit_path(&self, registry: impl AsRef, name: impl AsRef) -> PathBuf { - self.wit_dir() - .join(safe_name(registry).as_ref()) - .join(safe_name(name).as_ref()) - } - /// Ensure the expected configuration directories are found in the root. /// └── /// └── registry /// └──manifests /// └──wasm /// └─-data - /// └─-wit pub async fn ensure_dirs(&self) -> Result<()> { tracing::debug!("using cache root directory {}", self.root.display()); @@ -180,30 +148,10 @@ impl Cache { .with_context(|| format!("failed to create assets directory `{}`", p.display()))?; } - let p = root.join(WIT_DIR); - if !p.is_dir() { - create_dir_all(&p) - .await - .with_context(|| format!("failed to create wit directory `{}`", p.display()))?; - } - self.dirs_ensured_once.store(true, Ordering::Relaxed); Ok(()) } - - async fn ensure_wit_dir(&self, registry: impl AsRef) -> Result<()> { - self.ensure_dirs().await?; - - let p = self.wit_dir().join(safe_name(registry).as_ref()); - if !p.is_dir() { - create_dir_all(&p) - .await - .with_context(|| format!("failed to create wit directory `{}`", p.display()))?; - } - - Ok(()) - } } #[cfg(windows)]