From 1023a648ef9d74f05eb728b8caf43c672e2ca48e Mon Sep 17 00:00:00 2001 From: Marlon Date: Mon, 11 Nov 2024 15:06:12 +0100 Subject: [PATCH] Rewrite from proc-macro to build.rs (#15) * Implemented build.rs flow, for single directory * Improve logging, require environment variable * Updated README * Add force-embed feature, support multiple directories (WIP) * Macro and/or build script * Always use OUT_DIR * Fix windows paths --------- Co-authored-by: cikzh --- .gitignore | 9 +- Cargo.toml | 5 +- README.md | 36 +++- example/Cargo.toml | 10 ++ {examples/test => example}/src/main.rs | 10 +- examples/test/Cargo.toml | 14 -- memory-serve-core/Cargo.toml | 14 ++ memory-serve-core/README.md | 1 + .../src/asset.rs | 15 +- memory-serve-core/src/code.rs | 62 +++++++ memory-serve-core/src/lib.rs | 20 +++ memory-serve-core/src/list.rs | 112 +++++++++++++ memory-serve-core/src/util.rs | 41 +++++ memory-serve-macros/Cargo.toml | 13 +- memory-serve-macros/src/lib.rs | 60 +++---- memory-serve-macros/src/utils.rs | 158 ------------------ memory-serve/Cargo.toml | 15 +- memory-serve/build.rs | 81 +++++++++ memory-serve/src/asset.rs | 26 +-- memory-serve/src/lib.rs | 100 ++++++++--- memory-serve/src/util.rs | 4 + memory-serve/static | 1 - 22 files changed, 510 insertions(+), 297 deletions(-) create mode 100644 example/Cargo.toml rename {examples/test => example}/src/main.rs (71%) delete mode 100644 examples/test/Cargo.toml create mode 100644 memory-serve-core/Cargo.toml create mode 120000 memory-serve-core/README.md rename {memory-serve-macros => memory-serve-core}/src/asset.rs (63%) create mode 100644 memory-serve-core/src/code.rs create mode 100644 memory-serve-core/src/lib.rs create mode 100644 memory-serve-core/src/list.rs create mode 100644 memory-serve-core/src/util.rs delete mode 100644 memory-serve-macros/src/utils.rs create mode 100644 memory-serve/build.rs delete mode 120000 memory-serve/static diff --git a/.gitignore b/.gitignore index 7e93dc3..c0e3a51 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,4 @@ .vscode/ target/ -/Cargo.lock +Cargo.lock - -# Added by cargo -# -# already existing elements were commented out - -/target -#/Cargo.lock diff --git a/Cargo.toml b/Cargo.toml index 73f5ae7..8fd62ab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,9 +1,10 @@ [workspace] -members = ["memory-serve", "memory-serve-macros", "examples/test"] +members = ["memory-serve", "memory-serve-macros", "memory-serve-core"] +exclude = ["example"] resolver = "2" [workspace.package] -version = "0.6.0" +version = "1.0.0-beta.0" edition = "2021" license = "Apache-2.0 OR MIT" repository = "https://github.com/tweedegolf/memory-serve" diff --git a/README.md b/README.md index 8e8d18b..b2d1f5c 100644 --- a/README.md +++ b/README.md @@ -33,24 +33,48 @@ memory-serve is designed to work with [axum](https://github.com/tokio-rs/axum) ## Usage -Provide a relative path to the directory containing your static assets -to the [`load_assets!`] macro. This macro creates a data structure intended to -be consumed by [`MemoryServe::new`]. Calling [`MemoryServe::into_router()`] on -the resulting instance produces a axum +There are two mechanisms to include assets at compile time. + +1. Specify the path using a enviroment variable `ASSET_PATH` and call: `MemoryServe::from_env()` (best-practice) +2. Call the `load_assets!` macro, and pass this to the constructor: `MemoryServe::new(load_assets!("/foo/bar"))` + +The environment variable is handled by a build script and instructs cargo to re-evaluate when an asset in the directory changes. +The output of the macro might be cached between build. + +Both options try to be smart in resolving absolute and relative paths. + +When an instance of `MemoryServe` is created, we can bind these to your axum instance. +Calling [`MemoryServe::into_router()`] on the `MemoryServe` instance produces an axum [`Router`](https://docs.rs/axum/latest/axum/routing/struct.Router.html) that can either be merged in another `Router` or used directly in a server by calling [`Router::into_make_service()`](https://docs.rs/axum/latest/axum/routing/struct.Router.html#method.into_make_service). +### Named directories + +Multiple directories can be included using different environment variables, all prefixed by `ASSET_PATH_`. +For example: if you specify `ASSET_PATH_FOO` and `ASSET_PATH_BAR` the memory serve instances can be loaded +using `MemoryServe::from_env_name("FOO")` and `MemoryServe::from_env_name("BAR")` respectively. + +### Features + +Use the `force-embed` feature flag to always include assets in the binary - also in debug builds. + +### Environment variables + +Use `MEMORY_SERVE_ROOT` to specify a root directory for relative paths provided to the `load_assets!` macro (or th `ASSET_PATH` variable). + +Uee `MEMORY_SERVE_QUIET=1` to not print log messages at compile time. + ## Example ```rust,no_run use axum::{response::Html, routing::get, Router}; -use memory_serve::{load_assets, MemoryServe}; +use memory_serve::{MemoryServe, load_assets}; use std::net::SocketAddr; #[tokio::main] async fn main() { - let memory_router = MemoryServe::new(load_assets!("static")) + let memory_router = MemoryServe::new(load_assets!("../static")) .index_file(Some("/index.html")) .into_router(); diff --git a/example/Cargo.toml b/example/Cargo.toml new file mode 100644 index 0000000..d4b39bf --- /dev/null +++ b/example/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "memory-serve-test" +edition = "2021" + +[dependencies] +memory-serve = { path = "../memory-serve" } +axum = "0.7" +tokio = { version = "1.0", features = ["full"] } +tracing-subscriber = "0.3" +tracing = "0.1" \ No newline at end of file diff --git a/examples/test/src/main.rs b/example/src/main.rs similarity index 71% rename from examples/test/src/main.rs rename to example/src/main.rs index 19d7b2a..fd22589 100644 --- a/examples/test/src/main.rs +++ b/example/src/main.rs @@ -1,13 +1,15 @@ use axum::{response::Html, routing::get, Router}; -use memory_serve::{load_assets, MemoryServe}; +use memory_serve::{MemoryServe, load_assets}; use std::net::SocketAddr; -use tracing::info; +use tracing::{info, Level}; #[tokio::main] async fn main() { - tracing_subscriber::fmt().init(); + tracing_subscriber::fmt() + .with_max_level(Level::TRACE) + .init(); - let memory_router = MemoryServe::new(load_assets!("../../static")) + let memory_router = MemoryServe::new(load_assets!("../static")) .index_file(Some("/index.html")) .into_router(); diff --git a/examples/test/Cargo.toml b/examples/test/Cargo.toml deleted file mode 100644 index 4bcfa4e..0000000 --- a/examples/test/Cargo.toml +++ /dev/null @@ -1,14 +0,0 @@ -[package] -name = "memory-serve-test" -version.workspace = true -edition.workspace = true -license.workspace = true -repository.workspace = true -publish = false - -[dependencies] -memory-serve = { path = "../../memory-serve" } -axum = "0.7" -tokio = { version = "1.0", features = ["full"] } -tracing-subscriber = "0.3" -tracing = "0.1" diff --git a/memory-serve-core/Cargo.toml b/memory-serve-core/Cargo.toml new file mode 100644 index 0000000..32f03d2 --- /dev/null +++ b/memory-serve-core/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "memory-serve-core" +description = "Shared code for memory-serve and memory-serve-macros" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +publish.workspace = true + +[dependencies] +sha256 = "1.4" +brotli = "7.0" +mime_guess = "2.0" +walkdir = "2" diff --git a/memory-serve-core/README.md b/memory-serve-core/README.md new file mode 120000 index 0000000..32d46ee --- /dev/null +++ b/memory-serve-core/README.md @@ -0,0 +1 @@ +../README.md \ No newline at end of file diff --git a/memory-serve-macros/src/asset.rs b/memory-serve-core/src/asset.rs similarity index 63% rename from memory-serve-macros/src/asset.rs rename to memory-serve-core/src/asset.rs index 88d853c..2a6b8bb 100644 --- a/memory-serve-macros/src/asset.rs +++ b/memory-serve-core/src/asset.rs @@ -1,13 +1,12 @@ -use syn::LitByteStr; +use std::path::PathBuf; /// Internal data structure -pub(crate) struct Asset { - pub(crate) route: String, - pub(crate) path: String, - pub(crate) etag: String, - pub(crate) content_type: String, - pub(crate) bytes: LitByteStr, - pub(crate) brotli_bytes: LitByteStr, +pub struct Asset { + pub route: String, + pub path: PathBuf, + pub etag: String, + pub content_type: String, + pub compressed_bytes: Option>, } impl PartialEq for Asset { diff --git a/memory-serve-core/src/code.rs b/memory-serve-core/src/code.rs new file mode 100644 index 0000000..d6b24e7 --- /dev/null +++ b/memory-serve-core/src/code.rs @@ -0,0 +1,62 @@ +use std::path::{Path, PathBuf}; + +use crate::{asset::Asset, list::list_assets}; + +/// Generate code with metadata and contents for the assets +pub fn assets_to_code(asset_dir: &str, path: &Path, embed: bool, log: fn(&str)) -> String { + let out_dir: String = std::env::var("OUT_DIR").expect("OUT_DIR environment variable not set."); + let out_dir = PathBuf::from(&out_dir); + + log(&format!("Loading static assets from {asset_dir}")); + + if embed { + log("Embedding assets into binary"); + } else { + log("Not embedding assets into binary, assets will load dynamically"); + } + + let assets = list_assets(path, embed, log); + + // using a string is faster than using quote ;) + let mut code = "&[".to_string(); + + for asset in assets { + let Asset { + route, + path, + etag, + content_type, + compressed_bytes, + } = asset; + + let bytes = if !embed { + "None".to_string() + } else if let Some(compressed_bytes) = &compressed_bytes { + let file_name = path.file_name().expect("Unable to get file name."); + let file_path = Path::new(&out_dir).join(file_name); + std::fs::write(&file_path, compressed_bytes).expect("Unable to write file to out dir."); + + format!("Some(include_bytes!(r\"{}\"))", file_path.to_string_lossy()) + } else { + format!("Some(include_bytes!(r\"{}\"))", path.to_string_lossy()) + }; + + let is_compressed = compressed_bytes.is_some(); + + code.push_str(&format!( + " + memory_serve::Asset {{ + route: r\"{route}\", + path: r{path:?}, + content_type: \"{content_type}\", + etag: \"{etag}\", + bytes: {bytes}, + is_compressed: {is_compressed}, + }}," + )); + } + + code.push(']'); + + code +} diff --git a/memory-serve-core/src/lib.rs b/memory-serve-core/src/lib.rs new file mode 100644 index 0000000..7f62187 --- /dev/null +++ b/memory-serve-core/src/lib.rs @@ -0,0 +1,20 @@ +mod asset; +mod code; +mod list; +mod util; + +pub use asset::Asset; +pub use code::assets_to_code; + +/// File mime types that can possibly be compressed +pub const COMPRESS_TYPES: &[&str] = &[ + "text/html", + "text/css", + "application/json", + "text/javascript", + "application/javascript", + "application/xml", + "text/xml", + "image/svg+xml", + "application/wasm", +]; diff --git a/memory-serve-core/src/list.rs b/memory-serve-core/src/list.rs new file mode 100644 index 0000000..cdea2aa --- /dev/null +++ b/memory-serve-core/src/list.rs @@ -0,0 +1,112 @@ +use std::path::Path; + +use walkdir::WalkDir; + +use crate::{ + asset::Asset, + util::{compress_brotli, path_to_content_type, path_to_route}, + COMPRESS_TYPES, +}; + +/// List all assets in the given directory (recursively) and return a list of assets with metadata +pub fn list_assets(base_path: &Path, embed: bool, log: fn(&str)) -> Vec { + let mut assets: Vec = WalkDir::new(base_path) + .into_iter() + .filter_map(|entry| entry.ok()) + .filter_map(|entry| { + let path = entry.path().to_owned(); + let route = path_to_route(base_path, entry.path()); + + let Ok(metadata) = entry.metadata() else { + log(&format!( + "skipping file {route}, could not get file metadata" + )); + return None; + }; + + // skip directories + if !metadata.is_file() { + return None; + }; + + // skip empty + if metadata.len() == 0 { + log(&format!("skipping file {route}: file empty")); + return None; + } + + let Some(content_type) = path_to_content_type(entry.path()) else { + log(&format!( + "skipping file {route}, could not determine file extension" + )); + return None; + }; + + // do not load assets into the binary in debug / development mode + if !embed { + log(&format!("including {route} (dynamically)")); + + return Some(Asset { + route, + path: path.to_owned(), + content_type, + etag: Default::default(), + compressed_bytes: None, + }); + } + + let Ok(bytes) = std::fs::read(entry.path()) else { + log(&format!("skipping file {route}: file is not readable")); + return None; + }; + + let etag: String = sha256::digest(&bytes); + let original_size = bytes.len(); + let is_compress_type = COMPRESS_TYPES.contains(&content_type.as_str()); + let brotli_bytes = if is_compress_type { + compress_brotli(&bytes) + } else { + None + }; + + let mut asset = Asset { + route: route.clone(), + path: path.to_owned(), + content_type, + etag, + compressed_bytes: None, + }; + + if is_compress_type { + match brotli_bytes { + Some(brotli_bytes) if brotli_bytes.len() >= original_size => { + log(&format!( + "including {route} {original_size} bytes (compression unnecessary)" + )); + } + Some(brotli_bytes) => { + log(&format!( + "including {route} {original_size} -> {} bytes (compressed)", + brotli_bytes.len() + )); + + asset.compressed_bytes = Some(brotli_bytes); + } + None => { + log(&format!( + "including {route} {original_size} bytes (compression failed)" + )); + } + } + } else { + log(&format!("including {route} {original_size} bytes")); + } + + Some(asset) + }) + .collect(); + + assets.sort(); + + assets +} diff --git a/memory-serve-core/src/util.rs b/memory-serve-core/src/util.rs new file mode 100644 index 0000000..e56df2b --- /dev/null +++ b/memory-serve-core/src/util.rs @@ -0,0 +1,41 @@ +use std::{io::Write, path::Path}; + +use mime_guess::mime; + +/// Convert a path to a (HTTP) path / route +pub(crate) fn path_to_route(base: &Path, path: &Path) -> String { + let relative_path = path + .strip_prefix(base) + .expect("Could not strap prefix from path"); + + let route = relative_path + .components() + .filter_map(|c| match c { + std::path::Component::Normal(s) => s.to_str(), + _ => None, + }) + .collect::>() + .join("/"); + + format!("/{route}") +} + +/// Determine the mime type of a file +pub(crate) fn path_to_content_type(path: &Path) -> Option { + let ext = path.extension()?; + + Some( + mime_guess::from_ext(&ext.to_string_lossy()) + .first_raw() + .unwrap_or(mime::APPLICATION_OCTET_STREAM.to_string().as_str()) + .to_owned(), + ) +} + +/// Compress a byte slice using brotli +pub(crate) fn compress_brotli(input: &[u8]) -> Option> { + let mut writer = brotli::CompressorWriter::new(Vec::new(), 4096, 11, 22); + writer.write_all(input).ok()?; + + Some(writer.into_inner()) +} diff --git a/memory-serve-macros/Cargo.toml b/memory-serve-macros/Cargo.toml index 9d5a3dc..d619765 100644 --- a/memory-serve-macros/Cargo.toml +++ b/memory-serve-macros/Cargo.toml @@ -10,13 +10,8 @@ publish.workspace = true [lib] proc-macro = true +[features] +force-embed = [] + [dependencies] -brotli = "6.0" -mime_guess = "2.0" -proc-macro2 = "1.0" -sha256 = "1.4" -walkdir = "2" -tracing = "0.1" -quote = { version = "1.0", default-features = false } -syn = { version = "2.0", default-features = false } -tracing-subscriber = { version = "0.3", features = ["fmt", "ansi"], default-features = false } +memory-serve-core = { path = "../memory-serve-core" } \ No newline at end of file diff --git a/memory-serve-macros/src/lib.rs b/memory-serve-macros/src/lib.rs index b8ce56a..5b4f291 100644 --- a/memory-serve-macros/src/lib.rs +++ b/memory-serve-macros/src/lib.rs @@ -1,62 +1,40 @@ +use memory_serve_core::assets_to_code; use proc_macro::TokenStream; use std::{env, path::Path}; -use utils::list_assets; - -mod asset; -mod utils; - -use crate::asset::Asset; #[proc_macro] pub fn load_assets(input: TokenStream) -> TokenStream { let input = input.to_string(); - let input = input.trim_matches('"'); - let mut asset_path = Path::new(&input).to_path_buf(); + let asset_dir = input.trim_matches('"'); + let mut path = Path::new(&asset_dir).to_path_buf(); - // skip if a subscriber is already registered (for instance by rust_analyzer) - let _ = tracing_subscriber::fmt() - .without_time() - .with_target(false) - .try_init(); + fn log(msg: &str) { + if std::env::var("MEMORY_SERVE_QUIET") != Ok("1".to_string()) { + println!(" memory_serve: {msg}"); + } + } - if asset_path.is_relative() { + if path.is_relative() { if let Ok(root_dir) = env::var("MEMORY_SERVE_ROOT") { - asset_path = Path::new(&root_dir).join(asset_path); + path = Path::new(&root_dir).join(path); } else if let Ok(crate_dir) = env::var("CARGO_MANIFEST_DIR") { - asset_path = Path::new(&crate_dir).join(asset_path); + path = Path::new(&crate_dir).join(path); } else { panic!("Relative path provided but CARGO_MANIFEST_DIR environment variable not set"); } } - asset_path = asset_path + path = path .canonicalize() .expect("Could not canonicalize the provided path"); - if !asset_path.exists() { - panic!("The path {:?} does not exists!", asset_path); + if !path.exists() { + panic!("The path {path:?} does not exists!"); } - let files: Vec = list_assets(&asset_path); - - let route = files.iter().map(|a| &a.route); - let path = files.iter().map(|a| &a.path); - let content_type = files.iter().map(|a| &a.content_type); - let etag = files.iter().map(|a| &a.etag); - let bytes = files.iter().map(|a| &a.bytes); - let brotli_bytes = files.iter().map(|a| &a.brotli_bytes); - - quote::quote! { - &[ - #(memory_serve::Asset { - route: #route, - path: #path, - content_type: #content_type, - etag: #etag, - bytes: #bytes, - brotli_bytes: #brotli_bytes, - }),* - ] - } - .into() + let embed = !cfg!(debug_assertions) || cfg!(feature = "force-embed"); + + let assets = assets_to_code(asset_dir, &path, embed, log); + + assets.parse().expect("Could not parse assets to code") } diff --git a/memory-serve-macros/src/utils.rs b/memory-serve-macros/src/utils.rs deleted file mode 100644 index 639ca5d..0000000 --- a/memory-serve-macros/src/utils.rs +++ /dev/null @@ -1,158 +0,0 @@ -use mime_guess::mime; -use proc_macro2::Span; -use std::{io::Write, path::Path}; -use syn::LitByteStr; -use tracing::{info, warn}; -use walkdir::WalkDir; - -use crate::Asset; - -const COMPRESS_TYPES: &[&str] = &[ - "text/html", - "text/css", - "application/json", - "text/javascript", - "application/javascript", - "application/xml", - "text/xml", - "image/svg+xml", - "application/wasm", -]; - -fn path_to_route(base: &Path, path: &Path) -> String { - let relative_path = path - .strip_prefix(base) - .expect("Could not strap prefix from path"); - - let route = relative_path - .components() - .filter_map(|c| match c { - std::path::Component::Normal(s) => s.to_str(), - _ => None, - }) - .collect::>() - .join("/"); - - format!("/{route}") -} - -fn path_to_content_type(path: &Path) -> Option { - let ext = path.extension()?; - - Some( - mime_guess::from_ext(&ext.to_string_lossy()) - .first_raw() - .unwrap_or(mime::APPLICATION_OCTET_STREAM.to_string().as_str()) - .to_owned(), - ) -} - -fn compress_brotli(input: &[u8]) -> Option> { - let mut writer = brotli::CompressorWriter::new(Vec::new(), 4096, 11, 22); - writer.write_all(input).ok()?; - - Some(writer.into_inner()) -} - -fn literal_bytes(bytes: Vec) -> LitByteStr { - LitByteStr::new(&bytes, Span::call_site()) -} - -// skip if compressed data is larger than the original -fn skip_larger(compressed: Vec, original: &[u8]) -> Vec { - if compressed.len() >= original.len() { - Default::default() - } else { - compressed - } -} - -pub(crate) fn list_assets(base_path: &Path) -> Vec { - let mut assets: Vec = WalkDir::new(base_path) - .into_iter() - .filter_map(|entry| entry.ok()) - .filter_map(|entry| { - let Some(path) = entry.path().to_str() else { - warn!("invalid file path {:?}", entry.path()); - return None; - }; - - let route = path_to_route(base_path, entry.path()); - - let Ok(metadata) = entry.metadata() else { - warn!("skipping file {route}, could not get file metadata"); - return None; - }; - - // skip directories - if !metadata.is_file() { - return None; - }; - - // skip empty - if metadata.len() == 0 { - warn!("skipping file {route}: file empty"); - return None; - } - - let Some(content_type) = path_to_content_type(entry.path()) else { - warn!("skipping file {route}, could not determine file extension"); - return None; - }; - - // do not load assets into the binary in debug / development mode - if cfg!(debug_assertions) { - return Some(Asset { - route, - path: path.to_owned(), - content_type, - etag: Default::default(), - bytes: literal_bytes(Default::default()), - brotli_bytes: literal_bytes(Default::default()), - }); - } - - let Ok(bytes) = std::fs::read(entry.path()) else { - warn!("skipping file {route}: file is not readable"); - return None; - }; - - let etag = sha256::digest(&bytes); - - let brotli_bytes = if COMPRESS_TYPES.contains(&content_type.as_str()) { - compress_brotli(&bytes) - .map(|v| skip_larger(v, &bytes)) - .unwrap_or_default() - } else { - Default::default() - }; - - if brotli_bytes.is_empty() { - info!("including {route} {} bytes", bytes.len()); - } else { - info!( - "including {route} {} -> {} bytes (compressed)", - bytes.len(), - brotli_bytes.len() - ); - }; - - Some(Asset { - route, - path: path.to_owned(), - content_type, - etag, - bytes: literal_bytes(if brotli_bytes.is_empty() { - bytes - } else { - Default::default() - }), - brotli_bytes: literal_bytes(brotli_bytes), - }) - }) - .collect(); - - assets.sort(); - - assets -} diff --git a/memory-serve/Cargo.toml b/memory-serve/Cargo.toml index cea4cb6..77ba3b0 100644 --- a/memory-serve/Cargo.toml +++ b/memory-serve/Cargo.toml @@ -1,21 +1,28 @@ [package] name = "memory-serve" +description.workspace = true version.workspace = true edition.workspace = true license.workspace = true repository.workspace = true publish.workspace = true -description.workspace = true + +[features] +force-embed = ["memory-serve-macros/force-embed"] [dependencies] -brotli = "6.0" +memory-serve-core = { path = "../memory-serve-core" } +memory-serve-macros = { path = "../memory-serve-macros" } +brotli = "7.0" flate2 = "1.0" axum = { version = "0.7", default-features = false } -memory-serve-macros = { version = "0.6", path = "../memory-serve-macros" } tracing = "0.1" sha256 = "1.4" +[build-dependencies] +memory-serve-core = { path = "../memory-serve-core" } + [dev-dependencies] tokio = { version = "1", features = ["full"] } -tower = "0.4" +tower = { version = "0.5", features = ["util"] } axum = { version = "0.7" } diff --git a/memory-serve/build.rs b/memory-serve/build.rs new file mode 100644 index 0000000..a888fa1 --- /dev/null +++ b/memory-serve/build.rs @@ -0,0 +1,81 @@ +use memory_serve_core::assets_to_code; +use std::path::{Path, PathBuf}; + +const ASSET_FILE: &str = "memory_serve_assets.rs"; +const ENV_NAME: &str = "ASSET_DIR"; +const QUIET_ENV_NAME: &str = "MEMORY_SERVE_QUIET"; + +fn resolve_asset_dir(out_dir: &Path, key: &str, asset_dir: &str) -> PathBuf { + let path = Path::new(&asset_dir); + + let path: PathBuf = if path.is_relative() { + if let Ok(root_dir) = std::env::var("MEMORY_SERVE_ROOT") { + let root_dir = Path::new(&root_dir); + root_dir.join(path) + } else { + // assume the out dit is in the target directory + let crate_root = out_dir + .parent() // memory-serve + .and_then(|p| p.parent()) // build + .and_then(|p| p.parent()) // debug/release + .and_then(|p| p.parent()) // target + .and_then(|p| p.parent()) // crate root + .expect("Unable to get crate root directory."); + + crate_root.join(path) + } + } else { + path.to_path_buf() + }; + + let path = match path.canonicalize() { + Ok(path) => path, + Err(e) => panic!("The path {path:?} specified by {key} is not a valid path: {e}"), + }; + + if !path.exists() { + panic!("The path {path:?} specified by {key} does not exists!"); + } + + path +} + +fn main() { + let out_dir: String = std::env::var("OUT_DIR").expect("OUT_DIR environment variable not set."); + let out_dir = PathBuf::from(&out_dir); + + fn log(msg: &str) { + if std::env::var(QUIET_ENV_NAME) != Ok("1".to_string()) { + println!("cargo:warning={}", msg); + } + } + + // deternmine wheter to dynamically load assets or embed them in the binary + let force_embed = std::env::var("CARGO_FEATURE_FORCE_EMBED").unwrap_or_default(); + let embed = !cfg!(debug_assertions) || force_embed == "1"; + + // using a string is faster than using quote ;) + let mut code = "&[".to_string(); + + for (key, asset_dir) in std::env::vars() { + if key.starts_with(ENV_NAME) { + let name = key.trim_start_matches(format!("{ENV_NAME}_").as_str()); + let path = resolve_asset_dir(&out_dir, &key, &asset_dir); + + let assets = assets_to_code(&asset_dir, &path, embed, log); + + println!("cargo::rerun-if-changed={asset_dir}"); + + code = format!("{code}(\"{name}\", {assets}),"); + } + } + + code.push(']'); + + println!("cargo::rerun-if-env-changed=CARGO_FEATURE_FORCE_EMBED"); + println!("cargo::rerun-if-env-changed={ENV_NAME}"); + println!("cargo::rerun-if-env-changed={QUIET_ENV_NAME}"); + + let target = out_dir.join(ASSET_FILE); + std::fs::write(target, code).expect("Unable to write memory-serve asset file."); +} diff --git a/memory-serve/src/asset.rs b/memory-serve/src/asset.rs index 74221cf..d5ba1a4 100644 --- a/memory-serve/src/asset.rs +++ b/memory-serve/src/asset.rs @@ -5,25 +5,14 @@ use axum::{ }, response::{IntoResponse, Response}, }; -use tracing::{debug, error}; +use memory_serve_core::COMPRESS_TYPES; +use tracing::debug; use crate::{ util::{compress_brotli, compress_gzip, content_length, supports_encoding}, ServeOptions, }; -pub const COMPRESS_TYPES: &[&str] = &[ - "text/html", - "text/css", - "application/json", - "text/javascript", - "application/javascript", - "application/xml", - "text/xml", - "image/svg+xml", - "application/wasm", -]; - const BROTLI_ENCODING: &str = "br"; #[allow(clippy::declare_interior_mutable_const)] const BROTLI_HEADER: (HeaderName, HeaderValue) = @@ -33,14 +22,15 @@ const GZIP_ENCODING: &str = "gzip"; const GZIP_HEADER: (HeaderName, HeaderValue) = (CONTENT_ENCODING, HeaderValue::from_static(GZIP_ENCODING)); +/// Represents a static asset that can be served #[derive(Debug)] pub struct Asset { pub route: &'static str, pub path: &'static str, pub etag: &'static str, pub content_type: &'static str, - pub bytes: &'static [u8], - pub brotli_bytes: &'static [u8], + pub bytes: Option<&'static [u8]>, + pub is_compressed: bool, } struct AssetResponse<'t, B> { @@ -142,7 +132,6 @@ impl Asset { options: &ServeOptions, ) -> Response { let Ok(bytes) = std::fs::read(self.path) else { - error!("File not found {}", self.path); return StatusCode::NOT_FOUND.into_response(); }; @@ -181,6 +170,7 @@ impl Asset { headers: &HeaderMap, status: StatusCode, bytes: &'static [u8], + brotli_bytes: &'static [u8], gzip_bytes: &'static [u8], options: &ServeOptions, ) -> Response { @@ -198,8 +188,8 @@ impl Asset { etag: self.etag, bytes_len: bytes.len(), bytes, - brotli_bytes_len: self.brotli_bytes.len(), - brotli_bytes: self.brotli_bytes, + brotli_bytes_len: brotli_bytes.len(), + brotli_bytes, gzip_bytes_len: gzip_bytes.len(), gzip_bytes, } diff --git a/memory-serve/src/lib.rs b/memory-serve/src/lib.rs index 1bc6e27..f1baee8 100644 --- a/memory-serve/src/lib.rs +++ b/memory-serve/src/lib.rs @@ -5,18 +5,16 @@ use axum::{ }; use std::future::ready; use tracing::info; - mod asset; mod cache_control; mod util; +#[allow(unused)] +use crate as memory_serve; + use crate::util::{compress_gzip, decompress_brotli}; pub use crate::{asset::Asset, cache_control::CacheControl}; - -/// Macro to load a directory of static files into the resulting binary -/// (possibly compressed) and create a data structure of (meta)data -/// as an input for [`MemoryServe::new`] pub use memory_serve_macros::load_assets; #[derive(Debug, Clone, Copy)] @@ -50,8 +48,6 @@ impl Default for ServeOptions { /// Helper struct to create and configure an axum to serve static files from /// memory. -/// Initiate an instance with the `MemoryServe::new` method and pass a call -/// to the `load_assets!` macro as the single argument. #[derive(Debug, Default)] pub struct MemoryServe { options: ServeOptions, @@ -70,9 +66,47 @@ impl MemoryServe { } } + /// Initiate a `MemoryServe` instance, takes the contents of `memory_serve_assets.bin` + /// created at build time. + /// Specify which asset directory to include using the environment variable `ASSET_DIR`. + pub fn from_env() -> Self { + let assets: &[(&str, &[Asset])] = + include!(concat!(env!("OUT_DIR"), "/memory_serve_assets.rs")); + + if assets.is_empty() { + panic!("No assets found, did you forget to set the ASSET_DIR environment variable?"); + } + + Self::new(assets[0].1) + } + + /// Include a directory using a named environment variable, prefixed by ASSRT_DIR_. + /// Specify which asset directory to include using the environment variable `ASSET_DIR_`. + /// The name should be in uppercase. + /// For example to include assets from the public directory using the name PUBLIC, set the enirobment variable + /// `ASSET_DIR_PUBLIC=./public` and call `MemoryServe::from_name("PUBLIC")`. + pub fn from_env_name(name: &str) -> Self { + let assets: &[(&str, &[Asset])] = + include!(concat!(env!("OUT_DIR"), "/memory_serve_assets.rs")); + + let assets = assets + .iter() + .find(|(n, _)| n == &name) + .map(|(_, a)| *a) + .unwrap_or_default(); + + if assets.is_empty() { + panic!( + "No assets found, did you forget to set the ASSET_DIR_{name} environment variable?" + ); + } + + Self::new(assets) + } + /// Which static file to serve on the route "/" (the index) - /// The path (or route) should be relative to the directory passed to - /// the `load_assets!` macro, but prepended with a slash. + /// The path (or route) should be relative to the directory set with + /// the `ASSET_DIR` variable, but prepended with a slash. /// By default this is `Some("/index.html")` pub fn index_file(mut self, index_file: Option<&'static str>) -> Self { self.options.index_file = index_file; @@ -90,8 +124,8 @@ impl MemoryServe { /// Which static file to serve when no other routes are matched, also see /// [fallback](https://docs.rs/axum/latest/axum/routing/struct.Router.html#method.fallback) - /// The path (or route) should be relative to the directory passed to - /// the `load_assets!` macro, but prepended with a slash. + /// The path (or route) should be relative to the directory set with + /// the `ASSET_DIR` variable, but prepended with a slash. /// By default this is `None`, which means axum will return an empty /// response with a HTTP 404 status code when no route matches. pub fn fallback(mut self, fallback: Option<&'static str>) -> Self { @@ -167,33 +201,48 @@ impl MemoryServe { let options = Box::leak(Box::new(self.options)); for asset in self.assets { - let bytes = if asset.bytes.is_empty() && !asset.brotli_bytes.is_empty() { - Box::new(decompress_brotli(asset.brotli_bytes).unwrap_or_default()).leak() + let mut bytes = asset.bytes.unwrap_or_default(); + + if asset.is_compressed { + bytes = Box::new(decompress_brotli(bytes).unwrap_or_default()).leak() + } + + let gzip_bytes = if asset.is_compressed && options.enable_gzip { + Box::new(compress_gzip(bytes).unwrap_or_default()).leak() } else { - asset.bytes + Default::default() }; - let gzip_bytes = if !asset.brotli_bytes.is_empty() && options.enable_gzip { - Box::new(compress_gzip(bytes).unwrap_or_default()).leak() + let brotli_bytes = if asset.is_compressed { + asset.bytes.unwrap_or_default() } else { Default::default() }; if !bytes.is_empty() { - if !asset.brotli_bytes.is_empty() { + if asset.is_compressed { info!( "serving {} {} -> {} bytes (compressed)", asset.route, bytes.len(), - asset.brotli_bytes.len() + brotli_bytes.len() ); } else { info!("serving {} {} bytes", asset.route, bytes.len()); } + } else { + info!("serving {} (dynamically)", asset.route); } let handler = |headers: HeaderMap| { - ready(asset.handler(&headers, StatusCode::OK, bytes, gzip_bytes, options)) + ready(asset.handler( + &headers, + StatusCode::OK, + bytes, + brotli_bytes, + gzip_bytes, + options, + )) }; if Some(asset.route) == options.fallback { @@ -204,6 +253,7 @@ impl MemoryServe { &headers, options.fallback_status, bytes, + brotli_bytes, gzip_bytes, options, )) @@ -246,7 +296,7 @@ impl MemoryServe { #[cfg(test)] mod tests { - use crate::{self as memory_serve, load_assets, Asset, CacheControl, MemoryServe}; + use crate::{self as memory_serve, Asset, CacheControl, MemoryServe}; use axum::{ body::Body, http::{ @@ -256,6 +306,7 @@ mod tests { }, Router, }; + use memory_serve_macros::load_assets; use tower::ServiceExt; async fn get( @@ -283,9 +334,9 @@ mod tests { headers.get(name).unwrap().to_str().unwrap() } - #[test] - fn test_load_assets() { - let assets: &'static [Asset] = load_assets!("../static"); + #[tokio::test] + async fn test_load_assets() { + let assets: &[Asset] = load_assets!("../static"); let routes: Vec<&str> = assets.iter().map(|a| a.route).collect(); let content_types: Vec<&str> = assets.iter().map(|a| a.content_type).collect(); let etags: Vec<&str> = assets.iter().map(|a| a.etag).collect(); @@ -314,7 +365,7 @@ mod tests { "text/html" ] ); - if cfg!(debug_assertions) { + if cfg!(debug_assertions) && !cfg!(feature = "force-embed") { assert_eq!(etags, ["", "", "", "", "", "", ""]); } else { assert_eq!( @@ -400,6 +451,7 @@ mod tests { "gzip", ) .await; + let encoding = get_header(&headers, &CONTENT_ENCODING); let length = get_header(&headers, &CONTENT_LENGTH); diff --git a/memory-serve/src/util.rs b/memory-serve/src/util.rs index 05d755e..32b6296 100644 --- a/memory-serve/src/util.rs +++ b/memory-serve/src/util.rs @@ -5,6 +5,7 @@ use axum::http::{ HeaderMap, HeaderName, HeaderValue, }; +/// Decompress a byte slice using brotli pub(crate) fn decompress_brotli(input: &[u8]) -> Option> { let mut writer = brotli::DecompressorWriter::new(Vec::new(), 1024); writer.write_all(input).ok()?; @@ -12,6 +13,7 @@ pub(crate) fn decompress_brotli(input: &[u8]) -> Option> { writer.into_inner().ok() } +/// Compress a byte slice using gzip pub(crate) fn compress_brotli(input: &[u8]) -> Option> { let mut writer = brotli::CompressorWriter::new(Vec::new(), 4096, 11, 22); writer.write_all(input).ok()?; @@ -19,6 +21,7 @@ pub(crate) fn compress_brotli(input: &[u8]) -> Option> { Some(writer.into_inner()) } +/// Compress a byte slice using gzip pub(crate) fn compress_gzip(input: &[u8]) -> Option> { let mut writer = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::best()); writer.write_all(input).ok()?; @@ -26,6 +29,7 @@ pub(crate) fn compress_gzip(input: &[u8]) -> Option> { writer.finish().ok() } +/// Check if the client supports the given encoding pub(crate) fn supports_encoding(headers: &HeaderMap, encoding: &str) -> bool { let Some(header_value) = headers .get(ACCEPT_ENCODING) diff --git a/memory-serve/static b/memory-serve/static deleted file mode 120000 index 4dab164..0000000 --- a/memory-serve/static +++ /dev/null @@ -1 +0,0 @@ -../static \ No newline at end of file