diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 629e45b..01b43d5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,6 +58,16 @@ jobs: shared-key: test - run: cargo test --all-features --all-targets + test-release: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: Swatinem/rust-cache@v2 + with: + prefix-key: cargo + shared-key: test + - run: cargo test --all-features --all-targets --release + udeps: runs-on: ubuntu-latest steps: diff --git a/Cargo.toml b/Cargo.toml index 7774ef4..04bb1b9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ members = [ resolver = "2" [workspace.package] -version = "0.2.2" +version = "0.3.0" edition = "2021" license = "Apache-2.0 OR MIT" repository = "https://github.com/tweedegolf/memory-serve" diff --git a/README.md b/README.md index 0ae46bb..c72a61b 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,9 @@ scripts into the rust binary at compile time and exposes them as an [axum](https://github.com/tokio-rs/axum) Router. It automatically adds cache headers and handles file compression. +During development (debug builds) files are served dynamically, +they are read and compressed at request time. + Text-based files like HTML or javascript are compressed using [brotli](https://en.wikipedia.org/wiki/Brotli) at compile time and decompressed at startup, to minimize the binary size. diff --git a/examples/test/src/main.rs b/examples/test/src/main.rs index c52dd99..e8c8e10 100644 --- a/examples/test/src/main.rs +++ b/examples/test/src/main.rs @@ -7,7 +7,7 @@ use tracing::info; async fn main() { tracing_subscriber::fmt().init(); - let memory_router = MemoryServe::new(load_assets!("static")) + let memory_router = MemoryServe::new(load_assets!("../../static")) .index_file(Some("/index.html")) .into_router(); diff --git a/memory-serve-macros/src/asset.rs b/memory-serve-macros/src/asset.rs index c1870fd..88d853c 100644 --- a/memory-serve-macros/src/asset.rs +++ b/memory-serve-macros/src/asset.rs @@ -3,6 +3,7 @@ use syn::LitByteStr; /// Internal data structure pub(crate) struct Asset { pub(crate) route: String, + pub(crate) path: String, pub(crate) etag: String, pub(crate) content_type: String, pub(crate) bytes: LitByteStr, diff --git a/memory-serve-macros/src/lib.rs b/memory-serve-macros/src/lib.rs index 678eea4..af3ad49 100644 --- a/memory-serve-macros/src/lib.rs +++ b/memory-serve-macros/src/lib.rs @@ -1,5 +1,5 @@ use proc_macro::TokenStream; -use std::path::Path; +use std::{env, path::Path}; use utils::list_assets; mod asset; @@ -11,7 +11,7 @@ use crate::asset::Asset; pub fn load_assets(input: TokenStream) -> TokenStream { let input = input.to_string(); let input = input.trim_matches('"'); - let path = Path::new(&input); + let mut asset_path = Path::new(&input).to_path_buf(); // skip if a subscriber is already registered (for instance by rust_analyzer) let _ = tracing_subscriber::fmt() @@ -19,13 +19,24 @@ pub fn load_assets(input: TokenStream) -> TokenStream { .with_target(false) .try_init(); - if !path.exists() { - panic!("The path {:?} does not exists!", path); + if asset_path.is_relative() { + let crate_dir = env::var("CARGO_MANIFEST_DIR") + .expect("CARGO_MANIFEST_DIR environment variable not set"); + asset_path = Path::new(&crate_dir).join(asset_path); } - let files: Vec = list_assets(path); + asset_path = asset_path + .canonicalize() + .expect("Could not canonicalize the provided path"); + + if !asset_path.exists() { + panic!("The path {:?} does not exists!", asset_path); + } + + let files: Vec = list_assets(&asset_path); let route = files.iter().map(|a| &a.route); + let path = files.iter().map(|a| &a.path); let content_type = files.iter().map(|a| &a.content_type); let etag = files.iter().map(|a| &a.etag); let bytes = files.iter().map(|a| &a.bytes); @@ -35,6 +46,7 @@ pub fn load_assets(input: TokenStream) -> TokenStream { &[ #(memory_serve::Asset { route: #route, + path: #path, content_type: #content_type, etag: #etag, bytes: #bytes, diff --git a/memory-serve-macros/src/utils.rs b/memory-serve-macros/src/utils.rs index 12a51a7..7818d28 100644 --- a/memory-serve-macros/src/utils.rs +++ b/memory-serve-macros/src/utils.rs @@ -18,18 +18,21 @@ const COMPRESS_TYPES: &[&str] = &[ "image/svg+xml", ]; -fn path_to_route(path: &Path) -> String { - "/".to_owned() - + path - .components() - .filter_map(|c| match c { - std::path::Component::Normal(s) => s.to_str(), - _ => None, - }) - .skip(1) - .collect::>() - .join("/") - .as_str() +fn path_to_route(base: &Path, path: &Path) -> String { + let relative_path = path + .strip_prefix(base) + .expect("Could not strap prefix from path"); + + let route = relative_path + .components() + .filter_map(|c| match c { + std::path::Component::Normal(s) => s.to_str(), + _ => None, + }) + .collect::>() + .join("/"); + + format!("/{route}") } fn path_to_content_type(path: &Path) -> Option { @@ -63,16 +66,20 @@ fn skip_larger(compressed: Vec, original: &[u8]) -> Vec { } } -pub(crate) fn list_assets>(path: P) -> Vec { - let mut assets: Vec = WalkDir::new(path) +pub(crate) fn list_assets(base_path: &Path) -> Vec { + let mut assets: Vec = WalkDir::new(base_path) .into_iter() .filter_map(|entry| entry.ok()) .filter_map(|entry| { + let Some(path) = entry.path().to_str() else { + warn!("invalid file path {:?}", entry.path()); + return None; + }; + + let route = path_to_route(base_path, entry.path()); + let Ok(metadata) = entry.metadata() else { - warn!( - "skipping file {:?}, could not get file metadata", - entry.path() - ); + warn!("skipping file {route}, could not get file metadata"); return None; }; @@ -81,22 +88,31 @@ pub(crate) fn list_assets>(path: P) -> Vec { return None; }; - let Some(content_type) = path_to_content_type(entry.path()) else { - warn!( - "skipping file {:?}, could not determine file extension", - entry.path() - ); + // skip empty + if metadata.len() == 0 { + warn!("skipping file {route}: file empty"); return None; - }; + } - let Ok(bytes) = std::fs::read(entry.path()) else { - warn!("skipping file {:?}: file is not readable", entry.path()); + let Some(content_type) = path_to_content_type(entry.path()) else { + warn!("skipping file {route}, could not determine file extension"); return None; }; - // skip empty - if bytes.is_empty() { - warn!("skipping file {:?}: file empty", entry.path()); + // do not load assets into the binary in debug / development mode + if cfg!(debug_assertions) { + return Some(Asset { + route, + path: path.to_owned(), + content_type, + etag: Default::default(), + bytes: literal_bytes(Default::default()), + brotli_bytes: literal_bytes(Default::default()), + }); + } + + let Ok(bytes) = std::fs::read(entry.path()) else { + warn!("skipping file {route}: file is not readable"); return None; }; @@ -110,14 +126,11 @@ pub(crate) fn list_assets>(path: P) -> Vec { Default::default() }; - let route = path_to_route(entry.path()); - if brotli_bytes.is_empty() { - info!("including {:?} {} bytes", route, bytes.len()); + info!("including {route} {} bytes", bytes.len()); } else { info!( - "including {:?} {} -> {} bytes (compressed)", - route, + "including {route} {} -> {} bytes (compressed)", bytes.len(), brotli_bytes.len() ); @@ -125,6 +138,7 @@ pub(crate) fn list_assets>(path: P) -> Vec { Some(Asset { route, + path: path.to_owned(), content_type, etag, bytes: literal_bytes(if brotli_bytes.is_empty() { diff --git a/memory-serve/Cargo.toml b/memory-serve/Cargo.toml index cc0a3c9..0ca4c7f 100644 --- a/memory-serve/Cargo.toml +++ b/memory-serve/Cargo.toml @@ -11,8 +11,9 @@ description.workspace = true brotli = "3.4" flate2 = "1.0" axum = { version = "0.6", default-features = false } -memory-serve-macros = { version = "0.2", path = "../memory-serve-macros" } +memory-serve-macros = { version = "0.3", path = "../memory-serve-macros" } tracing = "0.1" +sha256 = "1.4" [dev-dependencies] tokio = { version = "1.33", features = ["rt", "macros"] } diff --git a/memory-serve/src/asset.rs b/memory-serve/src/asset.rs index 36b3546..6c61914 100644 --- a/memory-serve/src/asset.rs +++ b/memory-serve/src/asset.rs @@ -3,14 +3,26 @@ use axum::{ header::{CONTENT_ENCODING, CONTENT_TYPE, ETAG, IF_NONE_MATCH}, HeaderMap, HeaderName, HeaderValue, StatusCode, }, - response::IntoResponse, + response::{IntoResponse, Response}, }; +use tracing::{debug, error}; use crate::{ - util::{content_length, supports_encoding}, + util::{compress_brotli, compress_gzip, content_length, supports_encoding}, ServeOptions, }; +pub const COMPRESS_TYPES: &[&str] = &[ + "text/html", + "text/css", + "application/json", + "application/javascript", + "text/javascript", + "application/xml", + "text/xml", + "image/svg+xml", +]; + const BROTLI_ENCODING: &str = "br"; #[allow(clippy::declare_interior_mutable_const)] const BROTLI_HEADER: (HeaderName, HeaderValue) = @@ -23,82 +35,173 @@ const GZIP_HEADER: (HeaderName, HeaderValue) = #[derive(Debug)] pub struct Asset { pub route: &'static str, + pub path: &'static str, pub etag: &'static str, pub content_type: &'static str, pub bytes: &'static [u8], pub brotli_bytes: &'static [u8], } -impl Asset { - pub(super) fn handler( - &self, - headers: &HeaderMap, - status: StatusCode, - bytes: &'static [u8], - gzip_bytes: &'static [u8], - options: &ServeOptions, - ) -> impl IntoResponse { - let content_type = HeaderValue::from_static(self.content_type); - let etag = HeaderValue::from_static(self.etag); - let cache_control = match self.content_type { - "text/html" => options.html_cache_control.as_header(), - _ => options.cache_control.as_header(), - }; +struct AssetResponse<'t, B> { + options: &'t ServeOptions, + headers: &'t HeaderMap, + status: StatusCode, + asset: &'t Asset, + etag: &'t str, + bytes: B, + bytes_len: usize, + brotli_bytes: B, + brotli_bytes_len: usize, + gzip_bytes: B, + gzip_bytes_len: usize, +} + +impl<'t, B: IntoResponse> AssetResponse<'t, B> { + fn into_response(self) -> Response { + let content_type = self.asset.content_type(); + let cache_control = self.asset.cache_control(self.options); + let etag_header = (ETAG, HeaderValue::from_str(self.etag).unwrap()); - if let Some(if_none_match) = headers.get(IF_NONE_MATCH) { + if let Some(if_none_match) = self.headers.get(IF_NONE_MATCH) { if if_none_match == self.etag { return ( StatusCode::NOT_MODIFIED, - [(CONTENT_TYPE, content_type), cache_control, (ETAG, etag)], + [content_type, cache_control, etag_header], ) .into_response(); } } - if options.enable_brotli - && !self.brotli_bytes.is_empty() - && supports_encoding(headers, BROTLI_ENCODING) + if self.options.enable_brotli + && self.brotli_bytes_len > 0 + && supports_encoding(self.headers, BROTLI_ENCODING) { - ( - status, + return ( + self.status, [ - content_length(self.brotli_bytes.len()), + content_length(self.brotli_bytes_len), BROTLI_HEADER, - (CONTENT_TYPE, content_type), + content_type, cache_control, - (ETAG, etag), + etag_header, ], self.brotli_bytes, ) - .into_response() - } else if options.enable_gzip - && !gzip_bytes.is_empty() - && supports_encoding(headers, GZIP_ENCODING) + .into_response(); + } + + if self.options.enable_gzip + && self.gzip_bytes_len > 0 + && supports_encoding(self.headers, GZIP_ENCODING) { - ( - status, + return ( + self.status, [ - content_length(gzip_bytes.len()), + content_length(self.gzip_bytes_len), GZIP_HEADER, - (CONTENT_TYPE, content_type), + content_type, cache_control, - (ETAG, etag), + etag_header, ], - gzip_bytes, + self.gzip_bytes, ) - .into_response() + .into_response(); + } + + ( + self.status, + [ + content_length(self.bytes_len), + content_type, + cache_control, + etag_header, + ], + self.bytes, + ) + .into_response() + } +} + +impl Asset { + fn cache_control(&self, options: &ServeOptions) -> (HeaderName, HeaderValue) { + match self.content_type { + "text/html" => options.html_cache_control.as_header(), + _ => options.cache_control.as_header(), + } + } + + fn content_type(&self) -> (HeaderName, HeaderValue) { + (CONTENT_TYPE, HeaderValue::from_static(self.content_type)) + } + + fn dynamic_handler( + &self, + headers: &HeaderMap, + status: StatusCode, + options: &ServeOptions, + ) -> Response { + let Ok(bytes) = std::fs::read(self.path) else { + error!("File not found {}", self.path); + return StatusCode::NOT_FOUND.into_response(); + }; + + let brotli_bytes = if options.enable_brotli && COMPRESS_TYPES.contains(&self.content_type) { + compress_brotli(&bytes).unwrap_or_default() } else { - ( - status, - [ - content_length(bytes.len()), - (CONTENT_TYPE, content_type), - cache_control, - (ETAG, etag), - ], - bytes, - ) - .into_response() + Default::default() + }; + + let gzip_bytes = if options.enable_gzip && COMPRESS_TYPES.contains(&self.content_type) { + compress_gzip(&bytes).unwrap_or_default() + } else { + Default::default() + }; + + let etag = sha256::digest(&bytes); + + AssetResponse { + options, + headers, + status, + asset: self, + etag: &etag, + bytes_len: bytes.len(), + bytes, + brotli_bytes_len: brotli_bytes.len(), + brotli_bytes, + gzip_bytes_len: gzip_bytes.len(), + gzip_bytes, + } + .into_response() + } + + pub(super) fn handler( + &self, + headers: &HeaderMap, + status: StatusCode, + bytes: &'static [u8], + gzip_bytes: &'static [u8], + options: &ServeOptions, + ) -> Response { + if bytes.is_empty() { + debug!("using dynamic handler for {}", self.path); + + return self.dynamic_handler(headers, status, options); + } + + AssetResponse { + options, + headers, + status, + asset: self, + etag: self.etag, + bytes_len: bytes.len(), + bytes, + brotli_bytes_len: self.brotli_bytes.len(), + brotli_bytes: self.brotli_bytes, + gzip_bytes_len: gzip_bytes.len(), + gzip_bytes, } + .into_response() } } diff --git a/memory-serve/src/lib.rs b/memory-serve/src/lib.rs index 5f174e6..3af8900 100644 --- a/memory-serve/src/lib.rs +++ b/memory-serve/src/lib.rs @@ -6,6 +6,9 @@ //! [axum](https://github.com/tokio-rs/axum) Router. It automatically adds cache //! headers and handles file compression. //! +//! During development (debug builds) files are served dynamically, +//! they are read and compressed at request time. +//! //! Text-based files like HTML or javascript //! are compressed using [brotli](https://en.wikipedia.org/wiki/Brotli) //! at compile time and decompressed at startup, to minimize the binary size. @@ -45,7 +48,7 @@ //! //! #[tokio::main] //! async fn main() { -//! let memory_router = MemoryServe::new(load_assets!("static")) +//! let memory_router = MemoryServe::new(load_assets!("../static")) //! .index_file(Some("/index.html")) //! .into_router(); //! @@ -269,7 +272,7 @@ impl MemoryServe { let options = Box::leak(Box::new(self.options)); for asset in self.assets { - let bytes = if asset.bytes.is_empty() { + let bytes = if asset.bytes.is_empty() && !asset.brotli_bytes.is_empty() { Box::new(decompress_brotli(asset.brotli_bytes).unwrap_or_default()).leak() } else { asset.bytes @@ -281,22 +284,17 @@ impl MemoryServe { Default::default() }; - info!("serving {} {} bytes", asset.route, bytes.len()); - - if !asset.brotli_bytes.is_empty() { - info!( - "serving {} (brotli compressed) {} bytes", - asset.route, - asset.brotli_bytes.len() - ); - } - - if !gzip_bytes.is_empty() { - info!( - "serving {} (gzip compressed) {} bytes", - asset.route, - gzip_bytes.len() - ); + if !bytes.is_empty() { + if !asset.brotli_bytes.is_empty() { + info!( + "serving {} {} -> {} bytes (compressed)", + asset.route, + bytes.len(), + asset.brotli_bytes.is_empty() + ); + } else { + info!("serving {} {} bytes", asset.route, bytes.len()); + } } let handler = |headers: HeaderMap| { @@ -318,7 +316,7 @@ impl MemoryServe { } if Some(asset.route) == options.index_file { - info!("serving {} as index on /", asset.route,); + info!("serving {} as index on /", asset.route); router = router.route("/", get(handler)); } @@ -328,6 +326,8 @@ impl MemoryServe { // add all aliases that point to the asset route for (from, to) in self.aliases.iter() { if *to == asset.route { + info!("serving {} as index on {}", asset.route, from); + router = router.route(from, get(handler)); } } @@ -381,7 +381,7 @@ mod tests { #[test] fn test_load_assets() { - let assets: &'static [Asset] = load_assets!("static"); + let assets: &'static [Asset] = load_assets!("../static"); let routes: Vec<&str> = assets.iter().map(|a| a.route).collect(); let content_types: Vec<&str> = assets.iter().map(|a| a.content_type).collect(); let etags: Vec<&str> = assets.iter().map(|a| a.etag).collect(); @@ -406,21 +406,25 @@ mod tests { "text/html" ] ); - assert_eq!( - etags, - [ - "e64f4683bf82d854df40b7246666f6f0816666ad8cd886a8e159535896eb03d6", - "ec4edeea111c854901385011f403e1259e3f1ba016dcceabb6d566316be3677b", - "86a7fdfd19700843e5f7344a63d27e0b729c2554c8572903ceee71f5658d2ecf", - "bd9dccc152de48cb7bedc35b9748ceeade492f6f904710f9c5d480bd6299cc7d", - "0639dc8aac157b58c74f65bbb026b2fd42bc81d9a0a64141df456fa23c214537" - ] - ); + if cfg!(debug_assertions) { + assert_eq!(etags, ["", "", "", "", ""]); + } else { + assert_eq!( + etags, + [ + "e64f4683bf82d854df40b7246666f6f0816666ad8cd886a8e159535896eb03d6", + "ec4edeea111c854901385011f403e1259e3f1ba016dcceabb6d566316be3677b", + "86a7fdfd19700843e5f7344a63d27e0b729c2554c8572903ceee71f5658d2ecf", + "bd9dccc152de48cb7bedc35b9748ceeade492f6f904710f9c5d480bd6299cc7d", + "0639dc8aac157b58c74f65bbb026b2fd42bc81d9a0a64141df456fa23c214537" + ] + ); + } } #[tokio::test] async fn if_none_match_handling() { - let memory_router = MemoryServe::new(load_assets!("static")).into_router(); + let memory_router = MemoryServe::new(load_assets!("../static")).into_router(); let (code, headers) = get(memory_router.clone(), "/index.html", "accept", "text/html").await; let etag: &str = headers.get(header::ETAG).unwrap().to_str().unwrap(); @@ -440,7 +444,7 @@ mod tests { #[tokio::test] async fn brotli_compression() { - let memory_router = MemoryServe::new(load_assets!("static")).into_router(); + let memory_router = MemoryServe::new(load_assets!("../static")).into_router(); let (code, headers) = get( memory_router.clone(), "/index.html", @@ -456,7 +460,7 @@ mod tests { assert_eq!(length.parse::().unwrap(), 178); // check disable compression - let memory_router = MemoryServe::new(load_assets!("static")) + let memory_router = MemoryServe::new(load_assets!("../static")) .enable_brotli(false) .into_router(); let (code, headers) = get( @@ -474,7 +478,7 @@ mod tests { #[tokio::test] async fn gzip_compression() { - let memory_router = MemoryServe::new(load_assets!("static")).into_router(); + let memory_router = MemoryServe::new(load_assets!("../static")).into_router(); let (code, headers) = get( memory_router.clone(), "/index.html", @@ -490,7 +494,7 @@ mod tests { assert_eq!(length.parse::().unwrap(), 274); // check disable compression - let memory_router = MemoryServe::new(load_assets!("static")) + let memory_router = MemoryServe::new(load_assets!("../static")) .enable_gzip(false) .into_router(); let (code, headers) = get( @@ -508,14 +512,14 @@ mod tests { #[tokio::test] async fn index_file() { - let memory_router = MemoryServe::new(load_assets!("static")) + let memory_router = MemoryServe::new(load_assets!("../static")) .index_file(None) .into_router(); let (code, _) = get(memory_router.clone(), "/", "accept", "*").await; assert_eq!(code, 404); - let memory_router = MemoryServe::new(load_assets!("static")) + let memory_router = MemoryServe::new(load_assets!("../static")) .index_file(Some("/index.html")) .into_router(); @@ -525,11 +529,11 @@ mod tests { #[tokio::test] async fn fallback() { - let memory_router = MemoryServe::new(load_assets!("static")).into_router(); + let memory_router = MemoryServe::new(load_assets!("../static")).into_router(); let (code, _) = get(memory_router.clone(), "/foobar", "accept", "*").await; assert_eq!(code, 404); - let memory_router = MemoryServe::new(load_assets!("static")) + let memory_router = MemoryServe::new(load_assets!("../static")) .fallback(Some("/index.html")) .into_router(); let (code, headers) = get(memory_router.clone(), "/foobar", "accept", "*").await; @@ -537,7 +541,7 @@ mod tests { assert_eq!(code, 404); assert_eq!(length.parse::().unwrap(), 437); - let memory_router = MemoryServe::new(load_assets!("static")) + let memory_router = MemoryServe::new(load_assets!("../static")) .fallback(Some("/index.html")) .fallback_status(StatusCode::OK) .into_router(); @@ -550,7 +554,7 @@ mod tests { #[tokio::test] async fn cache_control() { async fn check_cache_control(cache_control: CacheControl, expected: &str) { - let memory_router = MemoryServe::new(load_assets!("static")) + let memory_router = MemoryServe::new(load_assets!("../static")) .cache_control(cache_control) .into_router(); @@ -584,7 +588,7 @@ mod tests { .await; async fn check_html_cache_control(cache_control: CacheControl, expected: &str) { - let memory_router = MemoryServe::new(load_assets!("static")) + let memory_router = MemoryServe::new(load_assets!("../static")) .html_cache_control(cache_control) .into_router(); @@ -618,7 +622,7 @@ mod tests { #[tokio::test] async fn aliases() { - let memory_router = MemoryServe::new(load_assets!("static")) + let memory_router = MemoryServe::new(load_assets!("../static")) .add_alias("/foobar", "/index.html") .add_alias("/baz", "/index.html") .into_router(); diff --git a/memory-serve/src/util.rs b/memory-serve/src/util.rs index d865828..2f2bd00 100644 --- a/memory-serve/src/util.rs +++ b/memory-serve/src/util.rs @@ -12,6 +12,13 @@ pub(crate) fn decompress_brotli(input: &[u8]) -> Option> { writer.into_inner().ok() } +pub(crate) fn compress_brotli(input: &[u8]) -> Option> { + let mut writer = brotli::CompressorWriter::new(Vec::new(), 4096, 11, 22); + writer.write_all(input).ok()?; + + Some(writer.into_inner()) +} + pub(crate) fn compress_gzip(input: &[u8]) -> Option> { let mut writer = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); writer.write_all(input).ok()?;