Serve assets dynamically during development

.github/workflows/ci.yml

@@ -58,6 +58,16 @@ jobs:
         shared-key: test
     - run: cargo test --all-features --all-targets
 
+  test-release:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - uses: Swatinem/rust-cache@v2
+      with:
+        prefix-key: cargo
+        shared-key: test
+    - run: cargo test --all-features --all-targets --release
+
   udeps:
     runs-on: ubuntu-latest
     steps:

Cargo.toml

@@ -7,7 +7,7 @@ members = [
 resolver = "2"
 
 [workspace.package]
-version = "0.2.2"
+version = "0.3.0"
 edition = "2021"
 license = "Apache-2.0 OR MIT"
 repository = "https://github.com/tweedegolf/memory-serve"

README.md

@@ -8,6 +8,9 @@ scripts into the rust binary at compile time and exposes them as an
 [axum](https://github.com/tokio-rs/axum) Router. It automatically adds cache
 headers and handles file compression.
 
+During development (debug builds) files are served dynamically,
+they are read and compressed at request time.
+
 Text-based files like HTML or javascript
 are compressed using [brotli](https://en.wikipedia.org/wiki/Brotli)
 at compile time and decompressed at startup, to minimize the binary size.

examples/test/src/main.rs

@@ -7,7 +7,7 @@ use tracing::info;
 async fn main() {
     tracing_subscriber::fmt().init();
 
-    let memory_router = MemoryServe::new(load_assets!("static"))
+    let memory_router = MemoryServe::new(load_assets!("../../static"))
         .index_file(Some("/index.html"))
         .into_router();
 

memory-serve-macros/src/asset.rs

@@ -3,6 +3,7 @@ use syn::LitByteStr;
 /// Internal data structure
 pub(crate) struct Asset {
     pub(crate) route: String,
+    pub(crate) path: String,
     pub(crate) etag: String,
     pub(crate) content_type: String,
     pub(crate) bytes: LitByteStr,

memory-serve-macros/src/lib.rs

@@ -1,5 +1,5 @@
 use proc_macro::TokenStream;
-use std::path::Path;
+use std::{env, path::Path};
 use utils::list_assets;
 
 mod asset;
@@ -11,21 +11,32 @@ use crate::asset::Asset;
 pub fn load_assets(input: TokenStream) -> TokenStream {
     let input = input.to_string();
     let input = input.trim_matches('"');
-    let path = Path::new(&input);
+    let mut asset_path = Path::new(&input).to_path_buf();
 
     // skip if a subscriber is already registered (for instance by rust_analyzer)
     let _ = tracing_subscriber::fmt()
         .without_time()
         .with_target(false)
         .try_init();
 
-    if !path.exists() {
-        panic!("The path {:?} does not exists!", path);
+    if asset_path.is_relative() {
+        let crate_dir = env::var("CARGO_MANIFEST_DIR")
+            .expect("CARGO_MANIFEST_DIR environment variable not set");
+        asset_path = Path::new(&crate_dir).join(asset_path);
     }
 
-    let files: Vec<Asset> = list_assets(path);
+    asset_path = asset_path
+        .canonicalize()
+        .expect("Could not canonicalize the provided path");
+
+    if !asset_path.exists() {
+        panic!("The path {:?} does not exists!", asset_path);
+    }
+
+    let files: Vec<Asset> = list_assets(&asset_path);
 
     let route = files.iter().map(|a| &a.route);
+    let path = files.iter().map(|a| &a.path);
     let content_type = files.iter().map(|a| &a.content_type);
     let etag = files.iter().map(|a| &a.etag);
     let bytes = files.iter().map(|a| &a.bytes);
@@ -35,6 +46,7 @@ pub fn load_assets(input: TokenStream) -> TokenStream {
         &[
             #(memory_serve::Asset {
                 route: #route,
+                path: #path,
                 content_type: #content_type,
                 etag: #etag,
                 bytes: #bytes,

memory-serve-macros/src/utils.rs

@@ -18,18 +18,21 @@ const COMPRESS_TYPES: &[&str] = &[
     "image/svg+xml",
 ];
 
-fn path_to_route(path: &Path) -> String {
-    "/".to_owned()
-        + path
-            .components()
-            .filter_map(|c| match c {
-                std::path::Component::Normal(s) => s.to_str(),
-                _ => None,
-            })
-            .skip(1)
-            .collect::<Vec<&str>>()
-            .join("/")
-            .as_str()
+fn path_to_route(base: &Path, path: &Path) -> String {
+    let relative_path = path
+        .strip_prefix(base)
+        .expect("Could not strap prefix from path");
+
+    let route = relative_path
+        .components()
+        .filter_map(|c| match c {
+            std::path::Component::Normal(s) => s.to_str(),
+            _ => None,
+        })
+        .collect::<Vec<&str>>()
+        .join("/");
+
+    format!("/{route}")
 }
 
 fn path_to_content_type(path: &Path) -> Option<String> {
@@ -63,16 +66,20 @@ fn skip_larger(compressed: Vec<u8>, original: &[u8]) -> Vec<u8> {
     }
 }
 
-pub(crate) fn list_assets<P: AsRef<Path>>(path: P) -> Vec<Asset> {
-    let mut assets: Vec<Asset> = WalkDir::new(path)
+pub(crate) fn list_assets(base_path: &Path) -> Vec<Asset> {
+    let mut assets: Vec<Asset> = WalkDir::new(base_path)
         .into_iter()
         .filter_map(|entry| entry.ok())
         .filter_map(|entry| {
+            let Some(path) = entry.path().to_str() else {
+                warn!("invalid file path {:?}", entry.path());
+                return None;
+            };
+
+            let route = path_to_route(base_path, entry.path());
+
             let Ok(metadata) = entry.metadata() else {
-                warn!(
-                    "skipping file {:?}, could not get file metadata",
-                    entry.path()
-                );
+                warn!("skipping file {route}, could not get file metadata");
                 return None;
             };
 
@@ -81,22 +88,31 @@ pub(crate) fn list_assets<P: AsRef<Path>>(path: P) -> Vec<Asset> {
                 return None;
             };
 
-            let Some(content_type) = path_to_content_type(entry.path()) else {
-                warn!(
-                    "skipping file {:?}, could not determine file extension",
-                    entry.path()
-                );
+            // skip empty
+            if metadata.len() == 0 {
+                warn!("skipping file {route}: file empty");
                 return None;
-            };
+            }
 
-            let Ok(bytes) = std::fs::read(entry.path()) else {
-                warn!("skipping file {:?}: file is not readable", entry.path());
+            let Some(content_type) = path_to_content_type(entry.path()) else {
+                warn!("skipping file {route}, could not determine file extension");
                 return None;
             };
 
-            // skip empty
-            if bytes.is_empty() {
-                warn!("skipping file {:?}: file empty", entry.path());
+            // do not load assets into the binary in debug / development mode
+            if cfg!(debug_assertions) {
+                return Some(Asset {
+                    route,
+                    path: path.to_owned(),
+                    content_type,
+                    etag: Default::default(),
+                    bytes: literal_bytes(Default::default()),
+                    brotli_bytes: literal_bytes(Default::default()),
+                });
+            }
+
+            let Ok(bytes) = std::fs::read(entry.path()) else {
+                warn!("skipping file {route}: file is not readable");
                 return None;
             };
 
@@ -110,21 +126,19 @@ pub(crate) fn list_assets<P: AsRef<Path>>(path: P) -> Vec<Asset> {
                 Default::default()
             };
 
-            let route = path_to_route(entry.path());
-
             if brotli_bytes.is_empty() {
-                info!("including {:?} {} bytes", route, bytes.len());
+                info!("including {route} {} bytes", bytes.len());
             } else {
                 info!(
-                    "including {:?} {} -> {} bytes (compressed)",
-                    route,
+                    "including {route} {} -> {} bytes (compressed)",
                     bytes.len(),
                     brotli_bytes.len()
                 );
             };
 
             Some(Asset {
                 route,
+                path: path.to_owned(),
                 content_type,
                 etag,
                 bytes: literal_bytes(if brotli_bytes.is_empty() {

memory-serve/Cargo.toml

@@ -11,8 +11,9 @@ description.workspace = true
 brotli = "3.4"
 flate2 = "1.0"
 axum = { version = "0.6", default-features = false }
-memory-serve-macros = { version = "0.2", path = "../memory-serve-macros" }
+memory-serve-macros = { version = "0.3", path = "../memory-serve-macros" }
 tracing = "0.1"
+sha256 = "1.4"
 
 [dev-dependencies]
 tokio = { version = "1.33", features = ["rt", "macros"] }