Add http & https asset sources

.gitignore

@@ -21,6 +21,7 @@ dxil.dll
 assets/**/*.meta
 crates/bevy_asset/imported_assets
 imported_assets
+.http-asset-cache
 
 # Bevy Examples
 example_showcase_config.ron

Cargo.toml

@@ -433,6 +433,12 @@ file_watcher = ["bevy_internal/file_watcher"]
 # Enables watching in memory asset providers for Bevy Asset hot-reloading
 embedded_watcher = ["bevy_internal/embedded_watcher"]
 
+# Enables using assets from HTTP sources
+http_source = ["bevy_internal/http_source"]
+
+# Assets downloaded from HTTP sources are cached
+http_source_cache = ["bevy_internal/http_source_cache"]
+
 # Enable stepping-based debugging of Bevy systems
 bevy_debug_stepping = ["bevy_internal/bevy_debug_stepping"]
 
@@ -1637,12 +1643,23 @@ path = "examples/asset/extra_source.rs"
 doc-scrape-examples = true
 
 [package.metadata.example.extra_asset_source]
-name = "Extra asset source"
+name = "Extra Asset Source"
 description = "Load an asset from a non-standard asset source"
 category = "Assets"
 # Uses non-standard asset path
 wasm = false
 
+[[example]]
+name = "http_source"
+path = "examples/asset/http_source.rs"
+doc-scrape-examples = true
+
+[package.metadata.example.http_source]
+name = "HTTP Asset Source"
+description = "Load an asset from a http source"
+category = "Assets"
+wasm = true
+
 [[example]]
 name = "hot_asset_reloading"
 path = "examples/asset/hot_asset_reloading.rs"

crates/bevy_asset/Cargo.toml

@@ -14,6 +14,8 @@ keywords = ["bevy"]
 file_watcher = ["notify-debouncer-full", "watch"]
 embedded_watcher = ["file_watcher"]
 multi_threaded = ["bevy_tasks/multi_threaded"]
+http_source = ["ureq"]
+http_source_cache = []
 asset_processor = []
 watch = []
 trace = []
@@ -63,6 +65,7 @@ js-sys = "0.3"
 
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
 notify-debouncer-full = { version = "0.4.0", optional = true }
+ureq = { version = "2.10", optional = true }
 
 [dev-dependencies]
 bevy_core = { path = "../bevy_core", version = "0.15.0-dev" }

crates/bevy_asset/src/http_source.rs

@@ -0,0 +1,245 @@
+use crate::io::{AssetReader, AssetReaderError, Reader};
+use crate::io::{AssetSource, PathStream};
+use crate::AssetApp;
+use bevy_app::App;
+use bevy_utils::ConditionalSendFuture;
+use std::path::{Path, PathBuf};
+
+/// Adds the `http` and `https` asset sources to the app.
+/// Any asset path that begins with `http` or `https` will be loaded from the web
+/// via `fetch`(wasm) or `ureq`(native).
+pub fn http_source_plugin(app: &mut App) {
+    app.register_asset_source(
+        "http",
+        AssetSource::build().with_reader(|| Box::new(HttpSourceAssetReader::Http)),
+    );
+    app.register_asset_source(
+        "https",
+        AssetSource::build().with_reader(|| Box::new(HttpSourceAssetReader::Https)),
+    );
+}
+
+/// Asset reader that treats paths as urls to load assets from.
+pub enum HttpSourceAssetReader {
+    /// Unencrypted connections.
+    Http,
+    /// Use TLS for setting up connections.
+    Https,
+}
+
+impl HttpSourceAssetReader {
+    fn make_uri(&self, path: &Path) -> PathBuf {
+        PathBuf::from(match self {
+            Self::Http => "http://",
+            Self::Https => "https://",
+        })
+        .join(path)
+    }
+
+    /// See [`crate::io::get_meta_path`]
+    fn make_meta_uri(&self, path: &Path) -> Option<PathBuf> {
+        let mut uri = self.make_uri(path);
+        let mut extension = path.extension()?.to_os_string();
+        extension.push(".meta");
+        uri.set_extension(extension);
+        Some(uri)
+    }
+}
+
+#[cfg(target_arch = "wasm32")]
+async fn get<'a>(path: PathBuf) -> Result<Box<dyn Reader>, AssetReaderError> {
+    use crate::io::wasm::HttpWasmAssetReader;
+
+    HttpWasmAssetReader::new("")
+        .fetch_bytes(path)
+        .await
+        .map(|r| Box::new(r) as Box<dyn Reader>)
+}
+
+#[cfg(not(target_arch = "wasm32"))]
+async fn get<'a>(path: PathBuf) -> Result<Box<dyn Reader>, AssetReaderError> {
+    use crate::io::VecReader;
+    use std::io;
+
+    let str_path = path.to_str().ok_or_else(|| {
+        AssetReaderError::Io(
+            io::Error::new(
+                io::ErrorKind::Other,
+                format!("non-utf8 path: {}", path.display()),
+            )
+            .into(),
+        )
+    })?;
+
+    #[cfg(feature = "http_source_cache")]
+    if let Some(data) = http_asset_cache::try_load_from_cache(str_path)? {
+        return Ok(Box::new(VecReader::new(data)));
+    }
+
+    match ureq::get(str_path).call() {
+        Ok(response) => {
+            let mut reader = response.into_reader();
+            let mut buffer = Vec::new();
+            reader.read_to_end(&mut buffer)?;
+
+            #[cfg(feature = "http_source_cache")]
+            http_asset_cache::save_to_cache(str_path, &buffer)?;
+
+            Ok(Box::new(VecReader::new(buffer)))
+        }
+        // ureq considers all >=400 status codes as errors
+        Err(ureq::Error::Status(code, _response)) => {
+            if code == 404 {
+                Err(AssetReaderError::NotFound(path))
+            } else {
+                Err(AssetReaderError::HttpError(code))
+            }
+        }
+        Err(ureq::Error::Transport(err)) => Err(AssetReaderError::Io(
+            io::Error::new(
+                io::ErrorKind::Other,
+                format!(
+                    "unexpected error while loading asset {}: {}",
+                    path.display(),
+                    err
+                ),
+            )
+            .into(),
+        )),
+    }
+}
+
+impl AssetReader for HttpSourceAssetReader {
+    fn read<'a>(
+        &'a self,
+        path: &'a Path,
+    ) -> impl ConditionalSendFuture<Output = Result<Box<dyn Reader>, AssetReaderError>> {
+        get(self.make_uri(path))
+    }
+
+    async fn read_meta<'a>(&'a self, path: &'a Path) -> Result<Box<dyn Reader>, AssetReaderError> {
+        match self.make_meta_uri(path) {
+            Some(uri) => get(uri).await,
+            None => Err(AssetReaderError::NotFound(
+                "source path has no extension".into(),
+            )),
+        }
+    }
+
+    async fn is_directory<'a>(&'a self, _path: &'a Path) -> Result<bool, AssetReaderError> {
+        Ok(false)
+    }
+
+    async fn read_directory<'a>(
+        &'a self,
+        path: &'a Path,
+    ) -> Result<Box<PathStream>, AssetReaderError> {
+        Err(AssetReaderError::NotFound(self.make_uri(path)))
+    }
+}
+
+/// A naive implementation of an HTTP asset cache that never invalidates.
+/// `ureq` currently does not support caching, so this is a simple workaround.
+/// It should eventually be replaced by `http-cache` or similar, see [tracking issue](https://github.com/06chaynes/http-cache/issues/91)
+#[cfg(feature = "http_source_cache")]
+mod http_asset_cache {
+    use core::hash::{Hash, Hasher};
+    use std::collections::hash_map::DefaultHasher;
+    use std::fs::{self, File};
+    use std::io::{self, Read, Write};
+    use std::path::PathBuf;
+
+    const CACHE_DIR: &str = ".http-asset-cache";
+
+    fn url_to_hash(url: &str) -> String {
+        let mut hasher = DefaultHasher::new();
+        url.hash(&mut hasher);
+        format!("{:x}", hasher.finish())
+    }
+
+    pub fn try_load_from_cache(url: &str) -> Result<Option<Vec<u8>>, io::Error> {
+        let filename = url_to_hash(url);
+        let cache_path = PathBuf::from(CACHE_DIR).join(&filename);
+
+        if cache_path.exists() {
+            let mut file = File::open(&cache_path)?;
+            let mut buffer = Vec::new();
+            file.read_to_end(&mut buffer)?;
+            Ok(Some(buffer))
+        } else {
+            Ok(None)
+        }
+    }
+
+    pub fn save_to_cache(url: &str, data: &[u8]) -> Result<(), io::Error> {
+        let filename = url_to_hash(url);
+        let cache_path = PathBuf::from(CACHE_DIR).join(&filename);
+
+        fs::create_dir_all(CACHE_DIR).ok();
+
+        let mut cache_file = File::create(&cache_path)?;
+        cache_file.write_all(data)?;
+
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn make_http_uri() {
+        assert_eq!(
+            HttpSourceAssetReader::Http
+                .make_uri(Path::new("s3.johanhelsing.studio/dump/favicon.png"))
+                .to_str()
+                .unwrap(),
+            "http://s3.johanhelsing.studio/dump/favicon.png"
+        );
+    }
+
+    #[test]
+    fn make_https_uri() {
+        assert_eq!(
+            HttpSourceAssetReader::Https
+                .make_uri(Path::new("s3.johanhelsing.studio/dump/favicon.png"))
+                .to_str()
+                .unwrap(),
+            "https://s3.johanhelsing.studio/dump/favicon.png"
+        );
+    }
+
+    #[test]
+    fn make_http_meta_uri() {
+        assert_eq!(
+            HttpSourceAssetReader::Http
+                .make_meta_uri(Path::new("s3.johanhelsing.studio/dump/favicon.png"))
+                .expect("cannot create meta uri")
+                .to_str()
+                .unwrap(),
+            "http://s3.johanhelsing.studio/dump/favicon.png.meta"
+        );
+    }
+
+    #[test]
+    fn make_https_meta_uri() {
+        assert_eq!(
+            HttpSourceAssetReader::Https
+                .make_meta_uri(Path::new("s3.johanhelsing.studio/dump/favicon.png"))
+                .expect("cannot create meta uri")
+                .to_str()
+                .unwrap(),
+            "https://s3.johanhelsing.studio/dump/favicon.png.meta"
+        );
+    }
+
+    #[test]
+    fn make_https_without_extension_meta_uri() {
+        assert_eq!(
+            HttpSourceAssetReader::Https
+                .make_meta_uri(Path::new("s3.johanhelsing.studio/dump/favicon")),
+            None
+        );
+    }
+}

crates/bevy_asset/src/io/mod.rs

@@ -46,7 +46,8 @@ pub enum AssetReaderError {
     Io(Arc<std::io::Error>),
 
     /// The HTTP request completed but returned an unhandled [HTTP response status code](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status).
-    /// If the request fails before getting a status code (e.g. request timeout, interrupted connection, etc), expect [`AssetReaderError::Io`].
+    /// - If the request returns a 404 error, expect [`AssetReaderError::NotFound`].
+    /// - If the request fails before getting a status code (e.g. request timeout, interrupted connection, etc), expect [`AssetReaderError::Io`].
     #[error("Encountered HTTP status {0:?} when loading asset")]
     HttpError(u16),
 }

crates/bevy_asset/src/io/wasm.rs

@@ -51,7 +51,11 @@ fn js_value_to_err(context: &str) -> impl FnOnce(JsValue) -> std::io::Error + '_
 }
 
 impl HttpWasmAssetReader {
-    async fn fetch_bytes<'a>(&self, path: PathBuf) -> Result<impl Reader, AssetReaderError> {
+    // Also used by HttpSourceAssetReader
+    pub(crate) async fn fetch_bytes<'a>(
+        &self,
+        path: PathBuf,
+    ) -> Result<impl Reader, AssetReaderError> {
         // The JS global scope includes a self-reference via a specializing name, which can be used to determine the type of global context available.
         let global: Global = js_sys::global().unchecked_into();
         let promise = if !global.window().is_undefined() {

crates/bevy_asset/src/lib.rs

@@ -179,6 +179,9 @@ mod reflect;
 mod render_asset;
 mod server;
 
+#[cfg(feature = "http_source")]
+mod http_source;
+
 pub use assets::*;
 pub use bevy_asset_macros::Asset;
 pub use direct_access_ext::DirectAssetAccessExt;
@@ -310,6 +313,9 @@ impl AssetPlugin {
 
 impl Plugin for AssetPlugin {
     fn build(&self, app: &mut App) {
+        #[cfg(feature = "http_source")]
+        app.add_plugins(http_source::http_source_plugin);
+
         let embedded = EmbeddedAssetRegistry::default();
         {
             let mut sources = app

crates/bevy_internal/Cargo.toml

@@ -190,6 +190,12 @@ debug_glam_assert = ["bevy_math/debug_glam_assert"]
 
 default_font = ["bevy_text?/default_font"]
 
+# Enables using assets from HTTP sources
+http_source = ["bevy_asset?/http_source"]
+
+# Assets downloaded from HTTP sources are cached
+http_source_cache = ["bevy_asset?/http_source_cache"]
+
 # Enables the built-in asset processor for processed assets.
 asset_processor = ["bevy_asset?/asset_processor"]
 

docs/cargo_features.md

@@ -76,6 +76,8 @@ The default feature set enables most of the expected features of a game engine,
 |ghost_nodes|Experimental support for nodes that are ignored for UI layouting|
 |gif|GIF image format support|
 |glam_assert|Enable assertions to check the validity of parameters passed to glam|
+|http_source|Enables using assets from HTTP sources|
+|http_source_cache|Assets downloaded from HTTP sources are cached|
 |ico|ICO image format support|
 |ios_simulator|Enable support for the ios_simulator by downgrading some rendering capabilities|
 |jpeg|JPEG image format support|

examples/README.md

@@ -239,7 +239,8 @@ Example | Description
 [Custom Asset](../examples/asset/custom_asset.rs) | Implements a custom asset loader
 [Custom Asset IO](../examples/asset/custom_asset_reader.rs) | Implements a custom AssetReader
 [Embedded Asset](../examples/asset/embedded_asset.rs) | Embed an asset in the application binary and load it
-[Extra asset source](../examples/asset/extra_source.rs) | Load an asset from a non-standard asset source
+[Extra Asset Source](../examples/asset/extra_source.rs) | Load an asset from a non-standard asset source
+[HTTP Asset Source](../examples/asset/http_source.rs) | Load an asset from a http source
 [Hot Reloading of Assets](../examples/asset/hot_asset_reloading.rs) | Demonstrates automatic reloading of assets when modified on disk
 [Mult-asset synchronization](../examples/asset/multi_asset_sync.rs) | Demonstrates how to wait for multiple assets to be loaded.
 [Repeated texture configuration](../examples/asset/repeated_texture.rs) | How to configure the texture to repeat instead of the default clamp to edges