From 7da93bde2361b145970edda07917285b264859b4 Mon Sep 17 00:00:00 2001 From: victor <52110451+cs50victor@users.noreply.github.com> Date: Sat, 25 Nov 2023 11:02:33 -0500 Subject: [PATCH 1/2] feat/ci (#16) * feat: add github action * refactor: ci --- .github/workflows/ci-rs.yml | 5 ++--- .github/workflows/ci-ts.yml | 6 +----- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci-rs.yml b/.github/workflows/ci-rs.yml index dd5ee56..641dcdd 100644 --- a/.github/workflows/ci-rs.yml +++ b/.github/workflows/ci-rs.yml @@ -3,8 +3,7 @@ name: CI-RS on: push: paths: - - "**/server/**" - - "**/crates/**" + - "**/lkgpt/**" - "**/Cargo.lock" - "**/Cargo.toml" - "**/rust-toolchain" @@ -58,4 +57,4 @@ jobs: with: token: ${{ secrets.GITHUB_TOKEN }} kv_token: ${{ secrets.KV_TOKEN }} - included_packages: "repo_guided_graph demo_server" + included_packages: "lkgpt" diff --git a/.github/workflows/ci-ts.yml b/.github/workflows/ci-ts.yml index 23cc125..8f3560d 100644 --- a/.github/workflows/ci-ts.yml +++ b/.github/workflows/ci-ts.yml @@ -3,8 +3,7 @@ name: CI-TS on: push: paths: - - "**/web/**" - - "**/web_shared/**" + - "**/meet/**" - "**/.eslintrc.json" - "**/.node-version" - "**/package.json" @@ -13,9 +12,6 @@ on: env: CI: true - # https://turbo.build/repo/docs/ci/github-actions#remote-caching - TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} - TURBO_TEAM: ${{ vars.TURBO_TEAM }} # Cancel old builds on new commit for same workflow + branch/PR concurrency: From 8a6a1382f4cbff841a2808ab629e4325061eb9df Mon Sep 17 00:00:00 2001 From: victor <52110451+cs50victor@users.noreply.github.com> Date: Sat, 25 Nov 2023 15:48:06 -0500 Subject: [PATCH 2/2] refactor: multi-file mods to single file (#17) --- Cargo.lock | 2511 +++++++++++++++++++++- lkgpt/Cargo.toml | 1 + lkgpt/src/assets/gltf.rs | 216 -- lkgpt/src/assets/materials.rs | 256 --- lkgpt/src/assets/mesh.rs | 101 - lkgpt/src/assets/mod.rs | 5 - lkgpt/src/assets/model.rs | 15 - lkgpt/src/assets/texture.rs | 8 - lkgpt/src/core.rs | 174 -- lkgpt/src/gpt.rs | 81 - lkgpt/src/main.rs | 3097 ++++++++++++++++++++++++++- lkgpt/src/response.rs | 50 - lkgpt/src/room_events.rs | 91 - lkgpt/src/routes/health_check.rs | 7 - lkgpt/src/routes/lsdk_webhook.rs | 80 - lkgpt/src/routes/mod.rs | 8 - lkgpt/src/scene/camera.rs | 61 - lkgpt/src/scene/material_manager.rs | 44 - lkgpt/src/scene/mod.rs | 5 - lkgpt/src/scene/node.rs | 144 -- lkgpt/src/scene/scene.rs | 649 ------ lkgpt/src/scene/texture_manager.rs | 44 - lkgpt/src/state.rs | 32 - lkgpt/src/stt.rs | 168 -- lkgpt/src/track_pub.rs | 71 - lkgpt/src/tts.rs | 281 --- lkgpt/src/turbo.rs | 223 -- lkgpt/src/utils/create_bot_token.rs | 27 - lkgpt/src/utils/mod.rs | 3 - lkgpt/src/webrtc.rs | 133 -- 30 files changed, 5548 insertions(+), 3038 deletions(-) delete mode 100644 lkgpt/src/assets/gltf.rs delete mode 100644 lkgpt/src/assets/materials.rs delete mode 100644 lkgpt/src/assets/mesh.rs delete mode 100644 lkgpt/src/assets/mod.rs delete mode 100644 lkgpt/src/assets/model.rs delete mode 100644 lkgpt/src/assets/texture.rs delete mode 100644 lkgpt/src/core.rs delete mode 100644 lkgpt/src/gpt.rs delete mode 100644 lkgpt/src/response.rs delete mode 100644 lkgpt/src/room_events.rs delete mode 100644 lkgpt/src/routes/health_check.rs delete mode 100644 lkgpt/src/routes/lsdk_webhook.rs delete mode 100644 lkgpt/src/routes/mod.rs delete mode 100644 lkgpt/src/scene/camera.rs delete mode 100644 lkgpt/src/scene/material_manager.rs delete mode 100644 lkgpt/src/scene/mod.rs delete mode 100644 lkgpt/src/scene/node.rs delete mode 100644 lkgpt/src/scene/scene.rs delete mode 100644 lkgpt/src/scene/texture_manager.rs delete mode 100644 lkgpt/src/state.rs delete mode 100644 lkgpt/src/stt.rs delete mode 100644 lkgpt/src/track_pub.rs delete mode 100644 lkgpt/src/tts.rs delete mode 100644 lkgpt/src/turbo.rs delete mode 100644 lkgpt/src/utils/create_bot_token.rs delete mode 100644 lkgpt/src/utils/mod.rs delete mode 100644 lkgpt/src/webrtc.rs diff --git a/Cargo.lock b/Cargo.lock index 9a698ff..692c1d6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,75 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "ab_glyph" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80179d7dd5d7e8c285d67c4a1e652972a92de7475beddfb92028c76463b13225" +dependencies = [ + "ab_glyph_rasterizer", + "owned_ttf_parser", +] + +[[package]] +name = "ab_glyph_rasterizer" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c71b1793ee61086797f5c80b6efa2b8ffa6d5dd703f118545808a7f2e27f7046" + +[[package]] +name = "accesskit" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca8410747ed85a17c4a1e9ed3f5a74d3e7bdcc876cf9a18ff40ae21d645997b2" + +[[package]] +name = "accesskit_consumer" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c17cca53c09fbd7288667b22a201274b9becaa27f0b91bf52a526db95de45e6" +dependencies = [ + "accesskit", +] + +[[package]] +name = "accesskit_macos" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd3b6ae1eabbfbced10e840fd3fce8a93ae84f174b3e4ba892ab7bcb42e477a7" +dependencies = [ + "accesskit", + "accesskit_consumer", + "objc2", + "once_cell", +] + +[[package]] +name = "accesskit_windows" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afcae27ec0974fc7c3b0b318783be89fd1b2e66dd702179fe600166a38ff4a0b" +dependencies = [ + "accesskit", + "accesskit_consumer", + "once_cell", + "paste", + "static_assertions", + "windows 0.48.0", +] + +[[package]] +name = "accesskit_winit" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88e39fcec2e10971e188730b7a76bab60647dacc973d4591855ebebcadfaa738" +dependencies = [ + "accesskit", + "accesskit_macos", + "accesskit_windows", + "winit", +] + [[package]] name = "actix-codec" version = "0.5.1" @@ -244,12 +313,70 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "allocator-api2" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" + +[[package]] +name = "alsa" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2562ad8dcf0f789f65c6fdaad8a8a9708ed6b488e649da28c01656ad66b8b47" +dependencies = [ + "alsa-sys", + "bitflags 1.3.2", + "libc", + "nix 0.24.3", +] + +[[package]] +name = "alsa-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db8fee663d06c4e303404ef5f40488a53e062f89ba8bfed81f42325aafad1527" +dependencies = [ + "libc", + "pkg-config", +] + +[[package]] +name = "android-activity" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64529721f27c2314ced0890ce45e469574a73e5e6fdd6e9da1860eb29285f5e0" +dependencies = [ + "android-properties", + "bitflags 1.3.2", + "cc", + "jni-sys", + "libc", + "log", + "ndk", + "ndk-context", + "ndk-sys", + "num_enum 0.6.1", +] + +[[package]] +name = "android-properties" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7eb209b1518d6bb87b283c20095f5228ecda460da70b44f0802523dea6da04" + [[package]] name = "android-tzdata" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" +[[package]] +name = "android_log-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ecc8056bf6ab9892dcd53216c83d1597487d7dacac16c8df6b877d127df9937" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -265,6 +392,27 @@ version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" +[[package]] +name = "approx" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6" +dependencies = [ + "num-traits", +] + +[[package]] +name = "arrayref" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" + +[[package]] +name = "arrayvec" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" + [[package]] name = "ash" version = "0.37.3+1.3.251" @@ -274,6 +422,16 @@ dependencies = [ "libloading", ] +[[package]] +name = "async-broadcast" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c48ccdbf6ca6b121e0f586cbc0e73ae440e56c67c30fa0873b4e110d9c26d2b" +dependencies = [ + "event-listener 2.5.3", + "futures-core", +] + [[package]] name = "async-channel" version = "1.9.0" @@ -293,7 +451,7 @@ checksum = "d37875bd9915b7d67c2f117ea2c30a0989874d0b2cb694fe25403c85763c0c9e" dependencies = [ "concurrent-queue", "event-listener 3.1.0", - "event-listener-strategy", + "event-listener-strategy 0.3.0", "futures-core", "pin-project-lite", ] @@ -307,6 +465,52 @@ dependencies = [ "async-trait", ] +[[package]] +name = "async-executor" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17ae5ebefcc48e7452b4987947920dac9450be1110cadf34d1b8c116bdbaf97c" +dependencies = [ + "async-lock 3.1.2", + "async-task", + "concurrent-queue", + "fastrand 2.0.1", + "futures-lite 2.0.1", + "slab", +] + +[[package]] +name = "async-fs" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "279cf904654eeebfa37ac9bb1598880884924aab82e290aa65c9e77a0e142e06" +dependencies = [ + "async-lock 2.8.0", + "autocfg", + "blocking", + "futures-lite 1.13.0", +] + +[[package]] +name = "async-lock" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "287272293e9d8c41773cec55e365490fe034813a2f172f502d6ddcf75b2f582b" +dependencies = [ + "event-listener 2.5.3", +] + +[[package]] +name = "async-lock" +version = "3.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dea8b3453dd7cc96711834b75400d671b73e3656975fa68d9f277163b7f7e316" +dependencies = [ + "event-listener 4.0.0", + "event-listener-strategy 0.4.0", + "pin-project-lite", +] + [[package]] name = "async-openai" version = "0.16.3" @@ -332,6 +536,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "async-task" +version = "4.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4eb2cdb97421e01129ccb49169d8279ed21e829929144f4a22a6e54ac549ca1" + [[package]] name = "async-trait" version = "0.1.74" @@ -343,6 +553,12 @@ dependencies = [ "syn 2.0.39", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "atomic_enum" version = "0.2.0" @@ -364,48 +580,822 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" name = "backoff" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" +checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" +dependencies = [ + "futures-core", + "getrandom", + "instant", + "pin-project-lite", + "rand", + "tokio", +] + +[[package]] +name = "backtrace" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + +[[package]] +name = "base64" +version = "0.21.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" + +[[package]] +name = "base64ct" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" + +[[package]] +name = "bevy" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "329e344f835f5a9a4c46a6d1d57371f726aa2c482d1bd669b2b9c4eb1ee91fd7" +dependencies = [ + "bevy_internal", +] + +[[package]] +name = "bevy_a11y" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271b812e5734f5056a400f7d64592dd82d6c0e6179389c2f066f433ab8bc7692" +dependencies = [ + "accesskit", + "bevy_app", + "bevy_derive", + "bevy_ecs", +] + +[[package]] +name = "bevy_animation" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab94187a1253433e14f175293d8a86ec1c2822fda2a17807908f11ec21f45f00" +dependencies = [ + "bevy_app", + "bevy_asset", + "bevy_core", + "bevy_ecs", + "bevy_hierarchy", + "bevy_math", + "bevy_reflect", + "bevy_render", + "bevy_time", + "bevy_transform", + "bevy_utils", +] + +[[package]] +name = "bevy_app" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "172d532ea812e5954fa814dae003c207f2a0b20c6e50431787c94a7159677ece" +dependencies = [ + "bevy_derive", + "bevy_ecs", + "bevy_reflect", + "bevy_tasks", + "bevy_utils", + "downcast-rs", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "bevy_asset" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccb2b67984088b23e223cfe9ec1befd89a110665a679acb06839bc4334ed37d6" +dependencies = [ + "async-broadcast", + "async-fs", + "async-lock 2.8.0", + "bevy_app", + "bevy_asset_macros", + "bevy_ecs", + "bevy_log", + "bevy_reflect", + "bevy_tasks", + "bevy_utils", + "bevy_winit", + "blake3", + "crossbeam-channel", + "downcast-rs", + "futures-io", + "futures-lite 1.13.0", + "js-sys", + "parking_lot", + "ron", + "serde", + "thiserror", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "bevy_asset_macros" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b3245193e90fc8abcf1059a467cb224501dcda083d114c67c10ac66b7171e3a" +dependencies = [ + "bevy_macro_utils", + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "bevy_audio" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478de80ff25cb7decbcb22797774d1597e8c32914e81431c67d64faadc08f84a" +dependencies = [ + "bevy_app", + "bevy_asset", + "bevy_derive", + "bevy_ecs", + "bevy_math", + "bevy_reflect", + "bevy_transform", + "bevy_utils", + "oboe", + "rodio", +] + +[[package]] +name = "bevy_core" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "025e6800b73048092a55c3611e9327ad4c4c17b60517ec1c0086bb40b4b19ea8" +dependencies = [ + "bevy_app", + "bevy_ecs", + "bevy_math", + "bevy_reflect", + "bevy_tasks", + "bevy_utils", + "bytemuck", +] + +[[package]] +name = "bevy_core_pipeline" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e4b08a2d53ba62d9ec1fca3f7f4e0f556e9f59e1c8e63a4b7c2a18c0701152c" +dependencies = [ + "bevy_app", + "bevy_asset", + "bevy_core", + "bevy_derive", + "bevy_ecs", + "bevy_log", + "bevy_math", + "bevy_reflect", + "bevy_render", + "bevy_transform", + "bevy_utils", + "bitflags 2.4.1", + "radsort", + "serde", +] + +[[package]] +name = "bevy_derive" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24bf40259be12a1a24d9fd536f5ff18d31eeb5665b77e2732899783be6edc5d6" +dependencies = [ + "bevy_macro_utils", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "bevy_diagnostic" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41b5a99a9fb6cd7d1eb1714fad193944a0317f0887a15cccb8309c8d37951132" +dependencies = [ + "bevy_app", + "bevy_core", + "bevy_ecs", + "bevy_log", + "bevy_time", + "bevy_utils", + "sysinfo", +] + +[[package]] +name = "bevy_ecs" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae11a1f467c372b50e9d4b55e78370f5420c9db7416200cc441cc84f08174dd3" +dependencies = [ + "async-channel 1.9.0", + "bevy_ecs_macros", + "bevy_ptr", + "bevy_reflect", + "bevy_tasks", + "bevy_utils", + "downcast-rs", + "event-listener 2.5.3", + "fixedbitset", + "rustc-hash", + "serde", + "thiserror", + "thread_local", +] + +[[package]] +name = "bevy_ecs_macros" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f642c2b67c4d0daf8edf15074f6351457eb487a34b3de1290c760d8f3ac9ec16" +dependencies = [ + "bevy_macro_utils", + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "bevy_encase_derive" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65b9fb5a62c4e3ab70caaa839470d35fa932001b1b34b08bc7f7f1909bd2b3a7" +dependencies = [ + "bevy_macro_utils", + "encase_derive_impl", +] + +[[package]] +name = "bevy_gilrs" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad31cc2c84315e0759d793d6c5bcb7d8789bbc16359c98d1b766e708c1bbae49" +dependencies = [ + "bevy_app", + "bevy_ecs", + "bevy_input", + "bevy_log", + "bevy_time", + "bevy_utils", + "gilrs", + "thiserror", +] + +[[package]] +name = "bevy_gizmos" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87d1cc978b91f416b23eb16f00e69f95c3a04582021827d8082e92d4725cc510" +dependencies = [ + "bevy_app", + "bevy_asset", + "bevy_core", + "bevy_core_pipeline", + "bevy_ecs", + "bevy_math", + "bevy_pbr", + "bevy_reflect", + "bevy_render", + "bevy_sprite", + "bevy_transform", + "bevy_utils", +] + +[[package]] +name = "bevy_gltf" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f933745c0c86e2c07948def581259b466f99708328657054e956275430ccfd7" +dependencies = [ + "base64 0.13.1", + "bevy_animation", + "bevy_app", + "bevy_asset", + "bevy_core", + "bevy_core_pipeline", + "bevy_ecs", + "bevy_hierarchy", + "bevy_log", + "bevy_math", + "bevy_pbr", + "bevy_reflect", + "bevy_render", + "bevy_scene", + "bevy_tasks", + "bevy_transform", + "bevy_utils", + "gltf", + "percent-encoding", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "bevy_hierarchy" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64fa240011fce8ee23f9b46e5a26a628a31d7860d6d2e4e0e361bb3ea6d5a703" +dependencies = [ + "bevy_app", + "bevy_core", + "bevy_ecs", + "bevy_log", + "bevy_reflect", + "bevy_utils", + "smallvec", +] + +[[package]] +name = "bevy_input" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e86e241b3a10b79f65a69205552546723b855d3d4c1bd8261637c076144d32f" +dependencies = [ + "bevy_app", + "bevy_ecs", + "bevy_math", + "bevy_reflect", + "bevy_utils", + "thiserror", +] + +[[package]] +name = "bevy_internal" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55124e486814c4d3632d5cfad9c4f4e46d052c028593ec46fef5bfbfb0f840b1" +dependencies = [ + "bevy_a11y", + "bevy_animation", + "bevy_app", + "bevy_asset", + "bevy_audio", + "bevy_core", + "bevy_core_pipeline", + "bevy_derive", + "bevy_diagnostic", + "bevy_ecs", + "bevy_gilrs", + "bevy_gizmos", + "bevy_gltf", + "bevy_hierarchy", + "bevy_input", + "bevy_log", + "bevy_math", + "bevy_pbr", + "bevy_ptr", + "bevy_reflect", + "bevy_render", + "bevy_scene", + "bevy_sprite", + "bevy_tasks", + "bevy_text", + "bevy_time", + "bevy_transform", + "bevy_ui", + "bevy_utils", + "bevy_window", + "bevy_winit", +] + +[[package]] +name = "bevy_log" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "011417debf7868b45932bb97fc0d5bfdeaf9304e324aa94840e2f1e6deeed69d" +dependencies = [ + "android_log-sys", + "bevy_app", + "bevy_ecs", + "bevy_utils", + "console_error_panic_hook", + "tracing-log 0.1.4", + "tracing-subscriber", + "tracing-wasm", +] + +[[package]] +name = "bevy_macro_utils" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf6fba87c6d069fcbcd8a48625ca8ab4392ad40d2b260863ce7d641a0f42986d" +dependencies = [ + "proc-macro2", + "quote", + "rustc-hash", + "syn 2.0.39", + "toml_edit 0.20.7", +] + +[[package]] +name = "bevy_math" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "752764558a1f429c20704c3b836a019fa308961c43fdfef4f08e339d456c96be" +dependencies = [ + "glam", + "serde", +] + +[[package]] +name = "bevy_mikktspace" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b596c41a56f2268ec7cde560edc588bc7b5886e4b49c8b27c4dcc9f7c743424c" +dependencies = [ + "glam", +] + +[[package]] +name = "bevy_pbr" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeb6a35a78d355cc21c10f277dcd171eca65e30a90e76eb89f4dacf606621fe1" +dependencies = [ + "bevy_app", + "bevy_asset", + "bevy_core_pipeline", + "bevy_derive", + "bevy_ecs", + "bevy_math", + "bevy_reflect", + "bevy_render", + "bevy_transform", + "bevy_utils", + "bevy_window", + "bitflags 2.4.1", + "bytemuck", + "fixedbitset", + "naga_oil", + "radsort", + "smallvec", + "thread_local", +] + +[[package]] +name = "bevy_ptr" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "308a02679f6ce21ef71de20fae6d6a2016c07baa21d8e8d0558e6b7851e8adf2" + +[[package]] +name = "bevy_reflect" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdd56914a8ad57621d7a1a099f7e6b1f7482c9c76cedc9c3d4c175a203939c5d" +dependencies = [ + "bevy_math", + "bevy_ptr", + "bevy_reflect_derive", + "bevy_utils", + "downcast-rs", + "erased-serde", + "glam", + "serde", + "smallvec", + "smol_str", + "thiserror", +] + +[[package]] +name = "bevy_reflect_derive" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25f627907c40ac552f798423447fc331fc1ddacd94c5f7a2a70942eb06bc8447" +dependencies = [ + "bevy_macro_utils", + "proc-macro2", + "quote", + "syn 2.0.39", + "uuid", +] + +[[package]] +name = "bevy_render" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90d777f4c51bd58e9e40777c6cb8dde0778df7e2c5298b3f9e3455bd12a9856c" +dependencies = [ + "async-channel 1.9.0", + "bevy_app", + "bevy_asset", + "bevy_core", + "bevy_derive", + "bevy_ecs", + "bevy_encase_derive", + "bevy_hierarchy", + "bevy_log", + "bevy_math", + "bevy_mikktspace", + "bevy_reflect", + "bevy_render_macros", + "bevy_tasks", + "bevy_time", + "bevy_transform", + "bevy_utils", + "bevy_window", + "bitflags 2.4.1", + "bytemuck", + "codespan-reporting", + "downcast-rs", + "encase", + "futures-lite 1.13.0", + "hexasphere", + "image", + "js-sys", + "ktx2", + "naga", + "naga_oil", + "ruzstd", + "serde", + "smallvec", + "thiserror", + "thread_local", + "wasm-bindgen", + "web-sys", + "wgpu", +] + +[[package]] +name = "bevy_render_macros" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35b00c3d0abff94a729460fc9aa95c2ceac71b49b3041166bb5ba3098e9657e7" +dependencies = [ + "bevy_macro_utils", + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "bevy_scene" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba6294396a6375f0b14341d8003408c10aa040e3f833ac8bd49677170ec55d73" +dependencies = [ + "bevy_app", + "bevy_asset", + "bevy_derive", + "bevy_ecs", + "bevy_hierarchy", + "bevy_reflect", + "bevy_render", + "bevy_transform", + "bevy_utils", + "ron", + "serde", + "thiserror", + "uuid", +] + +[[package]] +name = "bevy_sprite" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4f7d1f88a6e5497fdafd95c20984a1d1b5517bc39d51600b4988cd60c51837a" +dependencies = [ + "bevy_app", + "bevy_asset", + "bevy_core_pipeline", + "bevy_derive", + "bevy_ecs", + "bevy_log", + "bevy_math", + "bevy_reflect", + "bevy_render", + "bevy_transform", + "bevy_utils", + "bitflags 2.4.1", + "bytemuck", + "fixedbitset", + "guillotiere", + "radsort", + "rectangle-pack", + "thiserror", +] + +[[package]] +name = "bevy_tasks" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a45be906618192515bc613e46546150089adbb4a82178dc462045acd1e89e92" +dependencies = [ + "async-channel 1.9.0", + "async-executor", + "async-task", + "concurrent-queue", + "futures-lite 1.13.0", + "wasm-bindgen-futures", +] + +[[package]] +name = "bevy_text" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c136af700af4f87c94f68d6e019528c371bf09ebf4a8ff7468bb3c73806b34f5" +dependencies = [ + "ab_glyph", + "bevy_app", + "bevy_asset", + "bevy_ecs", + "bevy_math", + "bevy_reflect", + "bevy_render", + "bevy_sprite", + "bevy_transform", + "bevy_utils", + "bevy_window", + "glyph_brush_layout", + "serde", + "thiserror", +] + +[[package]] +name = "bevy_time" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b29709cadf22d318a0b7c79f763e9c5ac414292bd0e850066fa935959021b276" +dependencies = [ + "bevy_app", + "bevy_ecs", + "bevy_reflect", + "bevy_utils", + "crossbeam-channel", + "thiserror", +] + +[[package]] +name = "bevy_transform" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70262c51e915b6224129206d23823364e650cf5eb5f4b6ce3ee379f608c180d2" +dependencies = [ + "bevy_app", + "bevy_ecs", + "bevy_hierarchy", + "bevy_math", + "bevy_reflect", + "thiserror", +] + +[[package]] +name = "bevy_ui" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd5ecbf2dceaab118769dd870e34d780bfde556af561fd10d8d613b0f237297e" +dependencies = [ + "bevy_a11y", + "bevy_app", + "bevy_asset", + "bevy_core_pipeline", + "bevy_derive", + "bevy_ecs", + "bevy_hierarchy", + "bevy_input", + "bevy_log", + "bevy_math", + "bevy_reflect", + "bevy_render", + "bevy_sprite", + "bevy_text", + "bevy_transform", + "bevy_utils", + "bevy_window", + "bytemuck", + "serde", + "smallvec", + "taffy", + "thiserror", +] + +[[package]] +name = "bevy_utils" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8e75d4a34ef0b15dffd1ee9079ef1f0f5139527e192b9d5708b3e158777c753" +dependencies = [ + "ahash", + "bevy_utils_proc_macros", + "getrandom", + "hashbrown 0.14.2", + "instant", + "nonmax", + "petgraph", + "thiserror", + "tracing", + "uuid", +] + +[[package]] +name = "bevy_utils_proc_macros" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7dfd3735a61a1b681ed1e176afe4eae731bbb03e51ad871e9eb39e76a2d170e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "bevy_window" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e60d1830b3fbd7db5bfea7ac9fcd0f5e1d1af88c91ab469e697ab176d8b3140b" dependencies = [ - "futures-core", - "getrandom", - "instant", - "pin-project-lite", - "rand", - "tokio", + "bevy_a11y", + "bevy_app", + "bevy_ecs", + "bevy_input", + "bevy_math", + "bevy_reflect", + "bevy_utils", + "raw-window-handle", ] [[package]] -name = "backtrace" -version = "0.3.69" +name = "bevy_winit" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" -dependencies = [ - "addr2line", - "cc", - "cfg-if", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", +checksum = "7f8294e78c6a1f9c34d36501a377c5d20bf0fa23a0958187bb270187741448ba" +dependencies = [ + "accesskit_winit", + "approx", + "bevy_a11y", + "bevy_app", + "bevy_derive", + "bevy_ecs", + "bevy_hierarchy", + "bevy_input", + "bevy_math", + "bevy_tasks", + "bevy_utils", + "bevy_window", + "crossbeam-channel", + "raw-window-handle", + "wasm-bindgen", + "web-sys", + "winit", ] [[package]] -name = "base64" -version = "0.13.1" +name = "bindgen" +version = "0.69.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +checksum = "9ffcebc3849946a7170a05992aac39da343a90676ab392c51a4280981d6379c2" +dependencies = [ + "bitflags 2.4.1", + "cexpr", + "clang-sys", + "lazy_static", + "lazycell", + "peeking_take_while", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn 2.0.39", +] [[package]] -name = "base64" -version = "0.21.5" +name = "bit-set" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" +checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +dependencies = [ + "bit-vec", +] [[package]] -name = "base64ct" -version = "1.6.0" +name = "bit-vec" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" [[package]] name = "bit_field" @@ -424,6 +1414,28 @@ name = "bitflags" version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +dependencies = [ + "serde", +] + +[[package]] +name = "blake3" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0231f06152bf547e9c2b5194f247cd97aacf6dcd8b15d8e5ec0663f64580da87" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq 0.3.0", +] + +[[package]] +name = "block" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d8c1fef690941d3e7788d328517591fecc684c084084702d6ff1641e993699a" [[package]] name = "block-buffer" @@ -434,6 +1446,41 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-sys" +version = "0.1.0-beta.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa55741ee90902547802152aaf3f8e5248aab7e21468089560d4c8840561146" +dependencies = [ + "objc-sys", +] + +[[package]] +name = "block2" +version = "0.2.0-alpha.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8dd9e63c1744f755c2f60332b88de39d341e5e86239014ad839bd71c106dec42" +dependencies = [ + "block-sys", + "objc2-encode", +] + +[[package]] +name = "blocking" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a37913e8dc4ddcc604f0c6d3bf2887c995153af3611de9e23c352b44c1b9118" +dependencies = [ + "async-channel 2.1.0", + "async-lock 3.1.2", + "async-task", + "fastrand 2.0.1", + "futures-io", + "futures-lite 2.0.1", + "piper", + "tracing", +] + [[package]] name = "brotli" version = "3.4.0" @@ -539,12 +1586,27 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg_aliases" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" + [[package]] name = "chrono" version = "0.4.31" @@ -569,6 +1631,17 @@ dependencies = [ "inout", ] +[[package]] +name = "clang-sys" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c688fc74432808e3eb684cae8830a86be1d66a2bd58e1f248ed0960a590baf6f" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "cmake" version = "0.1.50" @@ -594,6 +1667,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" +[[package]] +name = "com-rs" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf43edc576402991846b093a7ca18a3477e0ef9c588cde84964b5d3e43016642" + [[package]] name = "combine" version = "4.6.6" @@ -613,12 +1692,49 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "console_error_panic_hook" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc" +dependencies = [ + "cfg-if", + "wasm-bindgen", +] + +[[package]] +name = "const_panic" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6051f239ecec86fde3410901ab7860d458d160371533842974fc61f96d15879b" + +[[package]] +name = "const_soft_float" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ca1caa64ef4ed453e68bb3db612e51cf1b2f5b871337f0fcab1c8f87cc3dff" + [[package]] name = "constant_time_eq" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" +[[package]] +name = "constant_time_eq" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" + +[[package]] +name = "constgebra" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd23e864550e6dafc1e41ac78ce4f1ccddc8672b40c403524a04ff3f0518420" +dependencies = [ + "const_soft_float", +] + [[package]] name = "convert_case" version = "0.4.0" @@ -652,6 +1768,19 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +[[package]] +name = "core-graphics" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2581bbab3b8ffc6fcbd550bf46c355135d16e9ff2a6ea032ad6b9bf1d7efe4fb" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-graphics-types", + "foreign-types 0.3.2", + "libc", +] + [[package]] name = "core-graphics-types" version = "0.1.2" @@ -663,6 +1792,51 @@ dependencies = [ "libc", ] +[[package]] +name = "coreaudio-rs" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "321077172d79c662f64f5071a03120748d5bb652f5231570141be24cfcd2bace" +dependencies = [ + "bitflags 1.3.2", + "core-foundation-sys", + "coreaudio-sys", +] + +[[package]] +name = "coreaudio-sys" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3120ebb80a9de008e638ad833d4127d50ea3d3a960ea23ea69bc66d9358a028" +dependencies = [ + "bindgen", +] + +[[package]] +name = "cpal" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d959d90e938c5493000514b446987c07aed46c668faaa7d34d6c7a67b1a578c" +dependencies = [ + "alsa", + "core-foundation-sys", + "coreaudio-rs", + "dasp_sample", + "jni 0.19.0", + "js-sys", + "libc", + "mach2", + "ndk", + "ndk-context", + "oboe", + "once_cell", + "parking_lot", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "windows 0.46.0", +] + [[package]] name = "cpufeatures" version = "0.2.11" @@ -808,6 +1982,17 @@ dependencies = [ "syn 2.0.39", ] +[[package]] +name = "d3d12" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e16e44ab292b1dddfdaf7be62cfd8877df52f2f3fde5858d95bab606be259f20" +dependencies = [ + "bitflags 2.4.1", + "libloading", + "winapi", +] + [[package]] name = "darling" version = "0.14.4" @@ -843,6 +2028,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "dasp_sample" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f" + [[package]] name = "data-encoding" version = "2.5.0" @@ -936,18 +2127,62 @@ dependencies = [ "subtle", ] +[[package]] +name = "dispatch" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd0c93bb4b0c6d9b77f4435b0ae98c24d17f1c45b2ff844c6151a07256ca923b" + [[package]] name = "dotenvy" version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" +[[package]] +name = "downcast-rs" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea835d29036a4087793836fa931b08837ad5e957da9e23886b29586fb9b6650" + [[package]] name = "either" version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +[[package]] +name = "encase" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fce2eeef77fd4a293a54b62aa00ac9daebfbcda4bf8998c5a815635b004aa1c" +dependencies = [ + "const_panic", + "encase_derive", + "glam", + "thiserror", +] + +[[package]] +name = "encase_derive" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e520cde08cbf4f7cc097f61573ec06ce467019803de8ae82fb2823fa1554a0e" +dependencies = [ + "encase_derive_impl", +] + +[[package]] +name = "encase_derive_impl" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fe2568f851fd6144a45fa91cfed8fe5ca8fc0b56ba6797bfc1ed2771b90e37c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + [[package]] name = "encoding_rs" version = "0.8.33" @@ -990,6 +2225,15 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +[[package]] +name = "erased-serde" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c138974f9d5e7fe373eb04df7cae98833802ae4b11c24ac7039a21d5af4b26c" +dependencies = [ + "serde", +] + [[package]] name = "errno" version = "0.3.7" @@ -1000,6 +2244,15 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "euclid" +version = "0.22.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f253bc5c813ca05792837a0ff4b3a580336b224512d48f7eda1d7dd9210787" +dependencies = [ + "num-traits", +] + [[package]] name = "event-listener" version = "2.5.3" @@ -1017,6 +2270,17 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "event-listener" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "770d968249b5d99410d61f5bf89057f3199a077a04d087092f58e7d10692baae" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + [[package]] name = "event-listener-strategy" version = "0.3.0" @@ -1027,6 +2291,16 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "event-listener-strategy" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "958e4d70b6d5e81971bebec42271ec641e7ff4e170a6fa605f2b8a8b65cb97d3" +dependencies = [ + "event-listener 4.0.0", + "pin-project-lite", +] + [[package]] name = "eventsource-stream" version = "0.2.3" @@ -1080,6 +2354,15 @@ dependencies = [ "wasmtimer", ] +[[package]] +name = "fastrand" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" +dependencies = [ + "instant", +] + [[package]] name = "fastrand" version = "2.0.1" @@ -1144,12 +2427,33 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] -name = "foreign-types" -version = "0.3.2" +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared 0.1.1", +] + +[[package]] +name = "foreign-types" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" +dependencies = [ + "foreign-types-macros", + "foreign-types-shared 0.3.1", +] + +[[package]] +name = "foreign-types-macros" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" dependencies = [ - "foreign-types-shared", + "proc-macro2", + "quote", + "syn 2.0.39", ] [[package]] @@ -1158,6 +2462,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" +[[package]] +name = "foreign-types-shared" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -1225,6 +2535,35 @@ version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" +[[package]] +name = "futures-lite" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" +dependencies = [ + "fastrand 1.9.0", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite", + "waker-fn", +] + +[[package]] +name = "futures-lite" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3831c2651acb5177cbd83943f3d9c8912c5ad03c76afcc0e9511ba568ec5ebb" +dependencies = [ + "fastrand 2.0.1", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite", +] + [[package]] name = "futures-macro" version = "0.3.29" @@ -1305,6 +2644,40 @@ dependencies = [ "weezl", ] +[[package]] +name = "gilrs" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e9eec02069fcbd7abe00a28adf216547774889129a777cb5e53fdfb75d59f09" +dependencies = [ + "fnv", + "gilrs-core", + "log", + "uuid", + "vec_map", +] + +[[package]] +name = "gilrs-core" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "178769da179a47b187837d1ab2b5b9b684a21180166a77a4ca37e7e58ee3833d" +dependencies = [ + "core-foundation", + "inotify", + "io-kit-sys", + "js-sys", + "libc", + "libudev-sys", + "log", + "nix 0.27.1", + "uuid", + "vec_map", + "wasm-bindgen", + "web-sys", + "windows 0.51.1", +] + [[package]] name = "gimli" version = "0.28.0" @@ -1316,6 +2689,10 @@ name = "glam" version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5418c17512bdf42730f9032c74e1ae39afc408745ebb2acf72fbc4691c17945" +dependencies = [ + "bytemuck", + "serde", +] [[package]] name = "glob" @@ -1335,6 +2712,18 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "glow" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca0fe580e4b60a8ab24a868bc08e2f03cbcb20d3d676601fa909386713333728" +dependencies = [ + "js-sys", + "slotmap", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "gltf" version = "1.3.0" @@ -1373,6 +2762,85 @@ dependencies = [ "serde_json", ] +[[package]] +name = "glyph_brush_layout" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc32c2334f00ca5ac3695c5009ae35da21da8c62d255b5b96d56e2597a637a38" +dependencies = [ + "ab_glyph", + "approx", + "xi-unicode", +] + +[[package]] +name = "gpu-alloc" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbcd2dba93594b227a1f57ee09b8b9da8892c34d55aa332e034a228d0fe6a171" +dependencies = [ + "bitflags 2.4.1", + "gpu-alloc-types", +] + +[[package]] +name = "gpu-alloc-types" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98ff03b468aa837d70984d55f5d3f846f6ec31fe34bbb97c4f85219caeee1ca4" +dependencies = [ + "bitflags 2.4.1", +] + +[[package]] +name = "gpu-allocator" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce95f9e2e11c2c6fadfce42b5af60005db06576f231f5c92550fdded43c423e8" +dependencies = [ + "backtrace", + "log", + "thiserror", + "winapi", + "windows 0.44.0", +] + +[[package]] +name = "gpu-descriptor" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc11df1ace8e7e564511f53af41f3e42ddc95b56fd07b3f4445d2a6048bc682c" +dependencies = [ + "bitflags 2.4.1", + "gpu-descriptor-types", + "hashbrown 0.14.2", +] + +[[package]] +name = "gpu-descriptor-types" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bf0b36e6f090b7e1d8a4b49c0cb81c1f8376f72198c65dd3ad9ff3556b8b78c" +dependencies = [ + "bitflags 2.4.1", +] + +[[package]] +name = "grid" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eec1c01eb1de97451ee0d60de7d81cf1e72aabefb021616027f3d1c3ec1c723c" + +[[package]] +name = "guillotiere" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b62d5865c036cb1393e23c50693df631d3f5d7bcca4c04fe4cc0fd592e74a782" +dependencies = [ + "euclid", + "svg_fmt", +] + [[package]] name = "h2" version = "0.3.22" @@ -1414,6 +2882,26 @@ name = "hashbrown" version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" +dependencies = [ + "ahash", + "allocator-api2", + "serde", +] + +[[package]] +name = "hassle-rs" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1397650ee315e8891a0df210707f0fc61771b0cc518c3023896064c5407cb3b0" +dependencies = [ + "bitflags 1.3.2", + "com-rs", + "libc", + "libloading", + "thiserror", + "widestring", + "winapi", +] [[package]] name = "heck" @@ -1427,6 +2915,22 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" +[[package]] +name = "hexasphere" +version = "9.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cb3df16a7bcb1b5bc092abd55e14f77ca70aea14445026e264586fc62889a10" +dependencies = [ + "constgebra", + "glam", +] + +[[package]] +name = "hexf-parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfa686283ad6dd069f105e5ab091b04c62850d3e4cf5d67debad1933f55023df" + [[package]] name = "hmac" version = "0.12.1" @@ -1620,6 +3124,26 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a257582fdcde896fd96463bf2d40eefea0580021c0712a0e2b028b60b47a837a" +[[package]] +name = "inotify" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdd168d97690d0b8c412d6b6c10360277f4d7ee495c5d0d5d5fe0854923255cc" +dependencies = [ + "bitflags 1.3.2", + "inotify-sys", + "libc", +] + +[[package]] +name = "inotify-sys" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e05c02b5e89bff3b946cedeca278abc628fe811e604f027c45a8aa3cf793d0eb" +dependencies = [ + "libc", +] + [[package]] name = "inout" version = "0.1.3" @@ -1636,6 +3160,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "io-kit-sys" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4769cb30e5dcf1710fc6730d3e94f78c47723a014a567de385e113c737394640" +dependencies = [ + "core-foundation-sys", + "mach2", ] [[package]] @@ -1670,6 +3207,34 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +[[package]] +name = "jni" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6df18c2e3db7e453d3c6ac5b3e9d5182664d28788126d39b91f2d1e22b017ec" +dependencies = [ + "cesu8", + "combine", + "jni-sys", + "log", + "thiserror", + "walkdir", +] + +[[package]] +name = "jni" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "039022cdf4d7b1cf548d31f60ae783138e5fd42013f6271049d7df7afadef96c" +dependencies = [ + "cesu8", + "combine", + "jni-sys", + "log", + "thiserror", + "walkdir", +] + [[package]] name = "jni" version = "0.21.1" @@ -1731,6 +3296,26 @@ dependencies = [ "serde_json", ] +[[package]] +name = "khronos-egl" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c2352bd1d0bceb871cb9d40f24360c8133c11d7486b68b5381c1dd1a32015e3" +dependencies = [ + "libc", + "libloading", + "pkg-config", +] + +[[package]] +name = "ktx2" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87d65e08a9ec02e409d27a0139eaa6b9756b4d81fe7cde71f6941a83730ce838" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "language-tags" version = "0.3.2" @@ -1743,12 +3328,29 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "lebe" version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8" +[[package]] +name = "lewton" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "777b48df9aaab155475a83a7df3070395ea1ac6902f5cd062b8f2b028075c030" +dependencies = [ + "byteorder", + "ogg", + "tinyvec", +] + [[package]] name = "libc" version = "0.2.150" @@ -1765,13 +3367,34 @@ dependencies = [ "winapi", ] +[[package]] +name = "libredox" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3af92c55d7d839293953fcd0fda5ecfe93297cfde6ffbdec13b41d99c0ba6607" +dependencies = [ + "bitflags 2.4.1", + "libc", + "redox_syscall 0.4.1", +] + +[[package]] +name = "libudev-sys" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c8469b4a23b962c1396b9b451dda50ef5b283e8dd309d69033475fa9b334324" +dependencies = [ + "libc", + "pkg-config", +] + [[package]] name = "libwebrtc" version = "0.2.0" source = "git+https://github.com/livekit/rust-sdks?rev=36d7690#36d769006525b60f772a25cc5fd9aa657c7b0938" dependencies = [ "cxx", - "jni", + "jni 0.21.1", "js-sys", "lazy_static", "livekit-protocol", @@ -1871,6 +3494,7 @@ dependencies = [ "async-openai", "async-trait", "base64 0.21.5", + "bevy", "bytes", "chrono", "crossbeam", @@ -1932,6 +3556,15 @@ version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +[[package]] +name = "mach2" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d0d1830bcd151a6fc4aea1369af235b36c1528fe976b8ff678683c9995eade8" +dependencies = [ + "libc", +] + [[package]] name = "malloc_buf" version = "0.0.6" @@ -1941,6 +3574,15 @@ dependencies = [ "libc", ] +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata 0.1.10", +] + [[package]] name = "memchr" version = "2.6.4" @@ -1956,6 +3598,21 @@ dependencies = [ "autocfg", ] +[[package]] +name = "metal" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "623b5e6cefd76e58f774bd3cc0c6f5c7615c58c03a97815245a25c3c9bdee318" +dependencies = [ + "bitflags 2.4.1", + "block", + "core-graphics-types", + "foreign-types 0.5.0", + "log", + "objc", + "paste", +] + [[package]] name = "mime" version = "0.3.17" @@ -2006,6 +3663,47 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" +[[package]] +name = "naga" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1ceaaa4eedaece7e4ec08c55c640ba03dbb73fb812a6570a59bcf1930d0f70e" +dependencies = [ + "bit-set", + "bitflags 2.4.1", + "codespan-reporting", + "hexf-parse", + "indexmap 1.9.3", + "log", + "num-traits", + "pp-rs", + "rustc-hash", + "spirv", + "termcolor", + "thiserror", + "unicode-xid", +] + +[[package]] +name = "naga_oil" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ac54c77b3529887f9668d3dd81e955e58f252b31a333f836e3548c06460b958" +dependencies = [ + "bit-set", + "codespan-reporting", + "data-encoding", + "indexmap 1.9.3", + "naga", + "once_cell", + "regex", + "regex-syntax 0.7.5", + "rustc-hash", + "thiserror", + "tracing", + "unicode-ident", +] + [[package]] name = "nanorand" version = "0.7.0" @@ -2033,6 +3731,57 @@ dependencies = [ "tempfile", ] +[[package]] +name = "ndk" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "451422b7e4718271c8b5b3aadf5adedba43dc76312454b387e98fae0fc951aa0" +dependencies = [ + "bitflags 1.3.2", + "jni-sys", + "ndk-sys", + "num_enum 0.5.11", + "raw-window-handle", + "thiserror", +] + +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + +[[package]] +name = "ndk-sys" +version = "0.4.1+23.1.7779620" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cf2aae958bd232cac5069850591667ad422d263686d75b52a065f9badeee5a3" +dependencies = [ + "jni-sys", +] + +[[package]] +name = "nix" +version = "0.24.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa52e972a9a719cecb6864fb88568781eb706bac2cd1d4f04a648542dbf78069" +dependencies = [ + "bitflags 1.3.2", + "cfg-if", + "libc", +] + +[[package]] +name = "nix" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" +dependencies = [ + "bitflags 2.4.1", + "cfg-if", + "libc", +] + [[package]] name = "nom" version = "7.1.3" @@ -2043,6 +3792,21 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nonmax" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "610a5acd306ec67f907abe5567859a3c693fb9886eb1f012ab8f2a47bef3db51" + +[[package]] +name = "ntapi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" +dependencies = [ + "winapi", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -2053,6 +3817,17 @@ dependencies = [ "winapi", ] +[[package]] +name = "num-derive" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "876a53fff98e03a936a674b29568b0e605f06b29372c2489ff4de23f1949743d" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "num-integer" version = "0.1.45" @@ -2093,6 +3868,48 @@ dependencies = [ "libc", ] +[[package]] +name = "num_enum" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f646caf906c20226733ed5b1374287eb97e3c2a5c227ce668c1f2ce20ae57c9" +dependencies = [ + "num_enum_derive 0.5.11", +] + +[[package]] +name = "num_enum" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a015b430d3c108a207fd776d2e2196aaf8b1cf8cf93253e3a097ff3085076a1" +dependencies = [ + "num_enum_derive 0.6.1", +] + +[[package]] +name = "num_enum_derive" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcbff9bc912032c62bf65ef1d5aea88983b420f4f839db1e9b0c281a25c9c799" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "num_enum_derive" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96667db765a921f7b295ffee8b60472b686a51d4f21c2ee4ffdb94c7013b65a6" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.39", +] + [[package]] name = "objc" version = "0.2.7" @@ -2100,6 +3917,42 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "915b1b472bc21c53464d6c8461c9d3af805ba1ef837e1cac254428f4a77177b1" dependencies = [ "malloc_buf", + "objc_exception", +] + +[[package]] +name = "objc-sys" +version = "0.2.0-beta.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b9834c1e95694a05a828b59f55fa2afec6288359cda67146126b3f90a55d7" + +[[package]] +name = "objc2" +version = "0.3.0-beta.3.patch-leaks.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e01640f9f2cb1220bbe80325e179e532cb3379ebcd1bf2279d703c19fe3a468" +dependencies = [ + "block2", + "objc-sys", + "objc2-encode", +] + +[[package]] +name = "objc2-encode" +version = "2.0.0-pre.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abfcac41015b00a120608fdaa6938c44cb983fee294351cc4bac7638b4e50512" +dependencies = [ + "objc-sys", +] + +[[package]] +name = "objc_exception" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad970fb455818ad6cba4c122ad012fae53ae8b4795f86378bce65e4f6bab2ca4" +dependencies = [ + "cc", ] [[package]] @@ -2111,6 +3964,38 @@ dependencies = [ "memchr", ] +[[package]] +name = "oboe" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8868cc237ee02e2d9618539a23a8d228b9bb3fc2e7a5b11eed3831de77c395d0" +dependencies = [ + "jni 0.20.0", + "ndk", + "ndk-context", + "num-derive", + "num-traits", + "oboe-sys", +] + +[[package]] +name = "oboe-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f44155e7fb718d3cfddcf70690b2b51ac4412f347cd9e4fbe511abe9cd7b5f2" +dependencies = [ + "cc", +] + +[[package]] +name = "ogg" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6951b4e8bf21c8193da321bcce9c9dd2e13c858fe078bf9054a288b419ae5d6e" +dependencies = [ + "byteorder", +] + [[package]] name = "once_cell" version = "1.18.0" @@ -2125,7 +4010,7 @@ checksum = "79a4c6c3a2b158f7f8f2a2fc5a969fa3a068df6fc9dbb4a43845436e3af7c800" dependencies = [ "bitflags 2.4.1", "cfg-if", - "foreign-types", + "foreign-types 0.3.2", "libc", "once_cell", "openssl-macros", @@ -2161,12 +4046,30 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "orbclient" +version = "0.3.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52f0d54bde9774d3a51dcf281a5def240c71996bc6ca05d2c847ec8b2b216166" +dependencies = [ + "libredox", +] + [[package]] name = "overload" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "owned_ttf_parser" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4586edfe4c648c71797a74c84bacb32b52b212eff5dfe2bb9f2c599844023e7" +dependencies = [ + "ttf-parser", +] + [[package]] name = "parking" version = "2.2.0" @@ -2191,7 +4094,7 @@ checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.4.1", "smallvec", "windows-targets 0.48.5", ] @@ -2262,6 +4165,12 @@ dependencies = [ "sha2", ] +[[package]] +name = "peeking_take_while" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" + [[package]] name = "percent-encoding" version = "2.3.1" @@ -2310,6 +4219,17 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "piper" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668d31b1c4eba19242f2088b2bf3316b82ca31082a8335764db4e083db7485d4" +dependencies = [ + "atomic-waker", + "fastrand 2.0.1", + "futures-io", +] + [[package]] name = "pkg-config" version = "0.3.27" @@ -2335,6 +4255,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "pp-rs" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb458bb7f6e250e6eb79d5026badc10a3ebb8f9a15d1fff0f13d17c71f4d6dee" +dependencies = [ + "unicode-xid", +] + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -2368,7 +4297,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" dependencies = [ "once_cell", - "toml_edit", + "toml_edit 0.19.15", ] [[package]] @@ -2380,6 +4309,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "profiling" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f89dff0959d98c9758c88826cc002e2c3d0b9dfac4139711d1f30de442f1139b" + [[package]] name = "prost" version = "0.12.3" @@ -2452,6 +4387,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "radsort" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17fd96390ed3feda12e1dfe2645ed587e0bea749e319333f104a33ff62f77a0b" + [[package]] name = "rand" version = "0.8.5" @@ -2482,6 +4423,18 @@ dependencies = [ "getrandom", ] +[[package]] +name = "range-alloc" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8a99fddc9f0ba0a85884b8d14e3592853e787d581ca1816c91349b10e4eeab" + +[[package]] +name = "raw-window-handle" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ff9a1f06a88b01621b7ae906ef0211290d1c8a168a15542486a8f61c0833b9" + [[package]] name = "rayon" version = "1.8.0" @@ -2502,6 +4455,21 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "rectangle-pack" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0d463f2884048e7153449a55166f91028d5b0ea53c79377099ce4e8cf0cf9bb" + +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.4.1" @@ -2519,8 +4487,17 @@ checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" dependencies = [ "aho-corasick", "memchr", - "regex-automata", - "regex-syntax", + "regex-automata 0.4.3", + "regex-syntax 0.8.2", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax 0.6.29", ] [[package]] @@ -2531,15 +4508,33 @@ checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-syntax 0.8.2", ] +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + +[[package]] +name = "regex-syntax" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" + [[package]] name = "regex-syntax" version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" +[[package]] +name = "renderdoc-sys" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "216080ab382b992234dda86873c18d4c48358f5cfcb70fd693d7f6f2131b628b" + [[package]] name = "reqwest" version = "0.11.22" @@ -2617,6 +4612,28 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "rodio" +version = "0.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b1bb7b48ee48471f55da122c0044fcc7600cfcc85db88240b89cb832935e611" +dependencies = [ + "cpal", + "lewton", +] + +[[package]] +name = "ron" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b91f7eff05f748767f183df4320a63d6936e9c6107d97c9e6bdd9784f4289c94" +dependencies = [ + "base64 0.21.5", + "bitflags 2.4.1", + "serde", + "serde_derive", +] + [[package]] name = "roxmltree" version = "0.14.1" @@ -2632,6 +4649,12 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc_version" version = "0.4.0" @@ -2697,6 +4720,17 @@ dependencies = [ "untrusted", ] +[[package]] +name = "ruzstd" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3ffab8f9715a0d455df4bbb9d21e91135aab3cd3ca187af0cd0c3c3f868fdc" +dependencies = [ + "byteorder", + "thiserror-core", + "twox-hash", +] + [[package]] name = "ryu" version = "1.0.15" @@ -2877,6 +4911,12 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shlex" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7cee0529a6d40f580e7a5e6c495c8fbfe21b7b52795ed4bb5e62cdf92bc6380" + [[package]] name = "signal-hook-registry" version = "1.4.1" @@ -2907,11 +4947,32 @@ dependencies = [ "autocfg", ] +[[package]] +name = "slotmap" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1e08e261d0e8f5c43123b7adf3e4ca1690d655377ac93a03b2c9d3e98de1342" +dependencies = [ + "version_check", +] + [[package]] name = "smallvec" version = "1.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" +dependencies = [ + "serde", +] + +[[package]] +name = "smol_str" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74212e6bbe9a4352329b2f68ba3130c15a3f26fe88ff22dbdc6cdd58fa85e99c" +dependencies = [ + "serde", +] [[package]] name = "socket2" @@ -2942,6 +5003,22 @@ dependencies = [ "lock_api", ] +[[package]] +name = "spirv" +version = "0.2.0+1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "246bfa38fe3db3f1dfc8ca5a2cdeb7348c78be2112740cc0ec8ef18b6d94f830" +dependencies = [ + "bitflags 1.3.2", + "num-traits", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "strsim" version = "0.10.0" @@ -2954,6 +5031,12 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" +[[package]] +name = "svg_fmt" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fb1df15f412ee2e9dfc1c504260fa695c1c3f10fe9f4a6ee2d2184d7d6450e2" + [[package]] name = "syn" version = "1.0.109" @@ -2976,6 +5059,20 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sysinfo" +version = "0.29.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a18d114d420ada3a891e6bc8e96a2023402203296a47cdd65083377dad18ba5" +dependencies = [ + "cfg-if", + "core-foundation-sys", + "libc", + "ntapi", + "once_cell", + "winapi", +] + [[package]] name = "system-configuration" version = "0.5.1" @@ -2997,6 +5094,18 @@ dependencies = [ "libc", ] +[[package]] +name = "taffy" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c2287b6d7f721ada4cddf61ade5e760b2c6207df041cac9bfaa192897362fd3" +dependencies = [ + "arrayvec", + "grid", + "num-traits", + "slotmap", +] + [[package]] name = "tempfile" version = "3.8.1" @@ -3004,8 +5113,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" dependencies = [ "cfg-if", - "fastrand", - "redox_syscall", + "fastrand 2.0.1", + "redox_syscall 0.4.1", "rustix", "windows-sys 0.48.0", ] @@ -3028,6 +5137,26 @@ dependencies = [ "thiserror-impl", ] +[[package]] +name = "thiserror-core" +version = "1.0.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c001ee18b7e5e3f62cbf58c7fe220119e68d902bb7443179c0c8aef30090e999" +dependencies = [ + "thiserror-core-impl", +] + +[[package]] +name = "thiserror-core-impl" +version = "1.0.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4c60d69f36615a077cc7663b9cb8e42275722d23e58a7fa3d2c7f2915d09d04" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + [[package]] name = "thiserror-impl" version = "1.0.50" @@ -3231,6 +5360,17 @@ dependencies = [ "winnow", ] +[[package]] +name = "toml_edit" +version = "0.20.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70f427fce4d84c72b5b732388bf4a9f4531b53f74e2887e3ecb2481f68f66d81" +dependencies = [ + "indexmap 2.1.0", + "toml_datetime", + "winnow", +] + [[package]] name = "tower-service" version = "0.3.2" @@ -3270,6 +5410,17 @@ dependencies = [ "valuable", ] +[[package]] +name = "tracing-log" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f751112709b4e791d8ce53e32c4ed2d353565a795ce84da2285393f41557bdf2" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + [[package]] name = "tracing-log" version = "0.2.0" @@ -3287,12 +5438,27 @@ version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" dependencies = [ + "matchers", "nu-ansi-term", + "once_cell", + "regex", "sharded-slab", "smallvec", "thread_local", + "tracing", "tracing-core", - "tracing-log", + "tracing-log 0.2.0", +] + +[[package]] +name = "tracing-wasm" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4575c663a174420fa2d78f4108ff68f65bf2fbb7dd89f33749b6e826b3626e07" +dependencies = [ + "tracing", + "tracing-subscriber", + "wasm-bindgen", ] [[package]] @@ -3301,6 +5467,12 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +[[package]] +name = "ttf-parser" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17f77d76d837a7830fe1d4f12b7b4ba4192c1888001c7164257e4bc6d21d96b4" + [[package]] name = "tungstenite" version = "0.20.1" @@ -3322,6 +5494,16 @@ dependencies = [ "utf-8", ] +[[package]] +name = "twox-hash" +version = "1.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" +dependencies = [ + "cfg-if", + "static_assertions", +] + [[package]] name = "typenum" version = "1.17.0" @@ -3364,6 +5546,12 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" +[[package]] +name = "unicode-xid" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" + [[package]] name = "untrusted" version = "0.9.0" @@ -3399,6 +5587,7 @@ version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" dependencies = [ + "getrandom", "serde", ] @@ -3414,6 +5603,12 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + [[package]] name = "version_check" version = "0.9.4" @@ -3485,6 +5680,12 @@ dependencies = [ "vulkano", ] +[[package]] +name = "waker-fn" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c4517f54858c779bbcbf228f4fca63d121bf85fbecb2dc578cdf4a39395690" + [[package]] name = "walkdir" version = "2.4.0" @@ -3603,6 +5804,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "wayland-scanner" +version = "0.29.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f4303d8fa22ab852f789e75a967f0a2cdc430a607751c0499bada3e451cbd53" +dependencies = [ + "proc-macro2", + "quote", + "xml-rs", +] + [[package]] name = "web-sys" version = "0.3.65" @@ -3651,6 +5863,105 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9193164d4de03a926d909d3bc7c30543cecb35400c02114792c2cae20d5e2dbb" +[[package]] +name = "wgpu" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "752e44d3998ef35f71830dd1ad3da513e628e2e4d4aedb0ab580f850827a0b41" +dependencies = [ + "arrayvec", + "cfg-if", + "js-sys", + "log", + "naga", + "parking_lot", + "profiling", + "raw-window-handle", + "smallvec", + "static_assertions", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "wgpu-core", + "wgpu-hal", + "wgpu-types", +] + +[[package]] +name = "wgpu-core" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f8a44dd301a30ceeed3c27d8c0090433d3da04d7b2a4042738095a424d12ae7" +dependencies = [ + "arrayvec", + "bit-vec", + "bitflags 2.4.1", + "codespan-reporting", + "log", + "naga", + "parking_lot", + "profiling", + "raw-window-handle", + "rustc-hash", + "smallvec", + "thiserror", + "web-sys", + "wgpu-hal", + "wgpu-types", +] + +[[package]] +name = "wgpu-hal" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a80bf0e3c77399bb52850cb0830af9bad073d5cfcb9dd8253bef8125c42db17" +dependencies = [ + "android_system_properties", + "arrayvec", + "ash", + "bit-set", + "bitflags 2.4.1", + "block", + "core-graphics-types", + "d3d12", + "glow", + "gpu-alloc", + "gpu-allocator", + "gpu-descriptor", + "hassle-rs", + "js-sys", + "khronos-egl", + "libc", + "libloading", + "log", + "metal", + "naga", + "objc", + "parking_lot", + "profiling", + "range-alloc", + "raw-window-handle", + "renderdoc-sys", + "rustc-hash", + "smallvec", + "thiserror", + "wasm-bindgen", + "web-sys", + "wgpu-types", + "winapi", +] + +[[package]] +name = "wgpu-types" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee64d7398d0c2f9ca48922c902ef69c42d000c759f3db41e355f4a570b052b67" +dependencies = [ + "bitflags 2.4.1", + "js-sys", + "web-sys", +] + [[package]] name = "which" version = "4.4.2" @@ -3663,6 +5974,12 @@ dependencies = [ "rustix", ] +[[package]] +name = "widestring" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "653f141f39ec16bba3c5abe400a0c60da7468261cc2cbf36805022876bc721a8" + [[package]] name = "winapi" version = "0.3.9" @@ -3694,6 +6011,45 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.44.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e745dab35a0c4c77aa3ce42d595e13d2003d6902d6b08c9ef5fc326d08da12b" +dependencies = [ + "windows-targets 0.42.2", +] + +[[package]] +name = "windows" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdacb41e6a96a052c6cb63a144f24900236121c6f63f4f8219fef5977ecb0c25" +dependencies = [ + "windows-targets 0.42.2", +] + +[[package]] +name = "windows" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-targets 0.48.5", +] + +[[package]] +name = "windows" +version = "0.51.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca229916c5ee38c2f2bc1e9d8f04df975b4bd93f9955dc69fabb5d91270045c9" +dependencies = [ + "windows-core", + "windows-targets 0.48.5", +] + [[package]] name = "windows-core" version = "0.51.1" @@ -3703,6 +6059,28 @@ dependencies = [ "windows-targets 0.48.5", ] +[[package]] +name = "windows-implement" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e2ee588991b9e7e6c8338edf3333fbe4da35dc72092643958ebb43f0ab2c49c" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "windows-interface" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6fb8df20c9bcaa8ad6ab513f7b40104840c8867d5751126e4df3b08388d0cc7" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "windows-sys" version = "0.45.0" @@ -3835,6 +6213,36 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "winit" +version = "0.28.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9596d90b45384f5281384ab204224876e8e8bf7d58366d9b795ad99aa9894b94" +dependencies = [ + "android-activity", + "bitflags 1.3.2", + "cfg_aliases", + "core-foundation", + "core-graphics", + "dispatch", + "instant", + "libc", + "log", + "mio", + "ndk", + "objc2", + "once_cell", + "orbclient", + "percent-encoding", + "raw-window-handle", + "redox_syscall 0.3.5", + "wasm-bindgen", + "wayland-scanner", + "web-sys", + "windows-sys 0.45.0", + "x11-dl", +] + [[package]] name = "winnow" version = "0.5.19" @@ -3854,6 +6262,23 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "x11-dl" +version = "2.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38735924fedd5314a6e548792904ed8c6de6636285cb9fec04d5b1db85c1516f" +dependencies = [ + "libc", + "once_cell", + "pkg-config", +] + +[[package]] +name = "xi-unicode" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a67300977d3dc3f8034dae89778f502b6ba20b269527b3223ba59c0cf393bb8a" + [[package]] name = "xml-rs" version = "0.8.19" @@ -3901,7 +6326,7 @@ dependencies = [ "aes", "byteorder", "bzip2", - "constant_time_eq", + "constant_time_eq 0.1.5", "crc32fast", "crossbeam-utils", "flate2", diff --git a/lkgpt/Cargo.toml b/lkgpt/Cargo.toml index 81d067b..97c6222 100644 --- a/lkgpt/Cargo.toml +++ b/lkgpt/Cargo.toml @@ -50,3 +50,4 @@ tracing-subscriber = "0.3.18" async-channel = "2.1.0" bytes = "1.5.0" url = "2.5.0" +bevy = "0.12.0" diff --git a/lkgpt/src/assets/gltf.rs b/lkgpt/src/assets/gltf.rs deleted file mode 100644 index 66eb891..0000000 --- a/lkgpt/src/assets/gltf.rs +++ /dev/null @@ -1,216 +0,0 @@ -use anyhow::{bail, Context, Result}; -use glam::Mat4; -use gltf::{buffer, Gltf}; -use std::{ - fs, - path::{self, Path}, - sync::Arc, -}; -use vulkano::{ - command_buffer::{ - allocator::StandardCommandBufferAllocator, AutoCommandBufferBuilder, - PrimaryAutoCommandBuffer, - }, - format, - image::{view::ImageView, ImageDimensions, ImmutableImage, MipmapsCount}, - memory::allocator::StandardMemoryAllocator, -}; - -use super::{ - materials, - mesh::{Mesh, MeshVertex}, - model::Model, - texture::Texture, -}; - -pub fn load_external_gltf( - memory_allocator: &StandardMemoryAllocator, - cmd_buffer_builder: &mut AutoCommandBufferBuilder< - PrimaryAutoCommandBuffer, - Arc, - >, - gltf_path: &str, -) -> Result { - if !gltf_path.ends_with(".gltf") { - bail!("gltf file's extension isn't .gltf"); - } - - let base_path = path::Path::new("assets"); - let full_path = base_path.join(gltf_path); - let gltf = Gltf::open(&full_path)?; - - // Load buffers - let buffer_data = gltf - .buffers() - .map(|buffer| { - let mut data = match buffer.source() { - buffer::Source::Bin => gltf - .blob - .as_deref() - .context("Failed to open gltf blob")? - .to_vec(), - buffer::Source::Uri(uri) => { - let uri = full_path - .parent() - .unwrap_or_else(|| Path::new("./")) - .join(uri); - let uri_display = uri.display().to_string(); - fs::read(uri) - .with_context(|| format!("Failed to read buffer uri: {}", uri_display))? - } - }; - if data.len() < buffer.length() { - bail!( - "Buffer length is too short, expected {}, got {}", - buffer.length(), - data.len() - ); - } - while data.len() % 4 != 0 { - data.push(0); - } - Ok(data) - }) - .collect::>>>()?; - - let textures = gltf - .images() - .map(|img| match img.source() { - gltf::image::Source::Uri { uri, .. } => { - if !(uri.ends_with(".png") || uri.ends_with(".jpeg") || uri.ends_with(".jpg")) { - bail!("only png, jpeg, and jpg texture images are supported {uri}") - } - let uri = full_path - .parent() - .unwrap_or_else(|| Path::new("./")) - .join(uri) - .display() - .to_string(); - let data = fs::read(&uri) - .with_context(|| format!("Failed to read image uri: {}", &uri))?; - - let image = image::load_from_memory(&data) - .with_context(|| format!("Failed to decode image bytes: {}", uri))? - // .grayscale() - .to_rgba8(); - - let (width, height) = image.dimensions(); - let dimensions = ImageDimensions::Dim2d { - width, - height, - array_layers: 1, - }; - - let image_data = image.into_raw(); - let image = ImmutableImage::from_iter( - memory_allocator, - image_data, - dimensions, - MipmapsCount::Log2, - format::Format::R8G8B8A8_UNORM, - cmd_buffer_builder, - )?; - let image_view = ImageView::new_default(image)?; - - Ok(Texture { image_view }) - } - _ => bail!("Only external images textures are supported"), - }) - .collect::>>()?; - - let materials = gltf - .materials() - .map(materials::get_material_data) - .collect::>(); - - let meshes = gltf - .meshes() - .flat_map(|mesh| { - mesh.primitives() - .map(|mesh_primitive| { - let material_index = - mesh_primitive.material().index().unwrap_or_default() as u8; - - let reader = mesh_primitive.reader(|buffer| Some(&buffer_data[buffer.index()])); - - let normals = reader - .read_normals() - .map_or(Vec::new(), |normals| normals.collect()); - - let colors = reader - .read_colors(0) - .map_or(Vec::new(), |colors| colors.into_rgba_f32().collect()); - - let tangents = reader - .read_tangents() - .map_or(Vec::new(), |tangents| tangents.collect()); - - let tex_coords = reader - .read_tex_coords(0) - .map_or(Vec::new(), |tex_coords| tex_coords.into_f32().collect()); - - let vertices = reader.read_positions().map_or(Vec::new(), |positions| { - let normals_len = normals.len(); - let tex_coords_len = tex_coords.len(); - let colors_len = colors.len(); - let tangents_len = tangents.len(); - positions - .enumerate() - .map(|(i, position)| { - let normal = if i >= normals_len { - Default::default() - } else { - normals[i] - }; - let tex_coords = if i >= tex_coords_len { - Default::default() - } else { - tex_coords[i] - }; - let color = if i >= colors_len { - Default::default() - } else { - colors[i] - }; - let tangent = if i >= tangents_len { - Default::default() - } else { - tangents[i] - }; - - MeshVertex { - position, - normal, - tex_coords, - color, - tangent, - material_index, - } - }) - .collect() - }); - - let indices = reader - .read_indices() - .map_or(Vec::new(), |i| i.into_u32().collect()); - - Mesh::new(memory_allocator, vertices, indices, material_index).with_context( - || { - format!( - "Failed to load gltf mesh: {}", - mesh.name().unwrap_or("Unnamed") - ) - }, - ) - }) - .collect::>() - }) - .collect::>>()?; - - Ok(Model { - meshes, - transforms: vec![Mat4::IDENTITY], - textures, - materials, - }) -} diff --git a/lkgpt/src/assets/materials.rs b/lkgpt/src/assets/materials.rs deleted file mode 100644 index 4de84df..0000000 --- a/lkgpt/src/assets/materials.rs +++ /dev/null @@ -1,256 +0,0 @@ -use gltf::material::AlphaMode; -use vulkano::buffer::BufferContents; - -#[derive(Clone, Debug)] -pub struct MeshMaterial { - name: String, - alpha_cutoff: Option, - alpha_mode: AlphaMode, - base_color_factor: [f32; 4], - base_color_texture_index: Option, - double_sided: bool, - emissive_factor: [f32; 3], - emissive_texture_index: Option, - ior: Option, - metallic_roughness_texture_index: Option, - normal_texture_index: Option, - occlusion_texture_index: Option, - specular_color_texture_index: Option, - specular_texture_index: Option, - specular_diffuse_texture_index: Option, - specular_glossiness_texture_index: Option, - transmission_texture_index: Option, - unlit: bool, - vol_thickness_texture_index: Option, - vol_thickness_factor: Option, - vol_attenuation_distance: Option, - vol_attenuation_color: Option<[f32; 3]>, -} - -#[derive(BufferContents, Clone, Copy, Debug)] -#[repr(C)] -pub struct GpuMaterial { - base_color_factor: [f32; 4], - base_color_texture_index: u8, - emissive_factor: [f32; 3], - emissive_texture_index: u8, - ior: f32, - metallic_roughness_texture_index: u8, - normal_texture_index: u8, - occlusion_texture_index: u8, - specular_color_texture_index: u8, - specular_texture_index: u8, - specular_diffuse_texture_index: u8, - specular_glossiness_texture_index: u8, - transmission_texture_index: u8, - unlit: u8, // 1 for true, 0 for false - vol_thickness_texture_index: u8, - vol_thickness_factor: f32, - vol_attenuation_distance: f32, - vol_attenuation_color: [f32; 3], -} - -impl MeshMaterial { - pub fn update_texture_indexs(&mut self, scene_tex_arr_len: u8) { - if let Some(base_color_texture_index) = &mut self.base_color_texture_index { - *base_color_texture_index += scene_tex_arr_len; - }; - if let Some(emissive_texture_index) = &mut self.emissive_texture_index { - *emissive_texture_index += scene_tex_arr_len; - }; - - if let Some(metallic_roughness_texture_index) = &mut self.metallic_roughness_texture_index { - *metallic_roughness_texture_index += scene_tex_arr_len; - }; - - if let Some(normal_texture_index) = &mut self.normal_texture_index { - *normal_texture_index += scene_tex_arr_len; - }; - - if let Some(occlusion_texture_index) = &mut self.occlusion_texture_index { - *occlusion_texture_index += scene_tex_arr_len; - }; - - if let Some(specular_color_texture_index) = &mut self.specular_color_texture_index { - *specular_color_texture_index += scene_tex_arr_len; - }; - - if let Some(specular_texture_index) = &mut self.specular_texture_index { - *specular_texture_index += scene_tex_arr_len; - }; - - if let Some(specular_diffuse_texture_index) = &mut self.specular_diffuse_texture_index { - *specular_diffuse_texture_index += scene_tex_arr_len; - }; - - if let Some(specular_glossiness_texture_index) = &mut self.specular_glossiness_texture_index - { - *specular_glossiness_texture_index += scene_tex_arr_len; - }; - - if let Some(transmission_texture_index) = &mut self.transmission_texture_index { - *transmission_texture_index += scene_tex_arr_len; - }; - - if let Some(vol_thickness_texture_index) = &mut self.vol_thickness_texture_index { - *vol_thickness_texture_index += scene_tex_arr_len; - }; - } - - pub fn to_gpu_material(&self) -> GpuMaterial { - GpuMaterial { - base_color_factor: self.base_color_factor, - base_color_texture_index: self - .base_color_texture_index - .map_or(Default::default(), |value| value), - emissive_factor: self.emissive_factor, - emissive_texture_index: self - .emissive_texture_index - .map_or(Default::default(), |value| value), - ior: self.ior.map_or(Default::default(), |value| value), - metallic_roughness_texture_index: self - .metallic_roughness_texture_index - .map_or(Default::default(), |value| value), - normal_texture_index: self - .normal_texture_index - .map_or(Default::default(), |value| value), - occlusion_texture_index: self - .occlusion_texture_index - .map_or(Default::default(), |value| value), - specular_color_texture_index: self - .specular_color_texture_index - .map_or(Default::default(), |value| value), - specular_texture_index: self - .specular_texture_index - .map_or(Default::default(), |value| value), - specular_diffuse_texture_index: self - .specular_diffuse_texture_index - .map_or(Default::default(), |value| value), - specular_glossiness_texture_index: self - .specular_glossiness_texture_index - .map_or(Default::default(), |value| value), - transmission_texture_index: self - .transmission_texture_index - .map_or(Default::default(), |value| value), - unlit: self.unlit.into(), - vol_thickness_texture_index: self - .vol_thickness_texture_index - .map_or(Default::default(), |value| value), - vol_thickness_factor: self - .vol_thickness_factor - .map_or(Default::default(), |value| value), - vol_attenuation_distance: self - .vol_attenuation_distance - .map_or(Default::default(), |value| value), - vol_attenuation_color: self - .vol_attenuation_color - .map_or(Default::default(), |value| value), - } - } -} - -pub fn get_material_data(material: gltf::Material) -> MeshMaterial { - let name = material - .name() - .map_or("UnNamed".to_string(), |name| name.to_string()); - - let pbr_metallic = material.pbr_metallic_roughness(); - // ---- - let alpha_cutoff = material.alpha_cutoff(); - let alpha_mode = material.alpha_mode(); - let base_color_factor = pbr_metallic.base_color_factor(); - let base_color_texture_index = pbr_metallic - .base_color_texture() - .map(|info| info.texture().index() as u8); - let double_sided = material.double_sided(); - let emissive_factor = material.emissive_factor(); - let emissive_texture_index = material - .emissive_texture() - .map(|info| info.texture().index() as u8); - let ior = material.ior(); - let metallic_roughness_texture_index = pbr_metallic - .metallic_roughness_texture() - .map(|info| info.texture().index() as u8); - let normal_texture_index = material - .normal_texture() - .map(|normal| normal.texture().index() as u8); - let occlusion_texture_index = material - .occlusion_texture() - .map(|occlusion| occlusion.texture().index() as u8); - let (specular_color_texture_index, specular_texture_index) = - material.specular().map_or((None, None), |specular| { - let specular_color_texture_index = specular - .specular_color_texture() - .map(|info| info.texture().index() as u8); - let specular_texture_index = specular - .specular_texture() - .map(|info| info.texture().index() as u8); - (specular_color_texture_index, specular_texture_index) - }); - let (specular_diffuse_texture_index, specular_glossiness_texture_index) = material - .pbr_specular_glossiness() - .map_or((None, None), |pbr_gloss| { - let specular_diffuse_texture_index = pbr_gloss - .diffuse_texture() - .map(|info| info.texture().index() as u8); - let specular_glossiness_texture_index = pbr_gloss - .specular_glossiness_texture() - .map(|info| info.texture().index() as u8); - ( - specular_diffuse_texture_index, - specular_glossiness_texture_index, - ) - }); - let transmission_texture_index = material.transmission().and_then(|transmission| { - transmission - .transmission_texture() - .map(|info| info.texture().index() as u8) - }); - let unlit = material.unlit(); - let ( - vol_thickness_texture_index, - vol_thickness_factor, - vol_attenuation_distance, - vol_attenuation_color, - ) = material.volume().map_or((None, None, None, None), |tex| { - let vol_thickness_texture_index = tex - .thickness_texture() - .map(|tex| tex.texture().index() as u8); - let vol_thickness_factor = tex.thickness_factor(); - let vol_attenuation_distance = tex.attenuation_distance(); - let vol_attenuation_color = tex.attenuation_color(); - ( - vol_thickness_texture_index, - Some(vol_thickness_factor), - Some(vol_attenuation_distance), - Some(vol_attenuation_color), - ) - }); - - MeshMaterial { - name, - alpha_cutoff, - alpha_mode, - base_color_factor, - base_color_texture_index, - double_sided, - emissive_factor, - emissive_texture_index, - ior, - metallic_roughness_texture_index, - normal_texture_index, - occlusion_texture_index, - specular_color_texture_index, - specular_texture_index, - specular_diffuse_texture_index, - specular_glossiness_texture_index, - transmission_texture_index, - unlit, - vol_thickness_texture_index, - vol_thickness_factor, - vol_attenuation_distance, - vol_attenuation_color, - } -} - -/* base_color_texture,emissive_texture,metallic_roughness_texture,normal_texture,occlusion_texture,specular_color_texture,specular_texture,specular_diffuse_texture,specular_glossiness_texture,transmission_texture,volume_texture */ diff --git a/lkgpt/src/assets/mesh.rs b/lkgpt/src/assets/mesh.rs deleted file mode 100644 index aa4b5c4..0000000 --- a/lkgpt/src/assets/mesh.rs +++ /dev/null @@ -1,101 +0,0 @@ -use anyhow::{Context, Result}; -use vulkano::{ - buffer::{Buffer, BufferContents, BufferCreateInfo, BufferUsage, Subbuffer}, - memory::allocator::{AllocationCreateInfo, MemoryUsage, StandardMemoryAllocator}, - pipeline::graphics::vertex_input::Vertex, -}; - -#[derive(BufferContents, Clone, Debug, Vertex)] -#[repr(C)] -pub struct MeshVertex { - #[format(R32G32B32_SFLOAT)] - pub position: [f32; 3], - #[format(R32G32_SFLOAT)] - pub tex_coords: [f32; 2], - #[format(R32G32B32_SFLOAT)] - pub normal: [f32; 3], - #[format(R32G32B32A32_SFLOAT)] - pub color: [f32; 4], - #[format(R32G32B32A32_SFLOAT)] - pub tangent: [f32; 4], - #[format(R8_UINT)] - pub material_index: u8, -} - -#[derive(Clone, Debug)] -pub struct Mesh { - pub vertex_buffer: Subbuffer<[MeshVertex]>, - pub index_buffer: Subbuffer<[u32]>, - vertices: Vec, - indices: Vec, - material_index: u8, -} - -impl Mesh { - pub fn new( - memory_allocator: &StandardMemoryAllocator, - vertices: Vec, - indices: Vec, - material_index: u8, - ) -> Result { - let vertex_buffer = Buffer::from_iter( - memory_allocator, - BufferCreateInfo { - usage: BufferUsage::VERTEX_BUFFER, - ..Default::default() - }, - AllocationCreateInfo { - usage: MemoryUsage::Upload, - ..Default::default() - }, - vertices.clone(), - ) - .context("failed to create vertex buffer")?; - - let index_buffer = Buffer::from_iter( - memory_allocator, - BufferCreateInfo { - usage: BufferUsage::INDEX_BUFFER, - ..Default::default() - }, - AllocationCreateInfo { - usage: MemoryUsage::Upload, - ..Default::default() - }, - indices.clone(), - ) - .context("failed to create index buffer")?; - - Ok(Self { - vertex_buffer, - index_buffer, - vertices, - indices, - material_index, - }) - } - - pub fn num_of_indices(&self) -> u64 { - self.indices.len() as u64 - } - - pub fn num_of_vertices(&self) -> u64 { - self.vertices.len() as u64 - } - - pub fn indices(&self) -> &[u32] { - &self.indices - } - - pub fn update_material_index(&mut self, scene_material_arr_len: u8) { - self.material_index += scene_material_arr_len; - - self.vertex_buffer - .write() - .expect("failed to write to vertex buffer") - .iter_mut() - .for_each(|vertex| { - vertex.material_index += scene_material_arr_len; - }); - } -} diff --git a/lkgpt/src/assets/mod.rs b/lkgpt/src/assets/mod.rs deleted file mode 100644 index 3fa9780..0000000 --- a/lkgpt/src/assets/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -pub mod gltf; -pub mod materials; -pub mod mesh; -pub mod model; -pub mod texture; diff --git a/lkgpt/src/assets/model.rs b/lkgpt/src/assets/model.rs deleted file mode 100644 index f379846..0000000 --- a/lkgpt/src/assets/model.rs +++ /dev/null @@ -1,15 +0,0 @@ -use glam::Mat4; - -use super::{ - materials::{self, MeshMaterial}, - mesh::{self, Mesh}, - texture::{self, Texture}, -}; - -#[derive(Clone, Debug)] -pub struct Model { - pub meshes: Vec, - pub textures: Vec, - pub transforms: Vec, - pub materials: Vec, -} diff --git a/lkgpt/src/assets/texture.rs b/lkgpt/src/assets/texture.rs deleted file mode 100644 index d8436f7..0000000 --- a/lkgpt/src/assets/texture.rs +++ /dev/null @@ -1,8 +0,0 @@ -use std::sync::Arc; - -use vulkano::image::{view::ImageView, ImmutableImage}; - -#[derive(Clone, Debug)] -pub struct Texture { - pub image_view: Arc>, -} diff --git a/lkgpt/src/core.rs b/lkgpt/src/core.rs deleted file mode 100644 index edf2c24..0000000 --- a/lkgpt/src/core.rs +++ /dev/null @@ -1,174 +0,0 @@ -#![allow(unused_parens, non_snake_case)] -use anyhow::{Context, Result}; -use log::{info, warn}; -use std::sync::Arc; -use vulkano::{ - device::{ - physical::{PhysicalDevice, PhysicalDeviceType}, - Device, DeviceCreateInfo, DeviceExtensions, Features, Properties, Queue, QueueCreateInfo, - QueueFlags, - }, - instance::{Instance, InstanceCreateInfo}, - Version, VulkanLibrary, -}; - -pub struct Engine { - vkdevice: Arc, - gfx_queue: Arc, - instance: Arc, - gfx_queue_family_index: u32, - num_of_triangles: u64, - num_of_vertices: u64, - avg_fps: f32, -} - -impl Engine { - pub fn new() -> Result { - warn!("ONLY WORKS / TESTED WITH VULKAN V1.3.250.1"); - let library = VulkanLibrary::new().context("no local Vulkan library/DLL")?; - let instance = Instance::new( - library, - InstanceCreateInfo { - #[cfg(target_os = "macos")] - enumerate_portability: true, - ..Default::default() - }, - ) - .context("failed to create instance")?; - - let mut device_extensions = DeviceExtensions::empty(); - - let features = Features { - dynamic_rendering: true, - fill_mode_non_solid: true, - multi_draw_indirect: true, - runtime_descriptor_array: true, - descriptor_binding_partially_bound: true, - descriptor_binding_variable_descriptor_count: true, - shader_sampled_image_array_non_uniform_indexing: true, - // extended_dynamic_state: true, - ..Features::empty() - }; - - let (processing_device, queue_family_index) = - Self::select_processing_device(&instance, &device_extensions, &features); - - if processing_device.api_version() < Version::V1_3 { - device_extensions.khr_dynamic_rendering = true; - } - - // a device represents an open channel of communication with the GPU/CPU. - let (vkdevice, mut queues) = Device::new( - processing_device, - DeviceCreateInfo { - enabled_extensions: device_extensions, - enabled_features: features, - queue_create_infos: vec![QueueCreateInfo { - queue_family_index, - ..Default::default() - }], - ..Default::default() - }, - ) - .context("failed to create device")?; - - // queues are used to submit work to the device. They are created along with the device. - // they are somewhat like the GPU equivalent of CPU threads. - let gfx_queue = queues - .next() - .expect("failed to get first queue in iterator"); - - Ok(Self { - vkdevice, - gfx_queue, - gfx_queue_family_index: queue_family_index, - instance, - avg_fps: 0.0, - num_of_triangles: 0, - num_of_vertices: 0, - }) - } - - pub fn get_vkdevice(&self) -> Arc { - self.vkdevice.clone() - } - - pub fn get_vkdevice_properties(&self) -> vulkano::device::Properties { - self.vkdevice.physical_device().properties().clone() - } - - pub fn get_gfx_queue(&self) -> Arc { - self.gfx_queue.clone() - } - - pub fn get_instance(&self) -> Arc { - self.instance.clone() - } - - pub fn get_avg_fps(&self) -> f32 { - self.avg_fps - } - - pub fn num_of_triangles(&self) -> u64 { - self.num_of_triangles - } - - pub fn num_of_vertices(&self) -> u64 { - self.num_of_vertices - } - - pub fn gfx_queue_family_index(&self) -> u32 { - self.gfx_queue_family_index - } - - fn select_processing_device( - instance: &Arc, - device_extensions: &DeviceExtensions, - features: &Features, - ) -> (Arc, u32) { - // processing device (CPU/GPU) to connect to - info!("Available devices:"); - let (processing_device, queue_family_index) = instance - .enumerate_physical_devices() - .expect("could not enumerate devices") - .filter(|p| { - let Properties { - device_name, - device_type, - .. - } = &p.properties(); - info!( - "- {} | {:?} | Vulkan v{:?}", - device_name, - device_type, - p.api_version() - ); - p.api_version() >= Version::V1_3 || p.supported_extensions().khr_dynamic_rendering - }) - .filter(|p| p.supported_extensions().contains(device_extensions)) - .filter(|p| p.supported_features().contains(features)) - .filter_map(|p| { - p.queue_family_properties() - .iter() - .enumerate() - .position(|(_, q)| { - q.queue_flags.intersects(QueueFlags::GRAPHICS) - // && p.surface_support(i as u32, &surface).unwrap_or(false) - }) - .map(|q| (p, q as u32)) - }) - .min_by_key(|(p, _)| match p.properties().device_type { - PhysicalDeviceType::DiscreteGpu => 0, - PhysicalDeviceType::IntegratedGpu => 1, - PhysicalDeviceType::VirtualGpu => 2, - PhysicalDeviceType::Cpu => 3, - PhysicalDeviceType::Other => 4, - _ => 5, - }) - .expect("no devices available"); - - info!("* Using {}", processing_device.properties().device_name,); - - (processing_device, queue_family_index) - } -} diff --git a/lkgpt/src/gpt.rs b/lkgpt/src/gpt.rs deleted file mode 100644 index 58edb7f..0000000 --- a/lkgpt/src/gpt.rs +++ /dev/null @@ -1,81 +0,0 @@ -use std::time::{Duration, Instant}; - -use async_openai::{ - config::OpenAIConfig, - types::{ - ChatCompletionRequestUserMessageArgs, ChatCompletionRequestUserMessageContent, - CreateChatCompletionRequestArgs, - }, - Client, -}; -use futures::StreamExt; -use log::{error, info, warn}; -use tokio::{sync::mpsc, time}; - -use crate::{stt::STT, tts::TTS}; - -/// Stream text chunks to gpt as it's being generated, with <1s latency. -/// Note: if chunks don't end with space or punctuation (" ", ".", "?", "!"), -/// the stream will wait for more text. -/// Used during input streaming to chunk text blocks and set last char to space -pub async fn gpt( - mut text_input_rx: mpsc::UnboundedReceiver, - openai_client: Client, - mut tts_client: TTS, -) -> anyhow::Result<()> { - let splitters = [ - '.', ',', '?', '!', ';', ':', '—', '-', '(', ')', '[', ']', '}', ' ', - ]; - - let mut txt_buffer = String::new(); - let mut tts_buffer = String::new(); - - let mut req_args = CreateChatCompletionRequestArgs::default(); - let openai_req = req_args.model("gpt-3.5-turbo").max_tokens(512u16); - - // let text_latency = Duration::from_millis(500); - while let Some(chunk) = text_input_rx.recv().await { - txt_buffer.push_str(&chunk); - if ends_with_splitter(&splitters, &txt_buffer){ - let request = openai_req - .messages([ChatCompletionRequestUserMessageArgs::default() - .content(ChatCompletionRequestUserMessageContent::Text( - txt_buffer.clone(), - )) - .build()? - .into()]) - .build()?; - - let mut gpt_resp_stream = openai_client.chat().create_stream(request).await?; - while let Some(result) = gpt_resp_stream.next().await { - match result { - Ok(response) => { - for chat_choice in response.choices { - if let Some(content) = chat_choice.delta.content { - tts_buffer.push_str(&content); - if ends_with_splitter(&splitters, &tts_buffer) { - if let Err(e) = tts_client.send(tts_buffer.clone()) { - error!("Coudln't send gpt text chunk to tts channel - {e}"); - } else { - tts_buffer.clear(); - }; - } - }; - } - } - Err(err) => { - warn!("chunk error: {err:#?}"); - } - } - } - txt_buffer.clear(); - } else if !txt_buffer.ends_with(' ') { - txt_buffer.push(' '); - } - } - Ok(()) -} - -fn ends_with_splitter(splitters: &[char], chunk: &str) -> bool { - !chunk.is_empty() && chunk != " " && splitters.iter().any(|&splitter| chunk.ends_with(splitter)) -} diff --git a/lkgpt/src/main.rs b/lkgpt/src/main.rs index 578258d..702f543 100644 --- a/lkgpt/src/main.rs +++ b/lkgpt/src/main.rs @@ -1,25 +1,3086 @@ #![feature(ascii_char, async_closure)] -mod assets; -mod core; -mod gpt; -mod room_events; -mod scene; -mod stt; -mod track_pub; -mod tts; -mod turbo; -mod webrtc; - -mod response; -mod routes; -mod state; -mod utils; - use actix_web::{middleware, web::Data, App, HttpServer}; +use log::info; +use std::env; + +mod assets{ + pub mod gltf{ + use anyhow::{bail, Context, Result}; + use glam::Mat4; + use gltf::{buffer, Gltf}; + use std::{ + fs, + path::{self, Path}, + sync::Arc, + }; + use vulkano::{ + command_buffer::{ + allocator::StandardCommandBufferAllocator, AutoCommandBufferBuilder, + PrimaryAutoCommandBuffer, + }, + format, + image::{view::ImageView, ImageDimensions, ImmutableImage, MipmapsCount}, + memory::allocator::StandardMemoryAllocator, + }; + + use super::{ + materials, + mesh::{Mesh, MeshVertex}, + model::Model, + texture::Texture, + }; + + pub fn load_external_gltf( + memory_allocator: &StandardMemoryAllocator, + cmd_buffer_builder: &mut AutoCommandBufferBuilder< + PrimaryAutoCommandBuffer, + Arc, + >, + gltf_path: &str, + ) -> Result { + if !gltf_path.ends_with(".gltf") { + bail!("gltf file's extension isn't .gltf"); + } + + let base_path = path::Path::new("assets"); + let full_path = base_path.join(gltf_path); + let gltf = Gltf::open(&full_path)?; + + // Load buffers + let buffer_data = gltf + .buffers() + .map(|buffer| { + let mut data = match buffer.source() { + buffer::Source::Bin => gltf + .blob + .as_deref() + .context("Failed to open gltf blob")? + .to_vec(), + buffer::Source::Uri(uri) => { + let uri = full_path + .parent() + .unwrap_or_else(|| Path::new("./")) + .join(uri); + let uri_display = uri.display().to_string(); + fs::read(uri) + .with_context(|| format!("Failed to read buffer uri: {}", uri_display))? + } + }; + if data.len() < buffer.length() { + bail!( + "Buffer length is too short, expected {}, got {}", + buffer.length(), + data.len() + ); + } + while data.len() % 4 != 0 { + data.push(0); + } + Ok(data) + }) + .collect::>>>()?; + + let textures = gltf + .images() + .map(|img| match img.source() { + gltf::image::Source::Uri { uri, .. } => { + if !(uri.ends_with(".png") || uri.ends_with(".jpeg") || uri.ends_with(".jpg")) { + bail!("only png, jpeg, and jpg texture images are supported {uri}") + } + let uri = full_path + .parent() + .unwrap_or_else(|| Path::new("./")) + .join(uri) + .display() + .to_string(); + let data = fs::read(&uri) + .with_context(|| format!("Failed to read image uri: {}", &uri))?; + + let image = image::load_from_memory(&data) + .with_context(|| format!("Failed to decode image bytes: {}", uri))? + // .grayscale() + .to_rgba8(); + + let (width, height) = image.dimensions(); + let dimensions = ImageDimensions::Dim2d { + width, + height, + array_layers: 1, + }; + + let image_data = image.into_raw(); + let image = ImmutableImage::from_iter( + memory_allocator, + image_data, + dimensions, + MipmapsCount::Log2, + format::Format::R8G8B8A8_UNORM, + cmd_buffer_builder, + )?; + let image_view = ImageView::new_default(image)?; + + Ok(Texture { image_view }) + } + _ => bail!("Only external images textures are supported"), + }) + .collect::>>()?; + + let materials = gltf + .materials() + .map(materials::get_material_data) + .collect::>(); + + let meshes = gltf + .meshes() + .flat_map(|mesh| { + mesh.primitives() + .map(|mesh_primitive| { + let material_index = + mesh_primitive.material().index().unwrap_or_default() as u8; + + let reader = mesh_primitive.reader(|buffer| Some(&buffer_data[buffer.index()])); + + let normals = reader + .read_normals() + .map_or(Vec::new(), |normals| normals.collect()); + + let colors = reader + .read_colors(0) + .map_or(Vec::new(), |colors| colors.into_rgba_f32().collect()); + + let tangents = reader + .read_tangents() + .map_or(Vec::new(), |tangents| tangents.collect()); + + let tex_coords = reader + .read_tex_coords(0) + .map_or(Vec::new(), |tex_coords| tex_coords.into_f32().collect()); + + let vertices = reader.read_positions().map_or(Vec::new(), |positions| { + let normals_len = normals.len(); + let tex_coords_len = tex_coords.len(); + let colors_len = colors.len(); + let tangents_len = tangents.len(); + positions + .enumerate() + .map(|(i, position)| { + let normal = if i >= normals_len { + Default::default() + } else { + normals[i] + }; + let tex_coords = if i >= tex_coords_len { + Default::default() + } else { + tex_coords[i] + }; + let color = if i >= colors_len { + Default::default() + } else { + colors[i] + }; + let tangent = if i >= tangents_len { + Default::default() + } else { + tangents[i] + }; + + MeshVertex { + position, + normal, + tex_coords, + color, + tangent, + material_index, + } + }) + .collect() + }); + + let indices = reader + .read_indices() + .map_or(Vec::new(), |i| i.into_u32().collect()); + + Mesh::new(memory_allocator, vertices, indices, material_index).with_context( + || { + format!( + "Failed to load gltf mesh: {}", + mesh.name().unwrap_or("Unnamed") + ) + }, + ) + }) + .collect::>() + }) + .collect::>>()?; + + Ok(Model { + meshes, + transforms: vec![Mat4::IDENTITY], + textures, + materials, + }) + } + + } + pub mod materials{ + use gltf::material::AlphaMode; + use vulkano::buffer::BufferContents; + + #[derive(Clone, Debug)] + pub struct MeshMaterial { + name: String, + alpha_cutoff: Option, + alpha_mode: AlphaMode, + base_color_factor: [f32; 4], + base_color_texture_index: Option, + double_sided: bool, + emissive_factor: [f32; 3], + emissive_texture_index: Option, + ior: Option, + metallic_roughness_texture_index: Option, + normal_texture_index: Option, + occlusion_texture_index: Option, + specular_color_texture_index: Option, + specular_texture_index: Option, + specular_diffuse_texture_index: Option, + specular_glossiness_texture_index: Option, + transmission_texture_index: Option, + unlit: bool, + vol_thickness_texture_index: Option, + vol_thickness_factor: Option, + vol_attenuation_distance: Option, + vol_attenuation_color: Option<[f32; 3]>, + } + + #[derive(BufferContents, Clone, Copy, Debug)] + #[repr(C)] + pub struct GpuMaterial { + base_color_factor: [f32; 4], + base_color_texture_index: u8, + emissive_factor: [f32; 3], + emissive_texture_index: u8, + ior: f32, + metallic_roughness_texture_index: u8, + normal_texture_index: u8, + occlusion_texture_index: u8, + specular_color_texture_index: u8, + specular_texture_index: u8, + specular_diffuse_texture_index: u8, + specular_glossiness_texture_index: u8, + transmission_texture_index: u8, + unlit: u8, // 1 for true, 0 for false + vol_thickness_texture_index: u8, + vol_thickness_factor: f32, + vol_attenuation_distance: f32, + vol_attenuation_color: [f32; 3], + } + + impl MeshMaterial { + pub fn update_texture_indexs(&mut self, scene_tex_arr_len: u8) { + if let Some(base_color_texture_index) = &mut self.base_color_texture_index { + *base_color_texture_index += scene_tex_arr_len; + }; + if let Some(emissive_texture_index) = &mut self.emissive_texture_index { + *emissive_texture_index += scene_tex_arr_len; + }; + + if let Some(metallic_roughness_texture_index) = &mut self.metallic_roughness_texture_index { + *metallic_roughness_texture_index += scene_tex_arr_len; + }; + + if let Some(normal_texture_index) = &mut self.normal_texture_index { + *normal_texture_index += scene_tex_arr_len; + }; + + if let Some(occlusion_texture_index) = &mut self.occlusion_texture_index { + *occlusion_texture_index += scene_tex_arr_len; + }; + + if let Some(specular_color_texture_index) = &mut self.specular_color_texture_index { + *specular_color_texture_index += scene_tex_arr_len; + }; + + if let Some(specular_texture_index) = &mut self.specular_texture_index { + *specular_texture_index += scene_tex_arr_len; + }; + + if let Some(specular_diffuse_texture_index) = &mut self.specular_diffuse_texture_index { + *specular_diffuse_texture_index += scene_tex_arr_len; + }; + + if let Some(specular_glossiness_texture_index) = &mut self.specular_glossiness_texture_index + { + *specular_glossiness_texture_index += scene_tex_arr_len; + }; + + if let Some(transmission_texture_index) = &mut self.transmission_texture_index { + *transmission_texture_index += scene_tex_arr_len; + }; + + if let Some(vol_thickness_texture_index) = &mut self.vol_thickness_texture_index { + *vol_thickness_texture_index += scene_tex_arr_len; + }; + } + + pub fn to_gpu_material(&self) -> GpuMaterial { + GpuMaterial { + base_color_factor: self.base_color_factor, + base_color_texture_index: self + .base_color_texture_index + .map_or(Default::default(), |value| value), + emissive_factor: self.emissive_factor, + emissive_texture_index: self + .emissive_texture_index + .map_or(Default::default(), |value| value), + ior: self.ior.map_or(Default::default(), |value| value), + metallic_roughness_texture_index: self + .metallic_roughness_texture_index + .map_or(Default::default(), |value| value), + normal_texture_index: self + .normal_texture_index + .map_or(Default::default(), |value| value), + occlusion_texture_index: self + .occlusion_texture_index + .map_or(Default::default(), |value| value), + specular_color_texture_index: self + .specular_color_texture_index + .map_or(Default::default(), |value| value), + specular_texture_index: self + .specular_texture_index + .map_or(Default::default(), |value| value), + specular_diffuse_texture_index: self + .specular_diffuse_texture_index + .map_or(Default::default(), |value| value), + specular_glossiness_texture_index: self + .specular_glossiness_texture_index + .map_or(Default::default(), |value| value), + transmission_texture_index: self + .transmission_texture_index + .map_or(Default::default(), |value| value), + unlit: self.unlit.into(), + vol_thickness_texture_index: self + .vol_thickness_texture_index + .map_or(Default::default(), |value| value), + vol_thickness_factor: self + .vol_thickness_factor + .map_or(Default::default(), |value| value), + vol_attenuation_distance: self + .vol_attenuation_distance + .map_or(Default::default(), |value| value), + vol_attenuation_color: self + .vol_attenuation_color + .map_or(Default::default(), |value| value), + } + } + } + + pub fn get_material_data(material: gltf::Material) -> MeshMaterial { + let name = material + .name() + .map_or("UnNamed".to_string(), |name| name.to_string()); + + let pbr_metallic = material.pbr_metallic_roughness(); + // ---- + let alpha_cutoff = material.alpha_cutoff(); + let alpha_mode = material.alpha_mode(); + let base_color_factor = pbr_metallic.base_color_factor(); + let base_color_texture_index = pbr_metallic + .base_color_texture() + .map(|info| info.texture().index() as u8); + let double_sided = material.double_sided(); + let emissive_factor = material.emissive_factor(); + let emissive_texture_index = material + .emissive_texture() + .map(|info| info.texture().index() as u8); + let ior = material.ior(); + let metallic_roughness_texture_index = pbr_metallic + .metallic_roughness_texture() + .map(|info| info.texture().index() as u8); + let normal_texture_index = material + .normal_texture() + .map(|normal| normal.texture().index() as u8); + let occlusion_texture_index = material + .occlusion_texture() + .map(|occlusion| occlusion.texture().index() as u8); + let (specular_color_texture_index, specular_texture_index) = + material.specular().map_or((None, None), |specular| { + let specular_color_texture_index = specular + .specular_color_texture() + .map(|info| info.texture().index() as u8); + let specular_texture_index = specular + .specular_texture() + .map(|info| info.texture().index() as u8); + (specular_color_texture_index, specular_texture_index) + }); + let (specular_diffuse_texture_index, specular_glossiness_texture_index) = material + .pbr_specular_glossiness() + .map_or((None, None), |pbr_gloss| { + let specular_diffuse_texture_index = pbr_gloss + .diffuse_texture() + .map(|info| info.texture().index() as u8); + let specular_glossiness_texture_index = pbr_gloss + .specular_glossiness_texture() + .map(|info| info.texture().index() as u8); + ( + specular_diffuse_texture_index, + specular_glossiness_texture_index, + ) + }); + let transmission_texture_index = material.transmission().and_then(|transmission| { + transmission + .transmission_texture() + .map(|info| info.texture().index() as u8) + }); + let unlit = material.unlit(); + let ( + vol_thickness_texture_index, + vol_thickness_factor, + vol_attenuation_distance, + vol_attenuation_color, + ) = material.volume().map_or((None, None, None, None), |tex| { + let vol_thickness_texture_index = tex + .thickness_texture() + .map(|tex| tex.texture().index() as u8); + let vol_thickness_factor = tex.thickness_factor(); + let vol_attenuation_distance = tex.attenuation_distance(); + let vol_attenuation_color = tex.attenuation_color(); + ( + vol_thickness_texture_index, + Some(vol_thickness_factor), + Some(vol_attenuation_distance), + Some(vol_attenuation_color), + ) + }); + + MeshMaterial { + name, + alpha_cutoff, + alpha_mode, + base_color_factor, + base_color_texture_index, + double_sided, + emissive_factor, + emissive_texture_index, + ior, + metallic_roughness_texture_index, + normal_texture_index, + occlusion_texture_index, + specular_color_texture_index, + specular_texture_index, + specular_diffuse_texture_index, + specular_glossiness_texture_index, + transmission_texture_index, + unlit, + vol_thickness_texture_index, + vol_thickness_factor, + vol_attenuation_distance, + vol_attenuation_color, + } + } + + /* base_color_texture,emissive_texture,metallic_roughness_texture,normal_texture,occlusion_texture,specular_color_texture,specular_texture,specular_diffuse_texture,specular_glossiness_texture,transmission_texture,volume_texture */ + + } + pub mod mesh{ + use anyhow::{Context, Result}; + use vulkano::{ + buffer::{Buffer, BufferContents, BufferCreateInfo, BufferUsage, Subbuffer}, + memory::allocator::{AllocationCreateInfo, MemoryUsage, StandardMemoryAllocator}, + pipeline::graphics::vertex_input::Vertex, + }; + + #[derive(BufferContents, Clone, Debug, Vertex)] + #[repr(C)] + pub struct MeshVertex { + #[format(R32G32B32_SFLOAT)] + pub position: [f32; 3], + #[format(R32G32_SFLOAT)] + pub tex_coords: [f32; 2], + #[format(R32G32B32_SFLOAT)] + pub normal: [f32; 3], + #[format(R32G32B32A32_SFLOAT)] + pub color: [f32; 4], + #[format(R32G32B32A32_SFLOAT)] + pub tangent: [f32; 4], + #[format(R8_UINT)] + pub material_index: u8, + } + + #[derive(Clone, Debug)] + pub struct Mesh { + pub vertex_buffer: Subbuffer<[MeshVertex]>, + pub index_buffer: Subbuffer<[u32]>, + vertices: Vec, + indices: Vec, + material_index: u8, + } + + impl Mesh { + pub fn new( + memory_allocator: &StandardMemoryAllocator, + vertices: Vec, + indices: Vec, + material_index: u8, + ) -> Result { + let vertex_buffer = Buffer::from_iter( + memory_allocator, + BufferCreateInfo { + usage: BufferUsage::VERTEX_BUFFER, + ..Default::default() + }, + AllocationCreateInfo { + usage: MemoryUsage::Upload, + ..Default::default() + }, + vertices.clone(), + ) + .context("failed to create vertex buffer")?; + + let index_buffer = Buffer::from_iter( + memory_allocator, + BufferCreateInfo { + usage: BufferUsage::INDEX_BUFFER, + ..Default::default() + }, + AllocationCreateInfo { + usage: MemoryUsage::Upload, + ..Default::default() + }, + indices.clone(), + ) + .context("failed to create index buffer")?; + + Ok(Self { + vertex_buffer, + index_buffer, + vertices, + indices, + material_index, + }) + } + + pub fn num_of_indices(&self) -> u64 { + self.indices.len() as u64 + } + + pub fn num_of_vertices(&self) -> u64 { + self.vertices.len() as u64 + } + + pub fn indices(&self) -> &[u32] { + &self.indices + } + + pub fn update_material_index(&mut self, scene_material_arr_len: u8) { + self.material_index += scene_material_arr_len; + + self.vertex_buffer + .write() + .expect("failed to write to vertex buffer") + .iter_mut() + .for_each(|vertex| { + vertex.material_index += scene_material_arr_len; + }); + } + } + + } + pub mod model{ + use glam::Mat4; + + use super::{ + materials::{self, MeshMaterial}, + mesh::{self, Mesh}, + texture::{self, Texture}, + }; + + #[derive(Clone, Debug)] + pub struct Model { + pub meshes: Vec, + pub textures: Vec, + pub transforms: Vec, + pub materials: Vec, + } + + } + pub mod texture{ + use std::sync::Arc; + + use vulkano::image::{view::ImageView, ImmutableImage}; + + #[derive(Clone, Debug)] + pub struct Texture { + pub image_view: Arc>, + } + + } +} + +mod scene{ + pub mod camera{ + + use glam::{Mat4, Vec3}; + + pub struct Camera { + position: Vec3, + rotation: f32, + fov: f32, + aspect_ratio: f32, + scale: f32, + view: Mat4, + z_near: f32, + z_far: f32, + } + + impl Default for Camera { + fn default() -> Self { + Self::new() + } + } + + impl Camera { + pub fn new() -> Self { + Self { + position: Vec3::new(0.0, 0.0, 0.0), + rotation: 0.0, + fov: 120.0, + aspect_ratio: 16.0 / 9.0, + z_near: 0.1, + z_far: 100.0, + scale: 0.02, + view: Mat4::look_at_rh( + Vec3::new(0.3, 0.3, 1.0), + Vec3::new(0.0, 0.0, 0.0), + Vec3::new(0.0, -1.0, 0.0), + ), + } + } + + pub fn update_aspect_ratio_from_scene(mut self, scene_dims: [u32; 2]) -> Self { + self.aspect_ratio = scene_dims[0] as f32 / scene_dims[1] as f32; + self + } + + pub fn update_rotation(&mut self, rotation: f32) { + self.rotation = rotation; + } + + pub fn format_to_subbuffer_data(&self) -> (Mat4, Mat4) { + let scale_matrix = Mat4::from_scale(Vec3::from_array([self.scale; 3])); + + let view_scale_dot_product = self.view * scale_matrix; + + let projection_matrix = + Mat4::perspective_rh(self.fov, self.aspect_ratio, self.z_near, self.z_far); + + (view_scale_dot_product, projection_matrix) + } + + pub fn get_model_matrix(&self) -> Mat4 { + Mat4::from_rotation_y(self.rotation) + } + } + + } + + mod material_manager{ + use crate::assets::materials::MeshMaterial; + use std::collections::HashMap; + + #[derive(Debug)] + pub struct OrderedMaterialsMap { + map: HashMap, + keys: Vec, + } + + impl Default for OrderedMaterialsMap { + fn default() -> Self { + Self::new() + } + } + + impl OrderedMaterialsMap { + pub fn new() -> Self { + OrderedMaterialsMap { + map: HashMap::new(), + keys: Vec::new(), + } + } + + pub fn insert(&mut self, key: u8, value: MeshMaterial) { + let result = self.map.insert(key, value); + if result.is_none() { + self.keys.push(key); + } + } + + pub fn get(&self, key: &u8) -> Option<&MeshMaterial> { + self.map.get(key) + } + + pub fn iter(&self) -> impl Iterator { + self.keys + .iter() + .filter_map(move |k| self.map.get_key_value(k)) + } + + pub fn len(&self) -> u8 { + self.map.len() as u8 + } + } + + } + + mod node{ + use std::sync::Arc; + + use crate::assets::{mesh::MeshVertex, model::Model}; + use vulkano::{ + buffer::Subbuffer, + command_buffer::{ + allocator::StandardCommandBufferAllocator, AutoCommandBufferBuilder, + PrimaryAutoCommandBuffer, + }, + }; + + use super::scene::Scene; + + pub struct Node { + id: u32, + pub parent: Option>, + num_of_descendants: u32, + num_of_vertices: u64, + num_of_vertices_in_tree: u64, + num_of_indices: u64, + num_of_indices_in_tree: u64, + // pub local_position: [f32; 3], + pub model: Option, + pub children: Vec, + } + + // when a node changes, update the parent + + impl Node { + pub fn new( + id: u32, + scene_ref: Option<&mut Scene>, + parent: Option>, + model: Option, + ) -> Self { + let (num_of_vertices, num_of_indices, model) = model.map_or((0, 0, None), |mut model| { + // DELETE THIS + if let Some(scene) = scene_ref { + let prev_scene_textures_len = scene.num_of_textures(); + let mut curr_scene_textures_len = prev_scene_textures_len; + + let prev_scene_materials_len = scene.num_of_materials(); + let mut curr_scene_materials_len = prev_scene_materials_len; + + // update material index and texture index for all model meshes (VERY IMPORTANT) + model.textures.iter().for_each(|texture| { + // TODO: use a better texture id/key + // what happens if textures are the same? how do we update the material index to the correct one? + scene + .all_textures + .insert(curr_scene_textures_len, texture.to_owned()); + curr_scene_textures_len += 1; + }); + + model.meshes.iter_mut().for_each(|mesh| { + mesh.update_material_index(prev_scene_textures_len); + }); + + model.materials.iter_mut().for_each(|material| { + material.update_texture_indexs(prev_scene_textures_len); + scene + .all_materials + .insert(curr_scene_materials_len, material.to_owned()); + curr_scene_materials_len += 1; + }); + } + + let num_of_vertices = model.meshes.iter().map(|mesh| mesh.num_of_vertices()).sum(); + let num_of_indices = model.meshes.iter().map(|mesh| mesh.num_of_indices()).sum(); + (num_of_vertices, num_of_indices, Some(model)) + }); + + Self { + id, + parent, + num_of_descendants: 0, + num_of_vertices, + num_of_vertices_in_tree: num_of_vertices, + num_of_indices, + num_of_indices_in_tree: num_of_indices, + children: Vec::new(), + model, + } + } + + pub fn add_node(&mut self, child: Node) { + // child.parent = Some(Arc::new(self.clone())); + self.num_of_descendants += 1 + child.num_of_descendants; + self.num_of_vertices_in_tree += child.num_of_vertices_in_tree; + self.num_of_indices_in_tree += child.num_of_indices_in_tree; + self.children.push(child); + } + + pub fn get_num_of_descendants(&self) -> u32 { + self.num_of_descendants + } + + pub fn get_index_and_vertex_buffers( + &self, + vertex_buffer_arr: &mut Vec>, + indicies_arr: &mut Vec, + command_buffer_builder: &mut AutoCommandBufferBuilder< + PrimaryAutoCommandBuffer, + Arc, + >, + ) { + if let Some(mode) = &self.model { + for mesh in &mode.meshes { + vertex_buffer_arr.push(mesh.vertex_buffer.clone()); + indicies_arr.extend_from_slice(mesh.indices()); + // command_buffer_builder.bind_vertex_buffers(0, mesh.vertex_buffer.clone()); + command_buffer_builder.bind_index_buffer(mesh.index_buffer.clone()); + } + } + + for child in &self.children { + child.get_index_and_vertex_buffers( + vertex_buffer_arr, + indicies_arr, + command_buffer_builder, + ); + } + } + + pub fn id(&self) -> u32 { + self.id + } + + pub fn num_of_indices_in_tree(&self) -> u64 { + self.num_of_indices_in_tree + } + + pub fn num_of_vertices_in_tree(&self) -> u64 { + self.num_of_vertices_in_tree + } + + pub fn num_of_vertices(&self) -> u64 { + self.num_of_vertices + } + + pub fn num_of_indices(&self) -> u64 { + self.num_of_indices + } + } + + } + + pub mod scene{ + use anyhow::{bail, Result}; + use log::info; + use std::{ + fs::File, + io::{Read, Write}, + sync::Arc, + }; + + use crate::assets::{gltf::load_external_gltf, mesh::MeshVertex, model::Model, texture::Texture}; + use glam::Mat4; + + use vulkano::{ + buffer::{ + allocator::{SubbufferAllocator, SubbufferAllocatorCreateInfo}, + subbuffer::BufferReadGuard, + Buffer, BufferCreateInfo, BufferUsage, Subbuffer, + }, + command_buffer::{ + allocator::StandardCommandBufferAllocator, AutoCommandBufferBuilder, CommandBufferUsage, + CopyImageToBufferInfo, PrimaryAutoCommandBuffer, RenderPassBeginInfo, SubpassContents, + }, + descriptor_set::{ + allocator::StandardDescriptorSetAllocator, + layout::{ + DescriptorSetLayout, DescriptorSetLayoutBinding, DescriptorSetLayoutCreateInfo, + DescriptorSetLayoutCreationError, DescriptorType, + }, + PersistentDescriptorSet, WriteDescriptorSet, + }, + device::{Device, DeviceOwned}, + format::Format, + image::{view::ImageView, AttachmentImage, ImageAccess, ImageDimensions, StorageImage}, + memory::allocator::{AllocationCreateInfo, MemoryUsage, StandardMemoryAllocator}, + pipeline::{ + cache::PipelineCache, + graphics::{ + depth_stencil::DepthStencilState, + input_assembly::InputAssemblyState, + vertex_input::Vertex, + viewport::{Viewport, ViewportState}, + }, + layout::{PipelineLayoutCreateInfo, PushConstantRange}, + GraphicsPipeline, Pipeline, PipelineBindPoint, PipelineLayout, + }, + render_pass::{Framebuffer, FramebufferCreateInfo, Subpass}, + sampler::{Filter, Sampler, SamplerAddressMode, SamplerCreateInfo}, + shader::{ShaderModule, ShaderStages}, + single_pass_renderpass, + }; + + use crate::scene::{ + material_manager::OrderedMaterialsMap, node::Node, texture_manager::OrderedTexturesMap, + }; + + pub struct Scene { + root: Node, + vertex_shader: Arc, + fragment_shader: Arc, + descriptor_set_allocator: StandardDescriptorSetAllocator, + texture_sampler: Arc, + // remove pub later + pub all_materials: OrderedMaterialsMap, + memory_allocator: Arc, + cmd_buffer_builder: + AutoCommandBufferBuilder>, + camera_subbuffer_allocator: SubbufferAllocator, + camera_subbuffer: Option>, + pipeline_cache: Arc, + pipeline_layout: Arc, + // remove pub later + pub all_textures: OrderedTexturesMap, + cmd_buffer_allocator: Arc, + queue_family_index: u32, + scene_img: Arc, + scene_img_buffer: Subbuffer<[u8]>, + } + + impl Scene { + const CAMERA_BINDING: u32 = 0; + // tiktok, facetime, (vertical video) [1080, 1920] + const DEFAULT_RESOLUTION: [u32; 2] = [1920, 1080]; + const MAIN_SET: usize = 0; + const MATERIALS_BINDING: u32 = 1; + // gotten from error message - MaxPerStageDescriptorSamplersExceeded + pub const MAX_DESCRIPTOR_COUNT: u32 = 15; + const SCENE_ROOT_ID: u32 = 0; + const TEXTURES_BINDING: u32 = 0; + const TEXTURE_SET: usize = 1; + + pub fn new(device: Arc, queue_family_index: u32) -> Result { + let texture_sampler = Sampler::new( + device.clone(), + SamplerCreateInfo { + mag_filter: Filter::Linear, + min_filter: Filter::Linear, + address_mode: [SamplerAddressMode::Repeat; 3], + ..Default::default() + }, + )?; + + let descriptor_set_allocator = StandardDescriptorSetAllocator::new(device.clone()); + + let root = Node::new(Self::SCENE_ROOT_ID, None, None, None); + + let memory_allocator = Arc::new(StandardMemoryAllocator::new_default(device.clone())); + + let cmd_buffer_allocator = Arc::new(StandardCommandBufferAllocator::new( + device.clone(), + Default::default(), + )); + + let vs = vs::load(device.clone()).expect("failed to load vertex shader"); + let fs = fs::load(device.clone()).expect("failed to load fragment shader"); + + let camera_subbuffer_allocator = SubbufferAllocator::new( + memory_allocator.clone(), + SubbufferAllocatorCreateInfo { + buffer_usage: BufferUsage::UNIFORM_BUFFER, + ..Default::default() + }, + ); + + let pipeline_layout = Self::create_pipeline_layout(device.clone())?; + + let pipeline_cache = Self::retrieve_or_create_pipeline_cache(device)?; + + let cmd_buffer_builder = + Self::create_cmd_buffer_builder(cmd_buffer_allocator.clone(), queue_family_index)?; + + let scene_img = Self::create_scene_img( + Self::DEFAULT_RESOLUTION, + queue_family_index, + &memory_allocator, + )?; + + let scene_img_buffer = Buffer::from_iter( + &memory_allocator, + BufferCreateInfo { + usage: BufferUsage::TRANSFER_DST, + ..Default::default() + }, + AllocationCreateInfo { + usage: MemoryUsage::Upload, + ..Default::default() + }, + (0..Self::DEFAULT_RESOLUTION[0] * Self::DEFAULT_RESOLUTION[1] * 4).map(|_| 0u8), + )?; + + Ok(Self { + root, + camera_subbuffer_allocator, + camera_subbuffer: None, + memory_allocator, + descriptor_set_allocator, + texture_sampler, + cmd_buffer_builder, + vertex_shader: vs, + fragment_shader: fs, + pipeline_layout, + pipeline_cache, + all_materials: OrderedMaterialsMap::new(), + all_textures: OrderedTexturesMap::new(), + cmd_buffer_allocator, + queue_family_index, + scene_img, + scene_img_buffer, + }) + } + + pub fn create_storage_imgs(&self) -> Vec> { + let [width, height] = self.width_height(); + (0..1) + .map(|_| { + StorageImage::new( + &self.memory_allocator, + ImageDimensions::Dim2d { + width, + height, + array_layers: 1, + }, + Format::R8G8B8A8_UNORM, + Some(self.queue_family_index), + ) + .unwrap() + }) + .collect::>() + } + + pub fn add_node(&mut self, node: Node) { + self.root.add_node(node); + } + + pub fn create_node(&mut self, model: Option) -> Node { + Node::new(self.num_of_nodes() + 1, Some(self), None, model) + } + + pub fn load_gltf(&mut self, gltf_path: &str) -> Result { + load_external_gltf( + &self.memory_allocator, + &mut self.cmd_buffer_builder, + gltf_path, + ) + } + + pub fn prepare_and_bind_to_cmd_buffer( + &mut self, + pipeline: &Arc, + camera_model_matrix: Mat4, + ) -> Result<()> { + let cam_buffer = match self.camera_subbuffer.take() { + Some(cam_buffer) => cam_buffer, + None => { + bail!("camera subbuffer shouldn't be None - did you forget to call update_camera?") + } + }; + + let set_zero_layout = match pipeline.layout().set_layouts().get(Self::MAIN_SET){ + Some(set_zero_layout) => set_zero_layout, + None => bail!("pipeline layout doesn't have a set 0 (MAIN_SET). Check the pipeline layout creation and your shader code"), + }; + + let set_one_layout = match pipeline.layout().set_layouts().get(Self::TEXTURE_SET){ + Some(set_one_layout) => set_one_layout, + None => bail!("pipeline layout doesn't have a set 1 (TEXTURE_SET). Check the pipeline layout creation and your shader code"), + }; + + let materials_buffer = Buffer::from_iter( + &self.memory_allocator, + BufferCreateInfo { + usage: BufferUsage::STORAGE_BUFFER, + ..Default::default() + }, + AllocationCreateInfo { + usage: MemoryUsage::Upload, + ..Default::default() + }, + self.all_materials + .iter() + .map(|(_, mat)| mat.to_gpu_material()) + .collect::>(), + )?; + + let camera_and_material_set = PersistentDescriptorSet::new( + self.descriptor_set_allocator(), + set_zero_layout.clone(), + [ + WriteDescriptorSet::buffer(Self::CAMERA_BINDING, cam_buffer), + WriteDescriptorSet::buffer(Self::MATERIALS_BINDING, materials_buffer), + ], + )?; + + let texture_set = PersistentDescriptorSet::new_variable( + self.descriptor_set_allocator(), + set_one_layout.clone(), + self.all_textures.len() as u32, + [WriteDescriptorSet::image_view_sampler_array( + Self::TEXTURES_BINDING, + 0, + self.all_textures + .iter() + .map(|(_, tex)| (tex.image_view.clone() as _, self.texture_sampler.clone())), + )], + )?; + + self.cmd_buffer_builder.bind_descriptor_sets( + PipelineBindPoint::Graphics, + pipeline.layout().clone(), + 0, + vec![camera_and_material_set, texture_set], + ); + + let mut vertex_buffer_arr: Vec> = vec![]; + let mut indicies_arr: Vec = vec![]; + + self.root.get_index_and_vertex_buffers( + &mut vertex_buffer_arr, + &mut indicies_arr, + &mut self.cmd_buffer_builder, + ); + + self.cmd_buffer_builder + .bind_vertex_buffers(0, vertex_buffer_arr); + + // self.cmd_buffer_builder.bind_index_buffer(Buffer::from_iter( + // &self.memory_allocator, + // BufferCreateInfo { usage: BufferUsage::INDEX_BUFFER, ..Default::default() }, + // AllocationCreateInfo { usage: MemoryUsage::Upload, ..Default::default() }, + // indicies_arr, + // )?); + + self.cmd_buffer_builder.push_constants( + pipeline.layout().clone(), + 0, + vs::PushConstants { + model: camera_model_matrix.to_cols_array_2d(), + }, + ); + + self.cmd_buffer_builder + .draw_indexed(self.root.num_of_indices_in_tree() as u32, 1, 0, 0, 0)? + .end_render_pass()?; + Ok(()) + } + + pub fn num_of_nodes(&self) -> u32 { + self.root.get_num_of_descendants() + } + + pub fn num_of_vertices(&self) -> u64 { + self.root.num_of_vertices_in_tree() + } + + pub fn num_of_indices(&self) -> u64 { + self.root.num_of_indices_in_tree() + } + + pub fn num_of_textures(&self) -> u8 { + self.all_textures.len() + } + + pub fn num_of_materials(&self) -> u8 { + self.all_materials.len() + } + + pub fn add_texture(&mut self, texture: Texture) -> u8 { + let index = self.all_textures.len(); + self.all_textures.insert(index, texture); + index + 1 + } + + pub fn descriptor_set_allocator(&self) -> &StandardDescriptorSetAllocator { + &self.descriptor_set_allocator + } + + pub fn memory_allocator(&self) -> Arc { + self.memory_allocator.clone() + } + + pub fn width_height(&self) -> [u32; 2] { + self.scene_img.dimensions().width_height() + } + + pub fn new_gfx_pipeline_and_frame_buffer( + &self, + ) -> Result<(Arc, Arc)> { + // This function is called once during initialization, then again whenever the window is resized. + + let dimensions = self.width_height(); + + let depth_buffer = ImageView::new_default(AttachmentImage::transient( + &self.memory_allocator, + dimensions, + Format::D16_UNORM, + )?)?; + + let render_pass = single_pass_renderpass!( + self.memory_allocator.device().clone(), + attachments: { + color: { + load: Clear, + store: Store, + format: self.scene_img.format(), + samples: 1, + }, + depth: { + load: Clear, + store: DontCare, + format: Format::D16_UNORM, + samples: 1, + }, + }, + pass: { + color: [color], + depth_stencil: {depth}, + }, + )?; + + let frame_buffer = { + let image_view = ImageView::new_default(self.scene_img.clone())?; + Framebuffer::new( + render_pass.clone(), + FramebufferCreateInfo { + attachments: vec![image_view, depth_buffer], + ..Default::default() + }, + )? + }; + + let pipeline = GraphicsPipeline::start() + .build_with_cache(self.pipeline_cache.clone()) + .vertex_input_state(MeshVertex::per_vertex()) + .vertex_shader( + self.vertex_shader + .entry_point("main") + .expect("invalid vertex shader entry point"), + (), + ) + .input_assembly_state(InputAssemblyState::new()) + .viewport_state(ViewportState::viewport_fixed_scissor_irrelevant([ + Viewport { + origin: [0.0, 0.0], + dimensions: [dimensions[0] as f32, dimensions[1] as f32], + depth_range: 0.0..1.0, + }, + ])) + .fragment_shader( + self.fragment_shader + .entry_point("main") + .expect("invalid fragment shader entry point"), + (), + ) + .depth_stencil_state(DepthStencilState::simple_depth_test()) + .render_pass(Subpass::from(render_pass, 0).unwrap()) + .with_pipeline_layout( + self.memory_allocator.device().clone(), + self.pipeline_layout.clone(), + )?; + + info!("Graphics Pipleline Created"); + self.save_pipeline_cache()?; + + // build_with_cache + Ok((pipeline, frame_buffer)) + } + + pub fn build_primary_cmd_buffer( + &mut self, + pipeline: &Arc, + framebuffer: Arc, + camera_model_matrix: Mat4, + ) -> Result { + self.cmd_buffer_builder + .begin_render_pass( + RenderPassBeginInfo { + clear_values: vec![Some([0.0, 0.0, 0.0, 1.0].into()), Some(1f32.into())], + ..RenderPassBeginInfo::framebuffer(framebuffer) + }, + SubpassContents::Inline, + )? + .bind_pipeline_graphics(pipeline.clone()); + + self.prepare_and_bind_to_cmd_buffer(pipeline, camera_model_matrix)?; + + self.cmd_buffer_builder + .copy_image_to_buffer(CopyImageToBufferInfo::image_buffer( + self.scene_img.clone(), + self.scene_img_buffer.clone(), + ))?; + + let new_cmd_buffer_builder = Self::create_cmd_buffer_builder( + self.cmd_buffer_allocator.clone(), + self.queue_family_index, + )?; + let prev_cmd_buffer_builder = + std::mem::replace(&mut self.cmd_buffer_builder, new_cmd_buffer_builder); + + Ok(prev_cmd_buffer_builder.build()?) + } + + pub fn img_buffer_content(&self) -> Result> { + Ok(self.scene_img_buffer.read()?) + } + + pub fn update_camera_subbuffer_allocator(&mut self, camera_info: (Mat4, Mat4)) -> Result<()> { + let uniform_data = vs::UniformBufferObject { + view: camera_info.0.to_cols_array_2d(), + proj: camera_info.1.to_cols_array_2d(), + }; + + let subbuffer = self.camera_subbuffer_allocator.allocate_sized()?; + *subbuffer.write()? = uniform_data; + + self.camera_subbuffer = Some(subbuffer); + Ok(()) + } + + pub fn print_stats(&self) { + info!( + "num of nodes: {} | num of vertices: {} | num of indices: {}", + self.num_of_nodes(), + self.num_of_vertices(), + self.num_of_indices() + ); + } + + pub fn create_pipeline_layout(device: Arc) -> Result> { + // CAMERA, MATERIALS, TEXTURES . in order + let layout_create_infos = vec![ + DescriptorSetLayoutCreateInfo { + bindings: [ + ( + Self::CAMERA_BINDING, + DescriptorSetLayoutBinding { + stages: ShaderStages::VERTEX, + ..DescriptorSetLayoutBinding::descriptor_type( + DescriptorType::UniformBuffer, + ) + }, + ), + ( + Self::MATERIALS_BINDING, + DescriptorSetLayoutBinding { + stages: ShaderStages::FRAGMENT, + ..DescriptorSetLayoutBinding::descriptor_type( + DescriptorType::StorageBuffer, + ) + }, + ), + ] + .into(), + ..Default::default() + }, + DescriptorSetLayoutCreateInfo { + bindings: [( + Self::TEXTURES_BINDING, + DescriptorSetLayoutBinding { + stages: ShaderStages::FRAGMENT, + variable_descriptor_count: true, + descriptor_count: Self::MAX_DESCRIPTOR_COUNT, + ..DescriptorSetLayoutBinding::descriptor_type( + DescriptorType::CombinedImageSampler, + ) + }, + )] + .into(), + ..Default::default() + }, + ]; + + let set_layouts = layout_create_infos + .into_iter() + .map(|desc| DescriptorSetLayout::new(device.clone(), desc)) + .collect::, DescriptorSetLayoutCreationError>>()?; + + let pipeline_layout = PipelineLayout::new( + device, + PipelineLayoutCreateInfo { + set_layouts, + push_constant_ranges: vec![PushConstantRange { + stages: ShaderStages::VERTEX, + offset: 0, + size: std::mem::size_of::() as u32, + }], + ..Default::default() + }, + )?; + + Ok(pipeline_layout) + } + + fn create_cmd_buffer_builder( + cmd_buffer_allocator: Arc, + queue_family_index: u32, + ) -> Result< + AutoCommandBufferBuilder>, + > { + Ok(AutoCommandBufferBuilder::primary( + &cmd_buffer_allocator, + queue_family_index, + CommandBufferUsage::OneTimeSubmit, + )?) + } + + pub fn scene_img(&self) -> &StorageImage { + self.scene_img.as_ref() + } + + fn retrieve_or_create_pipeline_cache(device: Arc) -> Result> { + let data = { + match File::open("pipeline_cache.bin") { + Ok(mut file) => { + let mut data = Vec::new(); + match file.read_to_end(&mut data) { + Ok(_) => { + info!("Using pipeline cache from file"); + Some(data) + } + Err(_) => { + info!("Failed to read pipeline cache file"); + None + } + } + } + Err(_) => None, + } + }; + let pipeline_cache = match data { + // This is unsafe because there is no way to be sure that the file contains valid data. + Some(data) => unsafe { PipelineCache::with_data(device, &data)? }, + None => PipelineCache::empty(device)?, + }; + + Ok(pipeline_cache) + } + + fn save_pipeline_cache(&self) -> Result<()> { + match self.pipeline_cache.get_data() { + Ok(data) => { + // TODO - try to avoid writing to disk if the data is the same + // (i.e. if the cache is already up to date) + // why not open and overwrite the file directly? + if let Ok(mut file) = File::create("pipeline_cache.bin.tmp") { + if file.write_all(&data).is_ok() { + std::fs::rename("pipeline_cache.bin.tmp", "pipeline_cache.bin")?; + } else { + std::fs::remove_file("pipeline_cache.bin.tmp")?; + } + } + } + Err(err) => { + bail!("Error while getting pipeline cache data: {:?}", err); + } + }; + Ok(()) + } + + fn create_scene_img( + dimensions: [u32; 2], + queue_family_index: u32, + memory_allocator: &Arc, + ) -> Result> { + let [width, height] = dimensions; + if height % 2 != 0 || width % 2 != 0 { + bail!("Scene width and height must be divisible by 2"); + } + + Ok(StorageImage::new( + memory_allocator, + ImageDimensions::Dim2d { + width, + height, + array_layers: 1, + }, + Format::R8G8B8A8_UNORM, + Some(queue_family_index), + )?) + } + + pub fn handle_input(&mut self, input: &str) { + info!("input: {}", input); + } + } + + mod vs { + vulkano_shaders::shader! {ty: "vertex",path: "shaders/vert.glsl"} + } + mod fs { + vulkano_shaders::shader! {ty: "fragment", path: "shaders/frag.glsl"} + } + + } + mod texture_manager { + + use crate::assets::texture::Texture; + use std::collections::HashMap; + + #[derive(Debug)] + pub struct OrderedTexturesMap { + map: HashMap, + keys: Vec, + } + + impl Default for OrderedTexturesMap { + fn default() -> Self { + Self::new() + } + } + + impl OrderedTexturesMap { + pub fn new() -> Self { + OrderedTexturesMap { + map: HashMap::new(), + keys: Vec::new(), + } + } + + pub fn insert(&mut self, key: u8, value: Texture) { + let result = self.map.insert(key, value); + if result.is_none() { + self.keys.push(key); + } + } + + pub fn get(&self, key: &u8) -> Option<&Texture> { + self.map.get(key) + } + + pub fn iter(&self) -> impl Iterator { + self.keys + .iter() + .filter_map(move |k| self.map.get_key_value(k)) + } + + pub fn len(&self) -> u8 { + self.map.len() as u8 + } + } + + } +} + +mod routes { + use actix_web::web; + + mod health_check { + use actix_web::{HttpResponse as Resp, Responder}; + + use crate::response::ServerMsg; + + pub async fn handler() -> impl Responder { + Resp::Ok().json(ServerMsg::data("OK")) + } + } + + mod lsdk_webhook { + use actix_web::{http::Method, web, HttpRequest, HttpResponse as Resp, Responder}; + use std::sync::Arc; + + use crate::{ + response::{CommonResponses, ServerMsg}, + state::ServerStateMutex, + utils, + webrtc::TurboLivekitConnector, + }; + use livekit_api::{ + access_token::{self}, + webhooks, + }; + use log::info; + + pub async fn handler( + req: HttpRequest, + server_data: web::Data, + body: web::Bytes, + ) -> impl Responder { + if req.method().ne(&Method::POST) { + return Resp::MethodNotAllowed().json(CommonResponses::MethodNotAllowed.json()); + } + let token_verifier = match access_token::TokenVerifier::new() { + Ok(i) => i, + Err(e) => return Resp::InternalServerError().json(ServerMsg::error(e.to_string())), + }; + let webhook_receiver = webhooks::WebhookReceiver::new(token_verifier); + + let jwt = req + .headers() + .get("Authorization") + .and_then(|hv| hv.to_str().ok()) + .unwrap_or_default() + .to_string(); + + let body = match std::str::from_utf8(&body) { + Ok(i) => i, + Err(e) => return Resp::BadRequest().json(ServerMsg::error(e.to_string())), + }; + + let event = match webhook_receiver.receive(body, &jwt) { + Ok(i) => i, + Err(e) => return Resp::InternalServerError().json(ServerMsg::error(e.to_string())), + }; + + if event.room.is_some() && event.event == "room_started" { + info!("ROOM STARTED 🎉"); + let livekit_protocol::Room { + name: participant_room_name, + max_participants, + num_participants, + .. + } = event.room.unwrap(); + + if num_participants < max_participants { + let mut turbo_webrtc = + match TurboLivekitConnector::new(participant_room_name).await { + Ok(turbo_webrtc) => turbo_webrtc, + Err(e) => { + return Resp::InternalServerError() + .json(ServerMsg::error(format!("{e}"))) + } + }; + + let mut server_data = server_data.lock(); + server_data.turbo_input_tx = Some(turbo_webrtc.get_txt_input_sender()); + server_data.turbo_livekit_connector_handle = Some(turbo_webrtc); + + info!("\nSERVER FINISHED PROCESSING ROOM_STARTED WEBHOOK"); + }; + } else { + info!("received event {}", event.event); + } + + Resp::Ok().json(ServerMsg::data("Livekit Webhook Successfully Processed")) + } + + // images will be in base64 + // stt & images & text go in -> [find a way of batching all this information and sending it to GPT ] -> stream the response from OPENAI to livekit + + // IT SHOULD NEVER TEXT & WRITE AT the same time + } + + pub fn top_level_routes(cfg: &mut web::ServiceConfig) { + cfg.service(web::scope("/").service(web::resource("").to(health_check::handler))) + .service(web::resource("/lsdk-webhook").to(lsdk_webhook::handler)); + } +} + +mod utils { + mod create_bot_token { + use livekit_api::access_token::{AccessToken, VideoGrants}; + + pub fn create_bot_token(room_name: String, ai_name: &str) -> anyhow::Result { + let api_key = std::env::var("LIVEKIT_API_KEY")?; + let api_secret = std::env::var("LIVEKIT_API_SECRET")?; + + let ttl = std::time::Duration::from_secs(60 * 5); // 10 minutes (in sync with frontend) + Ok( + AccessToken::with_api_key(api_key.as_str(), api_secret.as_str()) + .with_ttl(ttl) + .with_identity(ai_name) + .with_name(ai_name) + .with_grants(VideoGrants { + room: room_name, + room_list: true, + room_join: true, + room_admin: true, + can_publish: true, + room_record: true, + can_subscribe: true, + can_publish_data: true, + can_update_own_metadata: true, + ..Default::default() + }) + .to_jwt()?, + ) + } + } + + pub use create_bot_token::*; +} + +mod track_pub { + use std::sync::Arc; + + use livekit::{ + options::{TrackPublishOptions, VideoCodec}, + publication::LocalTrackPublication, + track::{LocalAudioTrack, LocalTrack, LocalVideoTrack, TrackSource}, + webrtc::{ + audio_source::native::NativeAudioSource, + prelude::{AudioSourceOptions, RtcAudioSource}, + video_source::{native::NativeVideoSource, RtcVideoSource}, + }, + Room, RoomError, + }; + + use crate::stt::STT; + + pub struct TracksPublicationData { + pub video_src: NativeVideoSource, + pub video_pub: LocalTrackPublication, + pub audio_src: NativeAudioSource, + pub audio_pub: LocalTrackPublication, + } + const BOT_NAME: &str = "donut"; + + pub async fn publish_tracks(room: Arc) -> Result { + let audio_src = NativeAudioSource::new( + AudioSourceOptions::default(), + STT::SAMPLE_RATE, + STT::NUM_OF_CHANNELS, + ); + let audio_track = LocalAudioTrack::create_audio_track( + BOT_NAME, + RtcAudioSource::Native(audio_src.clone()), + ); + + // TODO: Remove from here and import from Turbo. Resolution{} ? + let (width, height) = (1920, 1080); + let video_src = NativeVideoSource::new(livekit::webrtc::video_source::VideoResolution { + width, + height, + }); + let video_track = LocalVideoTrack::create_video_track( + BOT_NAME, + RtcVideoSource::Native(video_src.clone()), + ); + + let video_publication = room + .local_participant() + .publish_track( + LocalTrack::Video(video_track), + TrackPublishOptions { + source: TrackSource::Camera, + video_codec: VideoCodec::VP8, + ..Default::default() + }, + ) + .await; + let audio_publication = room + .local_participant() + .publish_track( + LocalTrack::Audio(audio_track), + TrackPublishOptions { + source: TrackSource::Microphone, + ..Default::default() + }, + ) + .await; -use log::{error, info}; -use std::{env, thread}; + let video_pub = video_publication?; + let audio_pub = audio_publication?; + Ok(TracksPublicationData { + video_src, + video_pub, + audio_src, + audio_pub, + }) + } +} + +mod stt { + use async_trait::async_trait; + use bytes::{BufMut, Bytes, BytesMut}; + use deepgram::Deepgram; + use ezsockets::{ + client::ClientCloseMode, Client, ClientConfig, CloseFrame, MessageSignal, MessageStatus, + RawMessage, SocketConfig, WSError, + }; + use futures::StreamExt; + use livekit::webrtc::audio_stream::native::NativeAudioStream; + use log::{error, info}; + use parking_lot::Mutex; + use serde::{Deserialize, Serialize}; + use serde_json::{json, Map, Value}; + use std::{ + sync::Arc, + time::{Duration, Instant}, + }; + use tokio::sync::mpsc::UnboundedSender; + + #[derive(Clone)] + pub struct STT { + ws_client: Client, + } + + struct WSClient { + to_gpt: tokio::sync::mpsc::UnboundedSender, + } + + #[async_trait] + impl ezsockets::ClientExt for WSClient { + type Call = (); + + async fn on_text(&mut self, text: String) -> Result<(), ezsockets::Error> { + let data: Value = serde_json::from_str(&text)?; + let transcript_details = data["channel"]["alternatives"][0].clone(); + + info!("\n\n\n received message from deepgram: {data}"); + info!("\n\n\n received message from deepgram: {transcript_details}"); + // if transcript_details!= Value::Null { + // self.to_gpt.send(transcript_details.to_string())?; + // } + Ok(()) + } + + async fn on_binary(&mut self, bytes: Vec) -> Result<(), ezsockets::Error> { + info!("received bytes: {bytes:?}"); + Ok(()) + } + + async fn on_call(&mut self, call: Self::Call) -> Result<(), ezsockets::Error> { + info!("DEEPGRAM ON CALL: {call:?}"); + let () = call; + Ok(()) + } + async fn on_connect(&mut self) -> Result<(), ezsockets::Error> { + info!("DEEPGRAM CONNECTED 🎉"); + Ok(()) + } + + async fn on_connect_fail( + &mut self, + _error: WSError, + ) -> Result { + info!("DEEPGRAM connection FAIL 💔"); + Ok(ClientCloseMode::Reconnect) + } + + async fn on_close( + &mut self, + _frame: Option, + ) -> Result { + info!("DEEPGRAM connection CLOSE 💔"); + Ok(ClientCloseMode::Reconnect) + } + + async fn on_disconnect(&mut self) -> Result { + info!("DEEPGRAM disconnect 💔"); + Ok(ClientCloseMode::Reconnect) + } + } + + impl STT { + pub const LATENCY_FRAMES: f32 = (Self::LATENCY_MS / 1_000.0) * Self::SAMPLE_RATE_F32; + // Uses a delay of `LATENCY_MS` milliseconds in case the default input and output streams are not precisely synchronised + pub const LATENCY_MS: f32 = 5000.0; + pub const LATENCY_SAMPLES: u32 = Self::LATENCY_FRAMES as u32 * Self::NUM_OF_CHANNELS; + pub const NUM_ITERS: usize = 2; + pub const NUM_ITERS_SAVED: usize = 2; + pub const NUM_OF_CHANNELS: u32 = 1; + pub const SAMPLE_RATE: u32 = 44100; //1600 + pub const SAMPLE_RATE_F32: f32 = Self::SAMPLE_RATE as f32; + pub const SAMPLING_FREQ: f32 = Self::SAMPLE_RATE_F32 / 2.0; + + const MIN_AUDIO_MS_CHUNK: u64 = 25; + + pub async fn new( + gpt_input_tx: tokio::sync::mpsc::UnboundedSender, + ) -> anyhow::Result { + let deepgram_api_key = std::env::var("DEEPGRAM_API_KEY") + .expect("The DEEPGRAM_API_KEY env variable is required!"); + + let config = ClientConfig::new("wss://api.deepgram.com/v1/listen") + .socket_config(SocketConfig { + heartbeat: Duration::from_secs(8), + timeout: Duration::from_secs(30 * 60), // 30 minutes + heartbeat_ping_msg_fn: Arc::new(|_t: Duration| { + RawMessage::Text(r#"{ "type": "KeepAlive" }"#.into()) + }), + }) + .header("authorization", &format!("token {}", deepgram_api_key)) + .query_parameter("encoding", "linear16") + .query_parameter("sample_rate", &Self::SAMPLE_RATE.to_string()) + .query_parameter("channels", &Self::NUM_OF_CHANNELS.to_string()) + .query_parameter("model", "2-conversationalai") + .query_parameter("smart_format", "true") + .query_parameter("filler_words", "true") + .query_parameter("version", "latest") + .query_parameter("tier", "nova"); + + let (ws_client, future) = ezsockets::connect( + |_client| WSClient { + to_gpt: gpt_input_tx, + }, + config, + ) + .await; + + Ok(Self { ws_client }) + } + fn send(&self, bytes: impl Into>) -> anyhow::Result { + let signal = self.ws_client.binary(bytes)?; + Ok(signal.status()) + } + } + + pub async fn transcribe(stt_client: STT, mut audio_stream: NativeAudioStream) { + // let mut curr_audio_len = 0.0_f32; // in ms + + let mut starting_time = Instant::now(); + let mut audio_buffer: Vec = Vec::new(); + + while let Some(frame) = audio_stream.next().await { + // curr_audio_len += (num_of_sample as u32 / frame.sample_rate) as f32 /1000.0; + + let num_of_samples = frame.data.len(); + + let mut bytes = BytesMut::with_capacity(num_of_samples * 2); + frame + .data + .iter() + .for_each(|sample| bytes.put_i16_le(*sample)); + + audio_buffer.extend(bytes); + + if starting_time.elapsed() > Duration::from_millis(STT::MIN_AUDIO_MS_CHUNK) { + match stt_client.send(audio_buffer.clone()) { + Ok(status) => info!("Sent audio to deegram | Msg status {status:?}"), + Err(e) => error!("Error sending audio bytes to deepgram ws {e}"), + }; + + starting_time = Instant::now(); + audio_buffer.clear(); + } + } + } + + impl Drop for STT { + fn drop(&mut self) { + if let Err(e) = self.send([]) { + error!("Error shutting down STT / Deepgram connection | Reason - {e}"); + }; + } + } +} + +mod room_events { + use std::sync::Arc; + + use chrono::TimeZone; + use futures::StreamExt; + use livekit::{ + track::RemoteTrack, + webrtc::{ + audio_stream::native::NativeAudioStream, video_stream::native::NativeVideoStream, + }, + DataPacketKind, RoomEvent, + }; + use log::{error, info, warn}; + use parking_lot::{Mutex, MutexGuard, RawMutex}; + use serde::{Deserialize, Serialize}; + + use crate::stt::{transcribe, STT}; + + #[derive(Serialize, Deserialize)] + struct RoomText { + message: String, + timestamp: i64, + } + + pub async fn handle_room_events( + gpt_input_tx: tokio::sync::mpsc::UnboundedSender, + stt_client: STT, + mut room_events: tokio::sync::mpsc::UnboundedReceiver, + ) -> anyhow::Result<()> { + while let Some(event) = room_events.recv().await { + match event { + RoomEvent::TrackSubscribed { + track, + publication: _, + participant: _user, + } => match track { + RemoteTrack::Audio(audio_track) => { + let audio_rtc_track = audio_track.rtc_track(); + let audio_stream = NativeAudioStream::new(audio_rtc_track); + let stt_client_for_thread = stt_client.clone(); + tokio::spawn(transcribe(stt_client_for_thread, audio_stream)); + } + RemoteTrack::Video(video_track) => { + let video_rtc_track = video_track.rtc_track(); + let video_stream = NativeVideoStream::new(video_rtc_track); + tokio::spawn(video_stream_handler(video_stream)); + } + }, + RoomEvent::DataReceived { + payload, + kind, + participant: _user, + } => { + if kind == DataPacketKind::Reliable { + if let Some(payload) = payload.as_ascii() { + let room_text: serde_json::Result = + serde_json::from_str(payload.as_str()); + match room_text { + Ok(room_text) => { + if let Err(e) = + gpt_input_tx.send(format!("{} ", room_text.message)) + { + error!("Couldn't send the text to gpt {e}") + }; + } + Err(e) => { + warn!("Couldn't deserialize room text. {e:#?}"); + } + } + + info!("text from room {:#?}", payload.as_str()); + } + } + } + // RoomEvents::TrackMuted {} =>{ + + // } + _ => info!("incoming event {:?}", event), + } + } + Ok(()) + } + + async fn video_stream_handler(mut video: NativeVideoStream) { + let mut counter = 0_u8; + let max_fps = 10; + + while let Some(frame) = video.next().await { + if counter % max_fps == 0 { + info!("video frame info - {frame:#?}"); + } + + counter = (counter + 1) % max_fps; + } + } +} +mod response { + use std::fmt::Debug; + + use log::warn; + use serde::{Deserialize, Serialize}; + + #[derive(Debug, Serialize, Deserialize)] + pub struct ServerMsg { + data: Option, + error: Option, + } + + impl ServerMsg { + pub fn data(data: T) -> Self { + Self { + data: Some(data), + error: None, + } + } + + pub fn error(error: T) -> Self { + let err_msg = error.to_string(); + warn!("Server error. {err_msg:?}"); + Self { + data: None, + error: Some(err_msg), + } + } + } + + #[derive(Serialize)] + pub struct DefaultGameResponse { + pub board: Vec, + pub state: String, + } + #[derive(Deserialize)] + pub struct PlayDetails { + pub position: u8, + } + + pub enum CommonResponses { + MethodNotAllowed, + } + + impl CommonResponses { + pub fn json(&self) -> ServerMsg { + match self { + CommonResponses::MethodNotAllowed => { + ServerMsg::error("Method not allowed".to_string()) + } + } + } + } +} + +mod gpt { + use std::time::{Duration, Instant}; + + use async_openai::{ + config::OpenAIConfig, + types::{ + ChatCompletionRequestUserMessageArgs, ChatCompletionRequestUserMessageContent, + CreateChatCompletionRequestArgs, + }, + Client, + }; + use futures::StreamExt; + use log::{error, info, warn}; + use tokio::{sync::mpsc, time}; + + use crate::{stt::STT, tts::TTS}; + + /// Stream text chunks to gpt as it's being generated, with <1s latency. + /// Note: if chunks don't end with space or punctuation (" ", ".", "?", "!"), + /// the stream will wait for more text. + /// Used during input streaming to chunk text blocks and set last char to space + pub async fn gpt( + mut text_input_rx: mpsc::UnboundedReceiver, + openai_client: Client, + mut tts_client: TTS, + ) -> anyhow::Result<()> { + let splitters = [ + '.', ',', '?', '!', ';', ':', '—', '-', '(', ')', '[', ']', '}', ' ', + ]; + + let mut txt_buffer = String::new(); + let mut tts_buffer = String::new(); + + let mut req_args = CreateChatCompletionRequestArgs::default(); + let openai_req = req_args.model("gpt-3.5-turbo").max_tokens(512u16); + + // let text_latency = Duration::from_millis(500); + while let Some(chunk) = text_input_rx.recv().await { + txt_buffer.push_str(&chunk); + if ends_with_splitter(&splitters, &txt_buffer) { + let request = openai_req + .messages([ChatCompletionRequestUserMessageArgs::default() + .content(ChatCompletionRequestUserMessageContent::Text( + txt_buffer.clone(), + )) + .build()? + .into()]) + .build()?; + + let mut gpt_resp_stream = openai_client.chat().create_stream(request).await?; + while let Some(result) = gpt_resp_stream.next().await { + match result { + Ok(response) => { + for chat_choice in response.choices { + if let Some(content) = chat_choice.delta.content { + tts_buffer.push_str(&content); + if ends_with_splitter(&splitters, &tts_buffer) { + if let Err(e) = tts_client.send(tts_buffer.clone()) { + error!( + "Coudln't send gpt text chunk to tts channel - {e}" + ); + } else { + tts_buffer.clear(); + }; + } + }; + } + } + Err(err) => { + warn!("chunk error: {err:#?}"); + } + } + } + txt_buffer.clear(); + } else if !txt_buffer.ends_with(' ') { + txt_buffer.push(' '); + } + } + Ok(()) + } + + fn ends_with_splitter(splitters: &[char], chunk: &str) -> bool { + !chunk.is_empty() + && chunk != " " + && splitters.iter().any(|&splitter| chunk.ends_with(splitter)) + } +} + +mod core { + #![allow(unused_parens, non_snake_case)] + use anyhow::{Context, Result}; + use log::{info, warn}; + use std::sync::Arc; + use vulkano::{ + device::{ + physical::{PhysicalDevice, PhysicalDeviceType}, + Device, DeviceCreateInfo, DeviceExtensions, Features, Properties, Queue, + QueueCreateInfo, QueueFlags, + }, + instance::{Instance, InstanceCreateInfo}, + Version, VulkanLibrary, + }; + + pub struct Engine { + vkdevice: Arc, + gfx_queue: Arc, + instance: Arc, + gfx_queue_family_index: u32, + num_of_triangles: u64, + num_of_vertices: u64, + avg_fps: f32, + } + + impl Engine { + pub fn new() -> Result { + warn!("ONLY WORKS / TESTED WITH VULKAN V1.3.250.1"); + let library = VulkanLibrary::new().context("no local Vulkan library/DLL")?; + let instance = Instance::new( + library, + InstanceCreateInfo { + #[cfg(target_os = "macos")] + enumerate_portability: true, + ..Default::default() + }, + ) + .context("failed to create instance")?; + + let mut device_extensions = DeviceExtensions::empty(); + + let features = Features { + dynamic_rendering: true, + fill_mode_non_solid: true, + multi_draw_indirect: true, + runtime_descriptor_array: true, + descriptor_binding_partially_bound: true, + descriptor_binding_variable_descriptor_count: true, + shader_sampled_image_array_non_uniform_indexing: true, + // extended_dynamic_state: true, + ..Features::empty() + }; + + let (processing_device, queue_family_index) = + Self::select_processing_device(&instance, &device_extensions, &features); + + if processing_device.api_version() < Version::V1_3 { + device_extensions.khr_dynamic_rendering = true; + } + + // a device represents an open channel of communication with the GPU/CPU. + let (vkdevice, mut queues) = Device::new( + processing_device, + DeviceCreateInfo { + enabled_extensions: device_extensions, + enabled_features: features, + queue_create_infos: vec![QueueCreateInfo { + queue_family_index, + ..Default::default() + }], + ..Default::default() + }, + ) + .context("failed to create device")?; + + // queues are used to submit work to the device. They are created along with the device. + // they are somewhat like the GPU equivalent of CPU threads. + let gfx_queue = queues + .next() + .expect("failed to get first queue in iterator"); + + Ok(Self { + vkdevice, + gfx_queue, + gfx_queue_family_index: queue_family_index, + instance, + avg_fps: 0.0, + num_of_triangles: 0, + num_of_vertices: 0, + }) + } + + pub fn get_vkdevice(&self) -> Arc { + self.vkdevice.clone() + } + + pub fn get_vkdevice_properties(&self) -> vulkano::device::Properties { + self.vkdevice.physical_device().properties().clone() + } + + pub fn get_gfx_queue(&self) -> Arc { + self.gfx_queue.clone() + } + + pub fn get_instance(&self) -> Arc { + self.instance.clone() + } + + pub fn get_avg_fps(&self) -> f32 { + self.avg_fps + } + + pub fn num_of_triangles(&self) -> u64 { + self.num_of_triangles + } + + pub fn num_of_vertices(&self) -> u64 { + self.num_of_vertices + } + + pub fn gfx_queue_family_index(&self) -> u32 { + self.gfx_queue_family_index + } + + fn select_processing_device( + instance: &Arc, + device_extensions: &DeviceExtensions, + features: &Features, + ) -> (Arc, u32) { + // processing device (CPU/GPU) to connect to + info!("Available devices:"); + let (processing_device, queue_family_index) = instance + .enumerate_physical_devices() + .expect("could not enumerate devices") + .filter(|p| { + let Properties { + device_name, + device_type, + .. + } = &p.properties(); + info!( + "- {} | {:?} | Vulkan v{:?}", + device_name, + device_type, + p.api_version() + ); + p.api_version() >= Version::V1_3 + || p.supported_extensions().khr_dynamic_rendering + }) + .filter(|p| p.supported_extensions().contains(device_extensions)) + .filter(|p| p.supported_features().contains(features)) + .filter_map(|p| { + p.queue_family_properties() + .iter() + .enumerate() + .position(|(_, q)| { + q.queue_flags.intersects(QueueFlags::GRAPHICS) + // && p.surface_support(i as u32, &surface).unwrap_or(false) + }) + .map(|q| (p, q as u32)) + }) + .min_by_key(|(p, _)| match p.properties().device_type { + PhysicalDeviceType::DiscreteGpu => 0, + PhysicalDeviceType::IntegratedGpu => 1, + PhysicalDeviceType::VirtualGpu => 2, + PhysicalDeviceType::Cpu => 3, + PhysicalDeviceType::Other => 4, + _ => 5, + }) + .expect("no devices available"); + + info!("* Using {}", processing_device.properties().device_name,); + + (processing_device, queue_family_index) + } + } +} + +mod tts { + use anyhow::bail; + use async_trait::async_trait; + use base64::{engine::general_purpose, Engine as _}; + use bytes::{Buf, BufMut, Bytes, BytesMut}; + use deepgram::Deepgram; + use ezsockets::{ + client::ClientCloseMode, Client, ClientConfig, CloseFrame, MessageSignal, MessageStatus, + RawMessage, SocketConfig, WSError, + }; + use futures::StreamExt; + use livekit::webrtc::{ + audio_frame::AudioFrame, audio_source::native::NativeAudioSource, + audio_stream::native::NativeAudioStream, native::audio_resampler, + }; + use log::{error, info}; + use parking_lot::Mutex; + use serde::{Deserialize, Serialize}; + use serde_json::{json, Map, Value}; + use std::{ + env, + sync::Arc, + time::{Duration, Instant}, + }; + use tokio::sync::mpsc::UnboundedSender; + + use crate::stt::STT; + + #[derive(Serialize)] + struct VoiceSettings { + stability: f32, + similarity_boost: bool, + } + + #[derive(Serialize)] + struct BOSMessage<'a> { + text: &'a str, + voice_settings: VoiceSettings, + } + + #[derive(Serialize)] + struct EOSMessage<'a> { + text: &'a str, + } + + #[derive(Serialize)] + struct RegularMessage { + text: String, + try_trigger_generation: bool, + } + + struct NormalizedAlignment { + char_start_times_ms: Vec, + chars_durations_ms: Vec, + chars: Vec, + } + struct ElevenLabs { + audio: String, + isFinal: bool, + normalizedAlignment: NormalizedAlignment, + } + + #[derive(Clone)] + pub struct TTS { + ws_client: Option>, + pub started: bool, + eleven_labs_api_key: String, + } + + struct WSClient { + audio_src: NativeAudioSource, + tts_client_ref: Arc>, + } + + fn vec_u8_to_vec_i16(input: Vec) -> Vec { + // Ensure that the input Vec has an even number of elements + if input.len() % 2 != 0 { + panic!("Input Vec must have an even number of elements"); + } + + input + .chunks(2) + .map(|chunk| { + // Convert each pair of u8 to one i16 + // Little-endian order: The first byte is the least significant + i16::from_le_bytes([chunk[0], chunk[1]]) + }) + .collect() + } + + #[async_trait] + impl ezsockets::ClientExt for WSClient { + type Call = (); + + async fn on_text(&mut self, text: String) -> Result<(), ezsockets::Error> { + let data: Value = serde_json::from_str(&text)?; + let transcript_details = data["audio"].clone(); + + if transcript_details != Value::Null { + let data = general_purpose::STANDARD_NO_PAD + .decode(transcript_details.as_str().unwrap())?; + + const FRAME_DURATION: Duration = Duration::from_millis(500); // Write 0.5s of audio at a time + let ms = FRAME_DURATION.as_millis() as u32; + + let num_channels = self.audio_src.num_channels(); + let sample_rate = self.audio_src.sample_rate(); + let num_samples = (sample_rate / 1000 * ms) as usize; + let samples_per_channel = num_samples as u32; + + // let mut resampler = audio_resampler::AudioResampler::default(); + // resampler. + + let audio_frame = AudioFrame { + data: vec_u8_to_vec_i16(data).into(), + num_channels, + sample_rate, + samples_per_channel, + }; + + self.audio_src.capture_frame(&audio_frame).await?; + } else { + error!("received message from eleven labs: {text}"); + } + + Ok(()) + } + + async fn on_binary(&mut self, bytes: Vec) -> Result<(), ezsockets::Error> { + info!("received bytes: {bytes:?}"); + Ok(()) + } + + async fn on_call(&mut self, call: Self::Call) -> Result<(), ezsockets::Error> { + info!("ELEVEN LABS WTF"); + let () = call; + Ok(()) + } + + async fn on_connect(&mut self) -> Result<(), ezsockets::Error> { + info!("ELEVEN LABS CONNECTED 🎉"); + Ok(()) + } + + async fn on_connect_fail( + &mut self, + _error: WSError, + ) -> Result { + info!("ELEVEN LABS connection FAIL 💔"); + Ok(ClientCloseMode::Reconnect) + } + + async fn on_close( + &mut self, + _frame: Option, + ) -> Result { + info!("ELEVEN LABS connection CLOSE 💔"); + let mut tts = self.tts_client_ref.lock(); + tts.started = false; + Ok(ClientCloseMode::Reconnect) + } + + async fn on_disconnect(&mut self) -> Result { + info!("ELEVEN LABS disconnect 💔"); + Ok(ClientCloseMode::Reconnect) + } + } + + impl TTS { + pub fn new() -> anyhow::Result { + let eleven_labs_api_key = std::env::var("ELEVENLABS_API_KEY") + .expect("The ELEVENLABS_API_KEY env variable is required!"); + + Ok(Self { + ws_client: None, + started: false, + eleven_labs_api_key, + }) + } + + pub async fn setup_ws_client( + &mut self, + audio_src: NativeAudioSource, + ) -> anyhow::Result<()> { + let ws_client = self.connect_ws_client(audio_src).await?; + self.started = true; + self.ws_client = Some(ws_client); + Ok(()) + } + + async fn connect_ws_client( + &mut self, + audio_src: NativeAudioSource, + ) -> anyhow::Result> { + let voice_id = "L1oawlP7wF6KPWjLuHcF"; + let model = "eleven_monolingual_v1"; + + let url = url::Url::parse(&format!( + "wss://api.elevenlabs.io/v1/text-to-speech/{voice_id}/stream-input?model_id={model}" + )) + .unwrap(); + + let config = ClientConfig::new(url) + .socket_config(SocketConfig { + heartbeat: Duration::from_secs(10), + timeout: Duration::from_secs(30 * 60), // 30 minutes + heartbeat_ping_msg_fn: Arc::new(|_t: Duration| { + RawMessage::Text( + serde_json::to_string(&RegularMessage { + text: " ".to_string(), + try_trigger_generation: true, + }) + .unwrap(), + ) + }), + }) + .header("xi-api-key", &self.eleven_labs_api_key) + .header("Content-Type", "application/json") + .header("optimize_streaming_latency", "3") + .header("output_format", "pcm_16000"); + + let (ws_client, future) = ezsockets::connect( + |_client| WSClient { + audio_src, + tts_client_ref: Arc::new(Mutex::new(self.clone())), + }, + config, + ) + .await; + + ws_client.text(serde_json::to_string(&BOSMessage { + text: " ", + voice_settings: VoiceSettings { + stability: 0.5, + similarity_boost: false, + }, + })?)?; + Ok(ws_client) + } + + pub fn start(&mut self) -> anyhow::Result<()> { + self.started = true; + self.send(" ".to_string())?; + Ok(()) + } + + pub fn send(&mut self, msg: String) -> anyhow::Result { + let msg = match msg.as_str() { + "" => serde_json::to_string(&EOSMessage { text: "" }), + " " => serde_json::to_string(&BOSMessage { + text: " ", + voice_settings: VoiceSettings { + stability: 0.5, + similarity_boost: false, + }, + }), + msg => serde_json::to_string(&RegularMessage { + text: format!("{msg} "), + try_trigger_generation: true, + }), + }; + let msg = msg?; + + if !self.started { + self.start()?; + } + + if self.ws_client.as_ref().is_none() { + bail!("ws_client is none"); + } + + info!("sending to eleven labs {msg}"); + + Ok(self.ws_client.as_ref().unwrap().text(msg)?.status()) + } + } + + impl Drop for TTS { + fn drop(&mut self) { + info!("DROPPING TTS"); + if let Err(e) = self.send("".to_owned()) { + error!("Error shutting down TTS / Eleven Labs connection | Reason - {e}"); + }; + } + } +} +mod state { + use log::{error, info}; + + use crate::webrtc::TurboLivekitConnector; + + pub type ServerStateMutex = parking_lot::Mutex; + + #[derive(Default)] + pub struct ServerState { + pub turbo_input_tx: Option>, + pub turbo_livekit_connector_handle: Option, + } + + impl ServerState { + pub fn new() -> Self { + Self::default() + } + } + + impl Drop for ServerState { + fn drop(&mut self) { + if let Some(turbo_input_tx) = self.turbo_input_tx.take() { + match turbo_input_tx.send("Goodbye".to_owned()) { + Ok(_) => info!("Turbo Renderer should be exiting..."), + Err(e) => error!("Error closing renderer: {e}"), + }; + } + + if let Some(render_thread_handle) = self.turbo_livekit_connector_handle.take() { + drop(render_thread_handle); + } + } + } +} + +mod turbo { + use std::{ + sync::{mpsc::Receiver, Arc}, + time::{Duration, Instant}, + }; + + use anyhow::{bail, Result}; + + use crate::{ + core::Engine, + scene::{camera::Camera, scene::Scene}, + }; + + use image::{ImageBuffer, Rgba}; + use livekit::webrtc::{ + audio_source::native::NativeAudioSource, + native::yuv_helper, + video_frame::{I420Buffer, VideoFrame, VideoRotation}, + video_source::native::NativeVideoSource, + }; + use log::{error, info, warn}; + use parking_lot::Mutex; + use vulkano::{command_buffer::PrimaryCommandBufferAbstract, sync::GpuFuture}; + + const PIXEL_SIZE: usize = 4; + const FB_WIDTH: usize = 1920; + const FB_HEIGHT: usize = 1080; + + #[derive(Clone)] + struct FrameData { + image: ImageBuffer, Vec>, + framebuffer: Arc>>, + video_frame: Arc>>, + } + + pub struct Turbo { + camera: Camera, + engine: Engine, + scene: Scene, + target_fps: f32, + } + + impl Turbo { + pub fn new() -> Result { + let engine = Engine::new()?; + let scene = Scene::new(engine.get_vkdevice(), engine.gfx_queue_family_index())?; + let camera = Camera::new().update_aspect_ratio_from_scene(scene.width_height()); + + let target_fps = 30.0_f32; + + Ok(Self { + engine, + scene, + camera, + target_fps, + }) + } + + pub fn load_basic_scene(mut self) -> Result { + let donut_model = self.scene.load_gltf("oreo_donut/scene.gltf")?; // , adamHead/adamHead.gltf + let donut_node = self.scene.create_node(Some(donut_model)); + self.scene.add_node(donut_node); + Ok(self) + } + + pub fn get_scene_width_height(&self) -> [u32; 2] { + self.scene.width_height() + } + + pub fn get_fps(&self) -> f32 { + self.target_fps + } + + fn get_fps_i32(&self) -> u32 { + self.target_fps as u32 + } + + pub async fn render( + &mut self, + // render_input_receiver: Receiver, + livekit_rtc_src: NativeVideoSource, + ) -> Result<()> { + let (pipeline, framebuffer) = self.scene.new_gfx_pipeline_and_frame_buffer()?; + let now = Instant::now(); + let rotation_start = now; + let mut num_of_frames = 0; + let fps = self.get_fps_i32(); + + let mut interval = + tokio::time::interval(Duration::from_millis(1000 / self.target_fps as u64)); + + let [w, h] = self.scene.width_height(); + let default_image = ImageBuffer::, Vec>::from_raw( + w, + h, + vec![0u8; FB_WIDTH * FB_HEIGHT * 4], + ) + .unwrap(); + let mut frame_data = FrameData { + image: default_image, + framebuffer: Arc::new(Mutex::new(vec![0u8; FB_WIDTH * FB_HEIGHT * 4])), + video_frame: Arc::new(Mutex::new(VideoFrame { + rotation: VideoRotation::VideoRotation0, + buffer: I420Buffer::new(FB_WIDTH as u32, FB_HEIGHT as u32), + timestamp_us: 0, + })), + }; + + loop { + // if let Some(input) = get_user_input(&render_input_receiver)? { + // let input = input.as_str(); + // if is_exit_cmd(input) { + // return Ok(()); + // } + // self.scene.handle_input(input); + // } + + let elapsed_time = rotation_start.elapsed(); + + self.camera.update_rotation( + elapsed_time.as_secs_f64() as f32 + + elapsed_time.subsec_nanos() as f32 / 20_000_000_000.0, + ); + + self.scene + .update_camera_subbuffer_allocator(self.camera.format_to_subbuffer_data())?; + + let command_buffer = self.scene.build_primary_cmd_buffer( + &pipeline, + framebuffer.clone(), + self.camera.get_model_matrix(), + )?; + + let finished = command_buffer.execute(self.engine.get_gfx_queue())?; + + finished.then_signal_fence_and_flush()?.wait(None)?; + + let buffer_content = self.scene.img_buffer_content()?.to_vec(); + + let [w, h] = self.scene.width_height(); + + let image = match ImageBuffer::, Vec>::from_raw(w, h, buffer_content) { + Some(img) => img, + None => bail!("Failed to create image buffer"), + }; + + frame_data.image = image; + + if let Err(e) = tokio::task::spawn_blocking({ + let frame_data = frame_data.clone(); + let source = livekit_rtc_src.clone(); + move || { + let img_vec = frame_data.image.as_raw(); + let mut framebuffer = frame_data.framebuffer.lock(); + let mut video_frame = frame_data.video_frame.lock(); + let i420_buffer = &mut video_frame.buffer; + + let (stride_y, stride_u, stride_v) = i420_buffer.strides(); + let (data_y, data_u, data_v) = i420_buffer.data_mut(); + + framebuffer.clone_from_slice(img_vec); + + yuv_helper::abgr_to_i420( + &framebuffer, + (FB_WIDTH * PIXEL_SIZE) as u32, + data_y, + stride_y, + data_u, + stride_u, + data_v, + stride_v, + FB_WIDTH as i32, + FB_HEIGHT as i32, + ); + + source.capture_frame(&*video_frame); + } + }) + .await + { + error!("Error sending video frame to livekit {e}"); + }; + + interval.tick().await; + + // if num_of_frames % fps == 0 { + // info!( + // "[ Frame time after limiter : {:.3} ms ]", + // curr_frame_time.elapsed().as_secs_f64() * 1000.0 + // ); + // } + + num_of_frames = (num_of_frames + 1) % fps; + } + } + } + + fn get_user_input(input_revr: &Receiver) -> Result> { + match input_revr.try_recv() { + Ok(input) => Ok(Some(input.clone())), + Err(err) => match err { + std::sync::mpsc::TryRecvError::Empty => Ok(None), + std::sync::mpsc::TryRecvError::Disconnected => { + bail!("Render input channel disconnected") + } + }, + } + } + + fn is_exit_cmd(input: &str) -> bool { + input == "exit" + || input == "quit" + || input == "q" + || input == "bye" + || input == "goodbye" + || input == "ciao" + || input == "adios" + } + + impl Drop for Turbo { + fn drop(&mut self) { + warn!("DROPPED Turbo"); + } + } + + unsafe impl Send for Turbo {} + unsafe impl Sync for Turbo {} +} + +mod webrtc { + use std::sync::Arc; + + use anyhow::Result; + use async_openai::{config::OpenAIConfig, Client as OPENAI_CLIENT}; + use livekit::{publication::LocalTrackPublication, Room}; + + use livekit as lsdk; + use log::{error, info, warn}; + use lsdk::RoomError; + use parking_lot::Mutex; + use tokio::{ + sync::mpsc::{Receiver, UnboundedReceiver}, + task::JoinHandle, + }; + + use crate::{ + gpt::gpt, + room_events::handle_room_events, + stt::STT, + track_pub::{publish_tracks, TracksPublicationData}, + tts::TTS, + turbo::Turbo, + utils, + }; + + pub struct TurboLivekitConnector { + room: Arc, + text_input_tx: tokio::sync::mpsc::UnboundedSender, + cmd_input_sender: std::sync::mpsc::Sender, + room_event_handle: JoinHandle>, + video_pub: LocalTrackPublication, + audio_pub: LocalTrackPublication, + gpt_thread_handle: JoinHandle<()>, + render_thread_handle: Option>, + } + + const BOT_NAME: &str = "talking_donut"; + + impl TurboLivekitConnector { + pub async fn new(participant_room_name: String) -> Result { + // ************** REQUIRED ENV VARS ************** + let open_ai_org_id = std::env::var("OPENAI_ORG_ID").expect("OPENAI_ORG_ID must be"); + let lvkt_url = std::env::var("LIVEKIT_WS_URL").expect("LIVEKIT_WS_URL is not set"); + + // ************** CONNECT TO ROOM ************** + let lvkt_token = utils::create_bot_token(participant_room_name, BOT_NAME)?; + let room_options = lsdk::RoomOptions { + ..Default::default() + }; + let (room, room_events) = + lsdk::Room::connect(&lvkt_url, &lvkt_token, room_options).await?; + info!("Established connection with room. ID -> [{}]", room.name()); + let room = Arc::new(room); + + // ************** CREATE MESSAGING CHANNELS ************** + let (cmd_input_sender, cmd_input_receiver) = std::sync::mpsc::channel::(); + let (gpt_input_tx, gpt_input_rx) = tokio::sync::mpsc::unbounded_channel::(); + let (to_voice_tx, from_gpt_rx) = tokio::sync::mpsc::unbounded_channel::(); + + // ************** SETUP OPENAI, TTS, & STT ************** + let TracksPublicationData { + video_pub, + video_src, + audio_src, + audio_pub, + } = publish_tracks(room.clone()).await?; + + let openai_client = + OPENAI_CLIENT::with_config(OpenAIConfig::new().with_org_id(open_ai_org_id)); + let mut turbo = Turbo::new()?.load_basic_scene()?; + let stt_cleint = STT::new(gpt_input_tx.clone()).await?; + let mut tts_client = TTS::new()?; + tts_client.setup_ws_client(audio_src).await?; + + // ************** CREATE THREADS TO KICK THINGS OFF ************** + let room_event_handle = tokio::spawn(handle_room_events( + gpt_input_tx.clone(), + stt_cleint, + room_events, + )); + + // let tts_receiver_handle = tokio::spawn(tts_receiver(from_gpt_rx, tts_client_for_receiver)); + + // let tts_thread_handle = tokio::spawn(tts.transcribe(main_input_rx)); + + let gpt_thread_handle = tokio::spawn(async { + if let Err(e) = gpt(gpt_input_rx, openai_client, tts_client).await { + error!("GPT thread exited with error: {e}"); + } + }); + + let render_thread_handle = tokio::spawn(async move { + if let Err(e) = turbo.render(video_src).await { + error!("Turbo graphics render thread exited with error: {e}"); + } + }); + + Ok(Self { + room, + text_input_tx: gpt_input_tx, + audio_pub, + video_pub, + room_event_handle, + cmd_input_sender, + gpt_thread_handle, + render_thread_handle: Some(render_thread_handle), + }) + } + + pub fn get_thread_handle(&mut self) -> JoinHandle<()> { + self.render_thread_handle + .take() + .expect("render thread handle should not be None") + } + + pub fn get_txt_input_sender(&mut self) -> tokio::sync::mpsc::UnboundedSender { + self.text_input_tx.clone() + } + + async fn shutdown(&mut self) -> Result<(), RoomError> { + self.room.close().await + } + } + + impl Drop for TurboLivekitConnector { + fn drop(&mut self) { + if let Err(e) = futures::executor::block_on(self.shutdown()) { + warn!("Error shutting down turbo webrtc | {e}"); + }; + } + } + + unsafe impl Send for TurboLivekitConnector {} + unsafe impl Sync for TurboLivekitConnector {} +} +// ACTIX SERVER #[tokio::main] async fn main() -> std::io::Result<()> { // tracing_subscriber::fmt::init(); diff --git a/lkgpt/src/response.rs b/lkgpt/src/response.rs deleted file mode 100644 index fcce4de..0000000 --- a/lkgpt/src/response.rs +++ /dev/null @@ -1,50 +0,0 @@ -use std::fmt::Debug; - -use log::warn; -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Serialize, Deserialize)] -pub struct ServerMsg { - data: Option, - error: Option, -} - -impl ServerMsg { - pub fn data(data: T) -> Self { - Self { - data: Some(data), - error: None, - } - } - - pub fn error(error: T) -> Self { - let err_msg = error.to_string(); - warn!("Server error. {err_msg:?}"); - Self { - data: None, - error: Some(err_msg), - } - } -} - -#[derive(Serialize)] -pub struct DefaultGameResponse { - pub board: Vec, - pub state: String, -} -#[derive(Deserialize)] -pub struct PlayDetails { - pub position: u8, -} - -pub enum CommonResponses { - MethodNotAllowed, -} - -impl CommonResponses { - pub fn json(&self) -> ServerMsg { - match self { - CommonResponses::MethodNotAllowed => ServerMsg::error("Method not allowed".to_string()), - } - } -} diff --git a/lkgpt/src/room_events.rs b/lkgpt/src/room_events.rs deleted file mode 100644 index 1fa7713..0000000 --- a/lkgpt/src/room_events.rs +++ /dev/null @@ -1,91 +0,0 @@ -use std::sync::Arc; - -use chrono::TimeZone; -use futures::StreamExt; -use livekit::{ - track::RemoteTrack, - webrtc::{audio_stream::native::NativeAudioStream, video_stream::native::NativeVideoStream}, - DataPacketKind, RoomEvent, -}; -use log::{error, info, warn}; -use parking_lot::{Mutex, MutexGuard, RawMutex}; -use serde::{Deserialize, Serialize}; - -use crate::stt::{transcribe, STT}; - -#[derive(Serialize, Deserialize)] -struct RoomText { - message: String, - timestamp: i64, -} - -pub async fn handle_room_events( - gpt_input_tx: tokio::sync::mpsc::UnboundedSender, - stt_client: STT, - mut room_events: tokio::sync::mpsc::UnboundedReceiver, -) -> anyhow::Result<()> { - while let Some(event) = room_events.recv().await { - match event { - RoomEvent::TrackSubscribed { - track, - publication: _, - participant: _user, - } => match track { - RemoteTrack::Audio(audio_track) => { - let audio_rtc_track = audio_track.rtc_track(); - let audio_stream = NativeAudioStream::new(audio_rtc_track); - let stt_client_for_thread = stt_client.clone(); - tokio::spawn(transcribe(stt_client_for_thread, audio_stream)); - } - RemoteTrack::Video(video_track) => { - let video_rtc_track = video_track.rtc_track(); - let video_stream = NativeVideoStream::new(video_rtc_track); - tokio::spawn(video_stream_handler(video_stream)); - } - }, - RoomEvent::DataReceived { - payload, - kind, - participant: _user, - } => { - if kind == DataPacketKind::Reliable { - if let Some(payload) = payload.as_ascii() { - let room_text: serde_json::Result = - serde_json::from_str(payload.as_str()); - match room_text { - Ok(room_text) => { - if let Err(e) = gpt_input_tx.send(format!("{} ", room_text.message)) - { - error!("Couldn't send the text to gpt {e}") - }; - } - Err(e) => { - warn!("Couldn't deserialize room text. {e:#?}"); - } - } - - info!("text from room {:#?}", payload.as_str()); - } - } - } - // RoomEvents::TrackMuted {} =>{ - - // } - _ => info!("incoming event {:?}", event), - } - } - Ok(()) -} - -async fn video_stream_handler(mut video: NativeVideoStream) { - let mut counter = 0_u8; - let max_fps = 10; - - while let Some(frame) = video.next().await { - if counter % max_fps == 0 { - info!("video frame info - {frame:#?}"); - } - - counter = (counter + 1) % max_fps; - } -} diff --git a/lkgpt/src/routes/health_check.rs b/lkgpt/src/routes/health_check.rs deleted file mode 100644 index 3b72b4f..0000000 --- a/lkgpt/src/routes/health_check.rs +++ /dev/null @@ -1,7 +0,0 @@ -use actix_web::{HttpResponse as Resp, Responder}; - -use crate::response::ServerMsg; - -pub async fn handler() -> impl Responder { - Resp::Ok().json(ServerMsg::data("OK")) -} diff --git a/lkgpt/src/routes/lsdk_webhook.rs b/lkgpt/src/routes/lsdk_webhook.rs deleted file mode 100644 index 615ed27..0000000 --- a/lkgpt/src/routes/lsdk_webhook.rs +++ /dev/null @@ -1,80 +0,0 @@ -use actix_web::{http::Method, web, HttpRequest, HttpResponse as Resp, Responder}; -use std::sync::Arc; - -use crate::{ - response::{CommonResponses, ServerMsg}, - state::ServerStateMutex, - utils, - webrtc::TurboLivekitConnector, -}; -use livekit_api::{ - access_token::{self}, - webhooks, -}; -use log::info; - -pub async fn handler( - req: HttpRequest, - server_data: web::Data, - body: web::Bytes, -) -> impl Responder { - if req.method().ne(&Method::POST) { - return Resp::MethodNotAllowed().json(CommonResponses::MethodNotAllowed.json()); - } - let token_verifier = match access_token::TokenVerifier::new() { - Ok(i) => i, - Err(e) => return Resp::InternalServerError().json(ServerMsg::error(e.to_string())), - }; - let webhook_receiver = webhooks::WebhookReceiver::new(token_verifier); - - let jwt = req - .headers() - .get("Authorization") - .and_then(|hv| hv.to_str().ok()) - .unwrap_or_default() - .to_string(); - - let body = match std::str::from_utf8(&body) { - Ok(i) => i, - Err(e) => return Resp::BadRequest().json(ServerMsg::error(e.to_string())), - }; - - let event = match webhook_receiver.receive(body, &jwt) { - Ok(i) => i, - Err(e) => return Resp::InternalServerError().json(ServerMsg::error(e.to_string())), - }; - - if event.room.is_some() && event.event == "room_started" { - info!("ROOM STARTED 🎉"); - let livekit_protocol::Room { - name: participant_room_name, - max_participants, - num_participants, - .. - } = event.room.unwrap(); - - if num_participants < max_participants { - let mut turbo_webrtc = match TurboLivekitConnector::new(participant_room_name).await { - Ok(turbo_webrtc) => turbo_webrtc, - Err(e) => { - return Resp::InternalServerError().json(ServerMsg::error(format!("{e}"))) - } - }; - - let mut server_data = server_data.lock(); - server_data.turbo_input_tx = Some(turbo_webrtc.get_txt_input_sender()); - server_data.turbo_livekit_connector_handle = Some(turbo_webrtc); - - info!("\nSERVER FINISHED PROCESSING ROOM_STARTED WEBHOOK"); - }; - } else { - info!("received event {}", event.event); - } - - Resp::Ok().json(ServerMsg::data("Livekit Webhook Successfully Processed")) -} - -// images will be in base64 -// stt & images & text go in -> [find a way of batching all this information and sending it to GPT ] -> stream the response from OPENAI to livekit - -// IT SHOULD NEVER TEXT & WRITE AT the same time diff --git a/lkgpt/src/routes/mod.rs b/lkgpt/src/routes/mod.rs deleted file mode 100644 index 691b5e8..0000000 --- a/lkgpt/src/routes/mod.rs +++ /dev/null @@ -1,8 +0,0 @@ -mod health_check; -mod lsdk_webhook; -use actix_web::web; - -pub fn top_level_routes(cfg: &mut web::ServiceConfig) { - cfg.service(web::scope("/").service(web::resource("").to(health_check::handler))) - .service(web::resource("/lsdk-webhook").to(lsdk_webhook::handler)); -} diff --git a/lkgpt/src/scene/camera.rs b/lkgpt/src/scene/camera.rs deleted file mode 100644 index 5ef96c0..0000000 --- a/lkgpt/src/scene/camera.rs +++ /dev/null @@ -1,61 +0,0 @@ -use glam::{Mat4, Vec3}; - -pub struct Camera { - position: Vec3, - rotation: f32, - fov: f32, - aspect_ratio: f32, - scale: f32, - view: Mat4, - z_near: f32, - z_far: f32, -} - -impl Default for Camera { - fn default() -> Self { - Self::new() - } -} - -impl Camera { - pub fn new() -> Self { - Self { - position: Vec3::new(0.0, 0.0, 0.0), - rotation: 0.0, - fov: 120.0, - aspect_ratio: 16.0 / 9.0, - z_near: 0.1, - z_far: 100.0, - scale: 0.02, - view: Mat4::look_at_rh( - Vec3::new(0.3, 0.3, 1.0), - Vec3::new(0.0, 0.0, 0.0), - Vec3::new(0.0, -1.0, 0.0), - ), - } - } - - pub fn update_aspect_ratio_from_scene(mut self, scene_dims: [u32; 2]) -> Self { - self.aspect_ratio = scene_dims[0] as f32 / scene_dims[1] as f32; - self - } - - pub fn update_rotation(&mut self, rotation: f32) { - self.rotation = rotation; - } - - pub fn format_to_subbuffer_data(&self) -> (Mat4, Mat4) { - let scale_matrix = Mat4::from_scale(Vec3::from_array([self.scale; 3])); - - let view_scale_dot_product = self.view * scale_matrix; - - let projection_matrix = - Mat4::perspective_rh(self.fov, self.aspect_ratio, self.z_near, self.z_far); - - (view_scale_dot_product, projection_matrix) - } - - pub fn get_model_matrix(&self) -> Mat4 { - Mat4::from_rotation_y(self.rotation) - } -} diff --git a/lkgpt/src/scene/material_manager.rs b/lkgpt/src/scene/material_manager.rs deleted file mode 100644 index 98c1816..0000000 --- a/lkgpt/src/scene/material_manager.rs +++ /dev/null @@ -1,44 +0,0 @@ -use crate::assets::materials::MeshMaterial; -use std::collections::HashMap; - -#[derive(Debug)] -pub struct OrderedMaterialsMap { - map: HashMap, - keys: Vec, -} - -impl Default for OrderedMaterialsMap { - fn default() -> Self { - Self::new() - } -} - -impl OrderedMaterialsMap { - pub fn new() -> Self { - OrderedMaterialsMap { - map: HashMap::new(), - keys: Vec::new(), - } - } - - pub fn insert(&mut self, key: u8, value: MeshMaterial) { - let result = self.map.insert(key, value); - if result.is_none() { - self.keys.push(key); - } - } - - pub fn get(&self, key: &u8) -> Option<&MeshMaterial> { - self.map.get(key) - } - - pub fn iter(&self) -> impl Iterator { - self.keys - .iter() - .filter_map(move |k| self.map.get_key_value(k)) - } - - pub fn len(&self) -> u8 { - self.map.len() as u8 - } -} diff --git a/lkgpt/src/scene/mod.rs b/lkgpt/src/scene/mod.rs deleted file mode 100644 index ea38878..0000000 --- a/lkgpt/src/scene/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -pub mod camera; -mod material_manager; -mod node; -pub mod scene; -mod texture_manager; diff --git a/lkgpt/src/scene/node.rs b/lkgpt/src/scene/node.rs deleted file mode 100644 index e81c238..0000000 --- a/lkgpt/src/scene/node.rs +++ /dev/null @@ -1,144 +0,0 @@ -use std::sync::Arc; - -use crate::assets::{mesh::MeshVertex, model::Model}; -use vulkano::{ - buffer::Subbuffer, - command_buffer::{ - allocator::StandardCommandBufferAllocator, AutoCommandBufferBuilder, - PrimaryAutoCommandBuffer, - }, -}; - -use super::scene::Scene; - -pub struct Node { - id: u32, - pub parent: Option>, - num_of_descendants: u32, - num_of_vertices: u64, - num_of_vertices_in_tree: u64, - num_of_indices: u64, - num_of_indices_in_tree: u64, - // pub local_position: [f32; 3], - pub model: Option, - pub children: Vec, -} - -// when a node changes, update the parent - -impl Node { - pub fn new( - id: u32, - scene_ref: Option<&mut Scene>, - parent: Option>, - model: Option, - ) -> Self { - let (num_of_vertices, num_of_indices, model) = model.map_or((0, 0, None), |mut model| { - // DELETE THIS - if let Some(scene) = scene_ref { - let prev_scene_textures_len = scene.num_of_textures(); - let mut curr_scene_textures_len = prev_scene_textures_len; - - let prev_scene_materials_len = scene.num_of_materials(); - let mut curr_scene_materials_len = prev_scene_materials_len; - - // update material index and texture index for all model meshes (VERY IMPORTANT) - model.textures.iter().for_each(|texture| { - // TODO: use a better texture id/key - // what happens if textures are the same? how do we update the material index to the correct one? - scene - .all_textures - .insert(curr_scene_textures_len, texture.to_owned()); - curr_scene_textures_len += 1; - }); - - model.meshes.iter_mut().for_each(|mesh| { - mesh.update_material_index(prev_scene_textures_len); - }); - - model.materials.iter_mut().for_each(|material| { - material.update_texture_indexs(prev_scene_textures_len); - scene - .all_materials - .insert(curr_scene_materials_len, material.to_owned()); - curr_scene_materials_len += 1; - }); - } - - let num_of_vertices = model.meshes.iter().map(|mesh| mesh.num_of_vertices()).sum(); - let num_of_indices = model.meshes.iter().map(|mesh| mesh.num_of_indices()).sum(); - (num_of_vertices, num_of_indices, Some(model)) - }); - - Self { - id, - parent, - num_of_descendants: 0, - num_of_vertices, - num_of_vertices_in_tree: num_of_vertices, - num_of_indices, - num_of_indices_in_tree: num_of_indices, - children: Vec::new(), - model, - } - } - - pub fn add_node(&mut self, child: Node) { - // child.parent = Some(Arc::new(self.clone())); - self.num_of_descendants += 1 + child.num_of_descendants; - self.num_of_vertices_in_tree += child.num_of_vertices_in_tree; - self.num_of_indices_in_tree += child.num_of_indices_in_tree; - self.children.push(child); - } - - pub fn get_num_of_descendants(&self) -> u32 { - self.num_of_descendants - } - - pub fn get_index_and_vertex_buffers( - &self, - vertex_buffer_arr: &mut Vec>, - indicies_arr: &mut Vec, - command_buffer_builder: &mut AutoCommandBufferBuilder< - PrimaryAutoCommandBuffer, - Arc, - >, - ) { - if let Some(mode) = &self.model { - for mesh in &mode.meshes { - vertex_buffer_arr.push(mesh.vertex_buffer.clone()); - indicies_arr.extend_from_slice(mesh.indices()); - // command_buffer_builder.bind_vertex_buffers(0, mesh.vertex_buffer.clone()); - command_buffer_builder.bind_index_buffer(mesh.index_buffer.clone()); - } - } - - for child in &self.children { - child.get_index_and_vertex_buffers( - vertex_buffer_arr, - indicies_arr, - command_buffer_builder, - ); - } - } - - pub fn id(&self) -> u32 { - self.id - } - - pub fn num_of_indices_in_tree(&self) -> u64 { - self.num_of_indices_in_tree - } - - pub fn num_of_vertices_in_tree(&self) -> u64 { - self.num_of_vertices_in_tree - } - - pub fn num_of_vertices(&self) -> u64 { - self.num_of_vertices - } - - pub fn num_of_indices(&self) -> u64 { - self.num_of_indices - } -} diff --git a/lkgpt/src/scene/scene.rs b/lkgpt/src/scene/scene.rs deleted file mode 100644 index 663451d..0000000 --- a/lkgpt/src/scene/scene.rs +++ /dev/null @@ -1,649 +0,0 @@ -use anyhow::{bail, Result}; -use log::info; -use std::{ - fs::File, - io::{Read, Write}, - sync::Arc, -}; - -use crate::assets::{gltf::load_external_gltf, mesh::MeshVertex, model::Model, texture::Texture}; -use glam::Mat4; - -use vulkano::{ - buffer::{ - allocator::{SubbufferAllocator, SubbufferAllocatorCreateInfo}, - subbuffer::BufferReadGuard, - Buffer, BufferCreateInfo, BufferUsage, Subbuffer, - }, - command_buffer::{ - allocator::StandardCommandBufferAllocator, AutoCommandBufferBuilder, CommandBufferUsage, - CopyImageToBufferInfo, PrimaryAutoCommandBuffer, RenderPassBeginInfo, SubpassContents, - }, - descriptor_set::{ - allocator::StandardDescriptorSetAllocator, - layout::{ - DescriptorSetLayout, DescriptorSetLayoutBinding, DescriptorSetLayoutCreateInfo, - DescriptorSetLayoutCreationError, DescriptorType, - }, - PersistentDescriptorSet, WriteDescriptorSet, - }, - device::{Device, DeviceOwned}, - format::Format, - image::{view::ImageView, AttachmentImage, ImageAccess, ImageDimensions, StorageImage}, - memory::allocator::{AllocationCreateInfo, MemoryUsage, StandardMemoryAllocator}, - pipeline::{ - cache::PipelineCache, - graphics::{ - depth_stencil::DepthStencilState, - input_assembly::InputAssemblyState, - vertex_input::Vertex, - viewport::{Viewport, ViewportState}, - }, - layout::{PipelineLayoutCreateInfo, PushConstantRange}, - GraphicsPipeline, Pipeline, PipelineBindPoint, PipelineLayout, - }, - render_pass::{Framebuffer, FramebufferCreateInfo, Subpass}, - sampler::{Filter, Sampler, SamplerAddressMode, SamplerCreateInfo}, - shader::{ShaderModule, ShaderStages}, - single_pass_renderpass, -}; - -use crate::scene::{ - material_manager::OrderedMaterialsMap, node::Node, texture_manager::OrderedTexturesMap, -}; - -pub struct Scene { - root: Node, - vertex_shader: Arc, - fragment_shader: Arc, - descriptor_set_allocator: StandardDescriptorSetAllocator, - texture_sampler: Arc, - // remove pub later - pub all_materials: OrderedMaterialsMap, - memory_allocator: Arc, - cmd_buffer_builder: - AutoCommandBufferBuilder>, - camera_subbuffer_allocator: SubbufferAllocator, - camera_subbuffer: Option>, - pipeline_cache: Arc, - pipeline_layout: Arc, - // remove pub later - pub all_textures: OrderedTexturesMap, - cmd_buffer_allocator: Arc, - queue_family_index: u32, - scene_img: Arc, - scene_img_buffer: Subbuffer<[u8]>, -} - -impl Scene { - const CAMERA_BINDING: u32 = 0; - // tiktok, facetime, (vertical video) [1080, 1920] - const DEFAULT_RESOLUTION: [u32; 2] = [1920, 1080]; - const MAIN_SET: usize = 0; - const MATERIALS_BINDING: u32 = 1; - // gotten from error message - MaxPerStageDescriptorSamplersExceeded - pub const MAX_DESCRIPTOR_COUNT: u32 = 15; - const SCENE_ROOT_ID: u32 = 0; - const TEXTURES_BINDING: u32 = 0; - const TEXTURE_SET: usize = 1; - - pub fn new(device: Arc, queue_family_index: u32) -> Result { - let texture_sampler = Sampler::new( - device.clone(), - SamplerCreateInfo { - mag_filter: Filter::Linear, - min_filter: Filter::Linear, - address_mode: [SamplerAddressMode::Repeat; 3], - ..Default::default() - }, - )?; - - let descriptor_set_allocator = StandardDescriptorSetAllocator::new(device.clone()); - - let root = Node::new(Self::SCENE_ROOT_ID, None, None, None); - - let memory_allocator = Arc::new(StandardMemoryAllocator::new_default(device.clone())); - - let cmd_buffer_allocator = Arc::new(StandardCommandBufferAllocator::new( - device.clone(), - Default::default(), - )); - - let vs = vs::load(device.clone()).expect("failed to load vertex shader"); - let fs = fs::load(device.clone()).expect("failed to load fragment shader"); - - let camera_subbuffer_allocator = SubbufferAllocator::new( - memory_allocator.clone(), - SubbufferAllocatorCreateInfo { - buffer_usage: BufferUsage::UNIFORM_BUFFER, - ..Default::default() - }, - ); - - let pipeline_layout = Self::create_pipeline_layout(device.clone())?; - - let pipeline_cache = Self::retrieve_or_create_pipeline_cache(device)?; - - let cmd_buffer_builder = - Self::create_cmd_buffer_builder(cmd_buffer_allocator.clone(), queue_family_index)?; - - let scene_img = Self::create_scene_img( - Self::DEFAULT_RESOLUTION, - queue_family_index, - &memory_allocator, - )?; - - let scene_img_buffer = Buffer::from_iter( - &memory_allocator, - BufferCreateInfo { - usage: BufferUsage::TRANSFER_DST, - ..Default::default() - }, - AllocationCreateInfo { - usage: MemoryUsage::Upload, - ..Default::default() - }, - (0..Self::DEFAULT_RESOLUTION[0] * Self::DEFAULT_RESOLUTION[1] * 4).map(|_| 0u8), - )?; - - Ok(Self { - root, - camera_subbuffer_allocator, - camera_subbuffer: None, - memory_allocator, - descriptor_set_allocator, - texture_sampler, - cmd_buffer_builder, - vertex_shader: vs, - fragment_shader: fs, - pipeline_layout, - pipeline_cache, - all_materials: OrderedMaterialsMap::new(), - all_textures: OrderedTexturesMap::new(), - cmd_buffer_allocator, - queue_family_index, - scene_img, - scene_img_buffer, - }) - } - - pub fn create_storage_imgs(&self) -> Vec> { - let [width, height] = self.width_height(); - (0..1) - .map(|_| { - StorageImage::new( - &self.memory_allocator, - ImageDimensions::Dim2d { - width, - height, - array_layers: 1, - }, - Format::R8G8B8A8_UNORM, - Some(self.queue_family_index), - ) - .unwrap() - }) - .collect::>() - } - - pub fn add_node(&mut self, node: Node) { - self.root.add_node(node); - } - - pub fn create_node(&mut self, model: Option) -> Node { - Node::new(self.num_of_nodes() + 1, Some(self), None, model) - } - - pub fn load_gltf(&mut self, gltf_path: &str) -> Result { - load_external_gltf( - &self.memory_allocator, - &mut self.cmd_buffer_builder, - gltf_path, - ) - } - - pub fn prepare_and_bind_to_cmd_buffer( - &mut self, - pipeline: &Arc, - camera_model_matrix: Mat4, - ) -> Result<()> { - let cam_buffer = match self.camera_subbuffer.take() { - Some(cam_buffer) => cam_buffer, - None => { - bail!("camera subbuffer shouldn't be None - did you forget to call update_camera?") - } - }; - - let set_zero_layout = match pipeline.layout().set_layouts().get(Self::MAIN_SET){ - Some(set_zero_layout) => set_zero_layout, - None => bail!("pipeline layout doesn't have a set 0 (MAIN_SET). Check the pipeline layout creation and your shader code"), - }; - - let set_one_layout = match pipeline.layout().set_layouts().get(Self::TEXTURE_SET){ - Some(set_one_layout) => set_one_layout, - None => bail!("pipeline layout doesn't have a set 1 (TEXTURE_SET). Check the pipeline layout creation and your shader code"), - }; - - let materials_buffer = Buffer::from_iter( - &self.memory_allocator, - BufferCreateInfo { - usage: BufferUsage::STORAGE_BUFFER, - ..Default::default() - }, - AllocationCreateInfo { - usage: MemoryUsage::Upload, - ..Default::default() - }, - self.all_materials - .iter() - .map(|(_, mat)| mat.to_gpu_material()) - .collect::>(), - )?; - - let camera_and_material_set = PersistentDescriptorSet::new( - self.descriptor_set_allocator(), - set_zero_layout.clone(), - [ - WriteDescriptorSet::buffer(Self::CAMERA_BINDING, cam_buffer), - WriteDescriptorSet::buffer(Self::MATERIALS_BINDING, materials_buffer), - ], - )?; - - let texture_set = PersistentDescriptorSet::new_variable( - self.descriptor_set_allocator(), - set_one_layout.clone(), - self.all_textures.len() as u32, - [WriteDescriptorSet::image_view_sampler_array( - Self::TEXTURES_BINDING, - 0, - self.all_textures - .iter() - .map(|(_, tex)| (tex.image_view.clone() as _, self.texture_sampler.clone())), - )], - )?; - - self.cmd_buffer_builder.bind_descriptor_sets( - PipelineBindPoint::Graphics, - pipeline.layout().clone(), - 0, - vec![camera_and_material_set, texture_set], - ); - - let mut vertex_buffer_arr: Vec> = vec![]; - let mut indicies_arr: Vec = vec![]; - - self.root.get_index_and_vertex_buffers( - &mut vertex_buffer_arr, - &mut indicies_arr, - &mut self.cmd_buffer_builder, - ); - - self.cmd_buffer_builder - .bind_vertex_buffers(0, vertex_buffer_arr); - - // self.cmd_buffer_builder.bind_index_buffer(Buffer::from_iter( - // &self.memory_allocator, - // BufferCreateInfo { usage: BufferUsage::INDEX_BUFFER, ..Default::default() }, - // AllocationCreateInfo { usage: MemoryUsage::Upload, ..Default::default() }, - // indicies_arr, - // )?); - - self.cmd_buffer_builder.push_constants( - pipeline.layout().clone(), - 0, - vs::PushConstants { - model: camera_model_matrix.to_cols_array_2d(), - }, - ); - - self.cmd_buffer_builder - .draw_indexed(self.root.num_of_indices_in_tree() as u32, 1, 0, 0, 0)? - .end_render_pass()?; - Ok(()) - } - - pub fn num_of_nodes(&self) -> u32 { - self.root.get_num_of_descendants() - } - - pub fn num_of_vertices(&self) -> u64 { - self.root.num_of_vertices_in_tree() - } - - pub fn num_of_indices(&self) -> u64 { - self.root.num_of_indices_in_tree() - } - - pub fn num_of_textures(&self) -> u8 { - self.all_textures.len() - } - - pub fn num_of_materials(&self) -> u8 { - self.all_materials.len() - } - - pub fn add_texture(&mut self, texture: Texture) -> u8 { - let index = self.all_textures.len(); - self.all_textures.insert(index, texture); - index + 1 - } - - pub fn descriptor_set_allocator(&self) -> &StandardDescriptorSetAllocator { - &self.descriptor_set_allocator - } - - pub fn memory_allocator(&self) -> Arc { - self.memory_allocator.clone() - } - - pub fn width_height(&self) -> [u32; 2] { - self.scene_img.dimensions().width_height() - } - - pub fn new_gfx_pipeline_and_frame_buffer( - &self, - ) -> Result<(Arc, Arc)> { - // This function is called once during initialization, then again whenever the window is resized. - - let dimensions = self.width_height(); - - let depth_buffer = ImageView::new_default(AttachmentImage::transient( - &self.memory_allocator, - dimensions, - Format::D16_UNORM, - )?)?; - - let render_pass = single_pass_renderpass!( - self.memory_allocator.device().clone(), - attachments: { - color: { - load: Clear, - store: Store, - format: self.scene_img.format(), - samples: 1, - }, - depth: { - load: Clear, - store: DontCare, - format: Format::D16_UNORM, - samples: 1, - }, - }, - pass: { - color: [color], - depth_stencil: {depth}, - }, - )?; - - let frame_buffer = { - let image_view = ImageView::new_default(self.scene_img.clone())?; - Framebuffer::new( - render_pass.clone(), - FramebufferCreateInfo { - attachments: vec![image_view, depth_buffer], - ..Default::default() - }, - )? - }; - - let pipeline = GraphicsPipeline::start() - .build_with_cache(self.pipeline_cache.clone()) - .vertex_input_state(MeshVertex::per_vertex()) - .vertex_shader( - self.vertex_shader - .entry_point("main") - .expect("invalid vertex shader entry point"), - (), - ) - .input_assembly_state(InputAssemblyState::new()) - .viewport_state(ViewportState::viewport_fixed_scissor_irrelevant([ - Viewport { - origin: [0.0, 0.0], - dimensions: [dimensions[0] as f32, dimensions[1] as f32], - depth_range: 0.0..1.0, - }, - ])) - .fragment_shader( - self.fragment_shader - .entry_point("main") - .expect("invalid fragment shader entry point"), - (), - ) - .depth_stencil_state(DepthStencilState::simple_depth_test()) - .render_pass(Subpass::from(render_pass, 0).unwrap()) - .with_pipeline_layout( - self.memory_allocator.device().clone(), - self.pipeline_layout.clone(), - )?; - - info!("Graphics Pipleline Created"); - self.save_pipeline_cache()?; - - // build_with_cache - Ok((pipeline, frame_buffer)) - } - - pub fn build_primary_cmd_buffer( - &mut self, - pipeline: &Arc, - framebuffer: Arc, - camera_model_matrix: Mat4, - ) -> Result { - self.cmd_buffer_builder - .begin_render_pass( - RenderPassBeginInfo { - clear_values: vec![Some([0.0, 0.0, 0.0, 1.0].into()), Some(1f32.into())], - ..RenderPassBeginInfo::framebuffer(framebuffer) - }, - SubpassContents::Inline, - )? - .bind_pipeline_graphics(pipeline.clone()); - - self.prepare_and_bind_to_cmd_buffer(pipeline, camera_model_matrix)?; - - self.cmd_buffer_builder - .copy_image_to_buffer(CopyImageToBufferInfo::image_buffer( - self.scene_img.clone(), - self.scene_img_buffer.clone(), - ))?; - - let new_cmd_buffer_builder = Self::create_cmd_buffer_builder( - self.cmd_buffer_allocator.clone(), - self.queue_family_index, - )?; - let prev_cmd_buffer_builder = - std::mem::replace(&mut self.cmd_buffer_builder, new_cmd_buffer_builder); - - Ok(prev_cmd_buffer_builder.build()?) - } - - pub fn img_buffer_content(&self) -> Result> { - Ok(self.scene_img_buffer.read()?) - } - - pub fn update_camera_subbuffer_allocator(&mut self, camera_info: (Mat4, Mat4)) -> Result<()> { - let uniform_data = vs::UniformBufferObject { - view: camera_info.0.to_cols_array_2d(), - proj: camera_info.1.to_cols_array_2d(), - }; - - let subbuffer = self.camera_subbuffer_allocator.allocate_sized()?; - *subbuffer.write()? = uniform_data; - - self.camera_subbuffer = Some(subbuffer); - Ok(()) - } - - pub fn print_stats(&self) { - info!( - "num of nodes: {} | num of vertices: {} | num of indices: {}", - self.num_of_nodes(), - self.num_of_vertices(), - self.num_of_indices() - ); - } - - pub fn create_pipeline_layout(device: Arc) -> Result> { - // CAMERA, MATERIALS, TEXTURES . in order - let layout_create_infos = vec![ - DescriptorSetLayoutCreateInfo { - bindings: [ - ( - Self::CAMERA_BINDING, - DescriptorSetLayoutBinding { - stages: ShaderStages::VERTEX, - ..DescriptorSetLayoutBinding::descriptor_type( - DescriptorType::UniformBuffer, - ) - }, - ), - ( - Self::MATERIALS_BINDING, - DescriptorSetLayoutBinding { - stages: ShaderStages::FRAGMENT, - ..DescriptorSetLayoutBinding::descriptor_type( - DescriptorType::StorageBuffer, - ) - }, - ), - ] - .into(), - ..Default::default() - }, - DescriptorSetLayoutCreateInfo { - bindings: [( - Self::TEXTURES_BINDING, - DescriptorSetLayoutBinding { - stages: ShaderStages::FRAGMENT, - variable_descriptor_count: true, - descriptor_count: Self::MAX_DESCRIPTOR_COUNT, - ..DescriptorSetLayoutBinding::descriptor_type( - DescriptorType::CombinedImageSampler, - ) - }, - )] - .into(), - ..Default::default() - }, - ]; - - let set_layouts = layout_create_infos - .into_iter() - .map(|desc| DescriptorSetLayout::new(device.clone(), desc)) - .collect::, DescriptorSetLayoutCreationError>>()?; - - let pipeline_layout = PipelineLayout::new( - device, - PipelineLayoutCreateInfo { - set_layouts, - push_constant_ranges: vec![PushConstantRange { - stages: ShaderStages::VERTEX, - offset: 0, - size: std::mem::size_of::() as u32, - }], - ..Default::default() - }, - )?; - - Ok(pipeline_layout) - } - - fn create_cmd_buffer_builder( - cmd_buffer_allocator: Arc, - queue_family_index: u32, - ) -> Result< - AutoCommandBufferBuilder>, - > { - Ok(AutoCommandBufferBuilder::primary( - &cmd_buffer_allocator, - queue_family_index, - CommandBufferUsage::OneTimeSubmit, - )?) - } - - pub fn scene_img(&self) -> &StorageImage { - self.scene_img.as_ref() - } - - fn retrieve_or_create_pipeline_cache(device: Arc) -> Result> { - let data = { - match File::open("pipeline_cache.bin") { - Ok(mut file) => { - let mut data = Vec::new(); - match file.read_to_end(&mut data) { - Ok(_) => { - info!("Using pipeline cache from file"); - Some(data) - } - Err(_) => { - info!("Failed to read pipeline cache file"); - None - } - } - } - Err(_) => None, - } - }; - let pipeline_cache = match data { - // This is unsafe because there is no way to be sure that the file contains valid data. - Some(data) => unsafe { PipelineCache::with_data(device, &data)? }, - None => PipelineCache::empty(device)?, - }; - - Ok(pipeline_cache) - } - - fn save_pipeline_cache(&self) -> Result<()> { - match self.pipeline_cache.get_data() { - Ok(data) => { - // TODO - try to avoid writing to disk if the data is the same - // (i.e. if the cache is already up to date) - // why not open and overwrite the file directly? - if let Ok(mut file) = File::create("pipeline_cache.bin.tmp") { - if file.write_all(&data).is_ok() { - std::fs::rename("pipeline_cache.bin.tmp", "pipeline_cache.bin")?; - } else { - std::fs::remove_file("pipeline_cache.bin.tmp")?; - } - } - } - Err(err) => { - bail!("Error while getting pipeline cache data: {:?}", err); - } - }; - Ok(()) - } - - fn create_scene_img( - dimensions: [u32; 2], - queue_family_index: u32, - memory_allocator: &Arc, - ) -> Result> { - let [width, height] = dimensions; - if height % 2 != 0 || width % 2 != 0 { - bail!("Scene width and height must be divisible by 2"); - } - - Ok(StorageImage::new( - memory_allocator, - ImageDimensions::Dim2d { - width, - height, - array_layers: 1, - }, - Format::R8G8B8A8_UNORM, - Some(queue_family_index), - )?) - } - - pub fn handle_input(&mut self, input: &str) { - info!("input: {}", input); - } -} - -mod vs { - vulkano_shaders::shader! {ty: "vertex",path: "shaders/vert.glsl"} -} -mod fs { - vulkano_shaders::shader! {ty: "fragment", path: "shaders/frag.glsl"} -} diff --git a/lkgpt/src/scene/texture_manager.rs b/lkgpt/src/scene/texture_manager.rs deleted file mode 100644 index 4dd0ae8..0000000 --- a/lkgpt/src/scene/texture_manager.rs +++ /dev/null @@ -1,44 +0,0 @@ -use crate::assets::texture::Texture; -use std::collections::HashMap; - -#[derive(Debug)] -pub struct OrderedTexturesMap { - map: HashMap, - keys: Vec, -} - -impl Default for OrderedTexturesMap { - fn default() -> Self { - Self::new() - } -} - -impl OrderedTexturesMap { - pub fn new() -> Self { - OrderedTexturesMap { - map: HashMap::new(), - keys: Vec::new(), - } - } - - pub fn insert(&mut self, key: u8, value: Texture) { - let result = self.map.insert(key, value); - if result.is_none() { - self.keys.push(key); - } - } - - pub fn get(&self, key: &u8) -> Option<&Texture> { - self.map.get(key) - } - - pub fn iter(&self) -> impl Iterator { - self.keys - .iter() - .filter_map(move |k| self.map.get_key_value(k)) - } - - pub fn len(&self) -> u8 { - self.map.len() as u8 - } -} diff --git a/lkgpt/src/state.rs b/lkgpt/src/state.rs deleted file mode 100644 index 0da89ea..0000000 --- a/lkgpt/src/state.rs +++ /dev/null @@ -1,32 +0,0 @@ -use log::{error, info}; - -use crate::webrtc::TurboLivekitConnector; - -pub type ServerStateMutex = parking_lot::Mutex; - -#[derive(Default)] -pub struct ServerState { - pub turbo_input_tx: Option>, - pub turbo_livekit_connector_handle: Option, -} - -impl ServerState { - pub fn new() -> Self { - Self::default() - } -} - -impl Drop for ServerState { - fn drop(&mut self) { - if let Some(turbo_input_tx) = self.turbo_input_tx.take() { - match turbo_input_tx.send("Goodbye".to_owned()) { - Ok(_) => info!("Turbo Renderer should be exiting..."), - Err(e) => error!("Error closing renderer: {e}"), - }; - } - - if let Some(render_thread_handle) = self.turbo_livekit_connector_handle.take() { - drop(render_thread_handle); - } - } -} diff --git a/lkgpt/src/stt.rs b/lkgpt/src/stt.rs deleted file mode 100644 index f12d52e..0000000 --- a/lkgpt/src/stt.rs +++ /dev/null @@ -1,168 +0,0 @@ -use async_trait::async_trait; -use bytes::{BufMut, Bytes, BytesMut}; -use deepgram::Deepgram; -use ezsockets::{ - client::ClientCloseMode, Client, ClientConfig, CloseFrame, MessageSignal, MessageStatus, - RawMessage, SocketConfig, WSError, -}; -use futures::StreamExt; -use livekit::webrtc::audio_stream::native::NativeAudioStream; -use log::{error, info}; -use parking_lot::Mutex; -use serde::{Deserialize, Serialize}; -use serde_json::{json, Map, Value}; -use std::{ - sync::Arc, - time::{Duration, Instant}, -}; -use tokio::sync::mpsc::UnboundedSender; - -#[derive(Clone)] -pub struct STT { - ws_client: Client, -} - -struct WSClient { - to_gpt: tokio::sync::mpsc::UnboundedSender, -} - -#[async_trait] -impl ezsockets::ClientExt for WSClient { - type Call = (); - - async fn on_text(&mut self, text: String) -> Result<(), ezsockets::Error> { - let data: Value = serde_json::from_str(&text)?; - let transcript_details = data["channel"]["alternatives"][0].clone(); - - info!("received message from deepgram: {transcript_details}"); - self.to_gpt.send(transcript_details.to_string())?; - Ok(()) - } - - async fn on_binary(&mut self, bytes: Vec) -> Result<(), ezsockets::Error> { - info!("received bytes: {bytes:?}"); - Ok(()) - } - - async fn on_call(&mut self, call: Self::Call) -> Result<(), ezsockets::Error> { - info!("DEEPGRAM ON CALL: {call:?}"); - let () = call; - Ok(()) - } - async fn on_connect(&mut self) -> Result<(), ezsockets::Error> { - info!("DEEPGRAM CONNECTED 🎉"); - Ok(()) - } - - async fn on_connect_fail( - &mut self, - _error: WSError, - ) -> Result { - info!("DEEPGRAM connection FAIL 💔"); - Ok(ClientCloseMode::Reconnect) - } - - async fn on_close( - &mut self, - _frame: Option, - ) -> Result { - info!("DEEPGRAM connection CLOSE 💔"); - Ok(ClientCloseMode::Reconnect) - } - - async fn on_disconnect(&mut self) -> Result { - info!("DEEPGRAM disconnect 💔"); - Ok(ClientCloseMode::Reconnect) - } -} - -impl STT { - pub const LATENCY_FRAMES: f32 = (Self::LATENCY_MS / 1_000.0) * Self::SAMPLE_RATE_F32; - // Uses a delay of `LATENCY_MS` milliseconds in case the default input and output streams are not precisely synchronised - pub const LATENCY_MS: f32 = 5000.0; - pub const LATENCY_SAMPLES: u32 = Self::LATENCY_FRAMES as u32 * Self::NUM_OF_CHANNELS; - pub const NUM_ITERS: usize = 2; - pub const NUM_ITERS_SAVED: usize = 2; - pub const NUM_OF_CHANNELS: u32 = 1; - pub const SAMPLE_RATE: u32 = 1600; - pub const SAMPLE_RATE_F32: f32 = Self::SAMPLE_RATE as f32; - pub const SAMPLING_FREQ: f32 = Self::SAMPLE_RATE_F32 / 2.0; - - const MIN_AUDIO_MS_CHUNK: u64 = 20; - - pub async fn new( - gpt_input_tx: tokio::sync::mpsc::UnboundedSender, - ) -> anyhow::Result { - let deepgram_api_key = std::env::var("DEEPGRAM_API_KEY") - .expect("The DEEPGRAM_API_KEY env variable is required!"); - - let config = ClientConfig::new("wss://api.deepgram.com/v1/listen") - .socket_config(SocketConfig { - heartbeat: Duration::from_secs(8), - timeout: Duration::from_secs(30 * 60), // 30 minutes - heartbeat_ping_msg_fn: Arc::new(|_t: Duration| { - RawMessage::Text(r#"{ "type": "KeepAlive" }"#.into()) - }), - }) - .header("authorization", &format!("token {}", deepgram_api_key)) - .query_parameter("encoding", "linear16") - .query_parameter("sample_rate", &Self::SAMPLE_RATE.to_string()) - .query_parameter("channels", &Self::NUM_OF_CHANNELS.to_string()) - .query_parameter("model", "2-conversationalai") - .query_parameter("smart_format", "true") - .query_parameter("filler_words", "true") - .query_parameter("version", "latest") - .query_parameter("tier", "nova"); - - let (ws_client, future) = ezsockets::connect( - |_client| WSClient { - to_gpt: gpt_input_tx, - }, - config, - ) - .await; - - Ok(Self { ws_client }) - } - fn send(&self, bytes: impl Into>) -> anyhow::Result { - let signal = self.ws_client.binary(bytes)?; - Ok(signal.status()) - } -} - -pub async fn transcribe(stt_client: STT, mut audio_stream: NativeAudioStream) { - // let mut curr_audio_len = 0.0_f32; // in ms - - let mut starting_time = Instant::now(); - let audio = ""; - - while let Some(frame) = audio_stream.next().await { - // curr_audio_len += (num_of_sample as u32 / frame.sample_rate) as f32 /1000.0; - - let num_of_samples = frame.data.len(); - - - if starting_time.elapsed() > Duration::from_millis(STT::MIN_AUDIO_MS_CHUNK) { - let mut bytes = BytesMut::with_capacity(num_of_samples * 2); - frame - .data - .iter() - .for_each(|sample| bytes.put_i16_le(*sample)); - - starting_time =Instant::now(); - - match stt_client.send(bytes.freeze()) { - Ok(status) => info!("Sent audio to deegram | Msg status {status:?}"), - Err(e) => error!("Error sending audio bytes to deepgram ws {e}"), - }; - } - } -} - -impl Drop for STT { - fn drop(&mut self) { - if let Err(e) = self.send([]) { - error!("Error shutting down STT / Deepgram connection | Reason - {e}"); - }; - } -} diff --git a/lkgpt/src/track_pub.rs b/lkgpt/src/track_pub.rs deleted file mode 100644 index 1918d50..0000000 --- a/lkgpt/src/track_pub.rs +++ /dev/null @@ -1,71 +0,0 @@ -use std::sync::Arc; - -use livekit::{ - options::{TrackPublishOptions, VideoCodec}, - publication::LocalTrackPublication, - track::{LocalAudioTrack, LocalTrack, LocalVideoTrack, TrackSource}, - webrtc::{ - audio_source::native::NativeAudioSource, - prelude::{AudioSourceOptions, RtcAudioSource}, - video_source::{native::NativeVideoSource, RtcVideoSource}, - }, - Room, RoomError, -}; - -use crate::stt::STT; - -pub struct TracksPublicationData { - pub video_src: NativeVideoSource, - pub video_pub: LocalTrackPublication, - pub audio_src: NativeAudioSource, - pub audio_pub: LocalTrackPublication, -} -const BOT_NAME: &str = "donut"; - -pub async fn publish_tracks(room: Arc) -> Result { - let audio_src = NativeAudioSource::new( - AudioSourceOptions::default(), - STT::SAMPLE_RATE, - STT::NUM_OF_CHANNELS, - ); - let audio_track = - LocalAudioTrack::create_audio_track(BOT_NAME, RtcAudioSource::Native(audio_src.clone())); - - // TODO: Remove from here and import from Turbo. Resolution{} ? - let (width, height) = (1920, 1080); - let video_src = - NativeVideoSource::new(livekit::webrtc::video_source::VideoResolution { width, height }); - let video_track = - LocalVideoTrack::create_video_track(BOT_NAME, RtcVideoSource::Native(video_src.clone())); - - let video_publication = room - .local_participant() - .publish_track( - LocalTrack::Video(video_track), - TrackPublishOptions { - source: TrackSource::Camera, - video_codec: VideoCodec::VP8, - ..Default::default() - }, - ) - .await; - let audio_publication = room - .local_participant() - .publish_track( - LocalTrack::Audio(audio_track), - TrackPublishOptions { - source: TrackSource::Microphone, - ..Default::default() - }, - ) - .await; - - let video_pub = video_publication?; - let audio_pub = audio_publication?; - Ok(TracksPublicationData { - video_src, - video_pub, - audio_src, - audio_pub, - }) -} diff --git a/lkgpt/src/tts.rs b/lkgpt/src/tts.rs deleted file mode 100644 index 3c4b726..0000000 --- a/lkgpt/src/tts.rs +++ /dev/null @@ -1,281 +0,0 @@ -use anyhow::bail; -use async_trait::async_trait; -use base64::{engine::general_purpose, Engine as _}; -use bytes::{Buf, BufMut, Bytes, BytesMut}; -use deepgram::Deepgram; -use ezsockets::{ - client::ClientCloseMode, Client, ClientConfig, CloseFrame, MessageSignal, MessageStatus, - RawMessage, SocketConfig, WSError, -}; -use futures::StreamExt; -use livekit::webrtc::{ - audio_frame::AudioFrame, audio_source::native::NativeAudioSource, - audio_stream::native::NativeAudioStream, native::audio_resampler, -}; -use log::{error, info}; -use parking_lot::Mutex; -use serde::{Deserialize, Serialize}; -use serde_json::{json, Map, Value}; -use std::{ - env, - sync::Arc, - time::{Duration, Instant}, -}; -use tokio::sync::mpsc::UnboundedSender; - -use crate::stt::STT; - -#[derive(Serialize)] -struct VoiceSettings { - stability: f32, - similarity_boost: bool, -} - -#[derive(Serialize)] -struct BOSMessage<'a> { - text: &'a str, - voice_settings: VoiceSettings, -} - -#[derive(Serialize)] -struct EOSMessage<'a> { - text: &'a str, -} - -#[derive(Serialize)] -struct RegularMessage { - text: String, - try_trigger_generation: bool, -} - -struct NormalizedAlignment { - char_start_times_ms: Vec, - chars_durations_ms: Vec, - chars: Vec, -} -struct ElevenLabs { - audio: String, - isFinal: bool, - normalizedAlignment: NormalizedAlignment, -} - -#[derive(Clone)] -pub struct TTS { - ws_client: Option>, - pub started: bool, - eleven_labs_api_key: String, -} - -struct WSClient { - audio_src: NativeAudioSource, - tts_client_ref: Arc>, -} - -fn vec_u8_to_vec_i16(input: Vec) -> Vec { - // Ensure that the input Vec has an even number of elements - if input.len() % 2 != 0 { - panic!("Input Vec must have an even number of elements"); - } - - input - .chunks(2) - .map(|chunk| { - // Convert each pair of u8 to one i16 - // Little-endian order: The first byte is the least significant - i16::from_le_bytes([chunk[0], chunk[1]]) - }) - .collect() -} - -#[async_trait] -impl ezsockets::ClientExt for WSClient { - type Call = (); - - async fn on_text(&mut self, text: String) -> Result<(), ezsockets::Error> { - let data: Value = serde_json::from_str(&text)?; - let transcript_details = data["audio"].clone(); - - if transcript_details != Value::Null { - let data = - general_purpose::STANDARD_NO_PAD.decode(transcript_details.as_str().unwrap())?; - - const FRAME_DURATION: Duration = Duration::from_millis(500); // Write 0.5s of audio at a time - let ms = FRAME_DURATION.as_millis() as u32; - - let num_channels = self.audio_src.num_channels(); - let sample_rate = self.audio_src.sample_rate(); - let num_samples = (sample_rate / 1000 * ms) as usize; - let samples_per_channel = num_samples as u32; - - // let mut resampler = audio_resampler::AudioResampler::default(); - // resampler. - - let audio_frame = AudioFrame { - data: vec_u8_to_vec_i16(data).into(), - num_channels, - sample_rate, - samples_per_channel, - }; - - self.audio_src.capture_frame(&audio_frame).await?; - } else { - error!("received message from eleven labs: {text}"); - } - - Ok(()) - } - - async fn on_binary(&mut self, bytes: Vec) -> Result<(), ezsockets::Error> { - info!("received bytes: {bytes:?}"); - Ok(()) - } - - async fn on_call(&mut self, call: Self::Call) -> Result<(), ezsockets::Error> { - info!("ELEVEN LABS WTF"); - let () = call; - Ok(()) - } - - async fn on_connect(&mut self) -> Result<(), ezsockets::Error> { - info!("ELEVEN LABS CONNECTED 🎉"); - Ok(()) - } - - async fn on_connect_fail( - &mut self, - _error: WSError, - ) -> Result { - info!("ELEVEN LABS connection FAIL 💔"); - Ok(ClientCloseMode::Reconnect) - } - - async fn on_close( - &mut self, - _frame: Option, - ) -> Result { - info!("ELEVEN LABS connection CLOSE 💔"); - let mut tts = self.tts_client_ref.lock(); - tts.started = false; - Ok(ClientCloseMode::Reconnect) - } - - async fn on_disconnect(&mut self) -> Result { - info!("ELEVEN LABS disconnect 💔"); - Ok(ClientCloseMode::Reconnect) - } -} - -impl TTS { - pub fn new() -> anyhow::Result { - let eleven_labs_api_key = std::env::var("ELEVENLABS_API_KEY") - .expect("The ELEVENLABS_API_KEY env variable is required!"); - - Ok(Self { - ws_client: None, - started: false, - eleven_labs_api_key, - }) - } - - pub async fn setup_ws_client(&mut self, audio_src: NativeAudioSource) -> anyhow::Result<()> { - let ws_client = self.connect_ws_client(audio_src).await?; - self.started = true; - self.ws_client = Some(ws_client); - Ok(()) - } - - async fn connect_ws_client( - &mut self, - audio_src: NativeAudioSource, - ) -> anyhow::Result> { - let voice_id = "L1oawlP7wF6KPWjLuHcF"; - let model = "eleven_monolingual_v1"; - - let url = url::Url::parse(&format!( - "wss://api.elevenlabs.io/v1/text-to-speech/{voice_id}/stream-input?model_id={model}" - )) - .unwrap(); - - let config = ClientConfig::new(url) - .socket_config(SocketConfig { - heartbeat: Duration::from_secs(10), - timeout: Duration::from_secs(30 * 60), // 30 minutes - heartbeat_ping_msg_fn: Arc::new(|_t: Duration| { - RawMessage::Text( - serde_json::to_string(&RegularMessage { - text: " ".to_string(), - try_trigger_generation: true, - }) - .unwrap(), - ) - }), - }) - .header("xi-api-key", &self.eleven_labs_api_key) - .header("Content-Type", "application/json") - .header("optimize_streaming_latency", "3") - .header("output_format", "pcm_16000"); - - let (ws_client, future) = ezsockets::connect( - |_client| WSClient { - audio_src, - tts_client_ref: Arc::new(Mutex::new(self.clone())), - }, - config, - ) - .await; - - ws_client.text(serde_json::to_string(&BOSMessage { - text: " ", - voice_settings: VoiceSettings { - stability: 0.5, - similarity_boost: false, - }, - })?)?; - Ok(ws_client) - } - - pub fn start(&mut self) -> anyhow::Result<()> { - self.started = true; - self.send(" ".to_string())?; - Ok(()) - } - - pub fn send(&mut self, msg: String) -> anyhow::Result { - let msg = match msg.as_str() { - "" => serde_json::to_string(&EOSMessage { text: "" }), - " " => serde_json::to_string(&BOSMessage { - text: " ", - voice_settings: VoiceSettings { - stability: 0.5, - similarity_boost: false, - }, - }), - msg => serde_json::to_string(&RegularMessage { - text: format!("{msg} "), - try_trigger_generation: true, - }), - }; - let msg = msg?; - - if !self.started { - self.start()?; - } - - if self.ws_client.as_ref().is_none() { - bail!("ws_client is none"); - } - - info!("sending to eleven labs {msg}"); - - Ok(self.ws_client.as_ref().unwrap().text(msg)?.status()) - } -} - -impl Drop for TTS { - fn drop(&mut self) { - info!("DROPPING TTS"); - if let Err(e) = self.send("".to_owned()) { - error!("Error shutting down TTS / Eleven Labs connection | Reason - {e}"); - }; - } -} diff --git a/lkgpt/src/turbo.rs b/lkgpt/src/turbo.rs deleted file mode 100644 index f9f6ca1..0000000 --- a/lkgpt/src/turbo.rs +++ /dev/null @@ -1,223 +0,0 @@ -use std::{ - sync::{mpsc::Receiver, Arc}, - time::{Duration, Instant}, -}; - -use anyhow::{bail, Result}; - -use crate::{ - core::Engine, - scene::{camera::Camera, scene::Scene}, -}; - -use image::{ImageBuffer, Rgba}; -use livekit::webrtc::{ - audio_source::native::NativeAudioSource, - native::yuv_helper, - video_frame::{I420Buffer, VideoFrame, VideoRotation}, - video_source::native::NativeVideoSource, -}; -use log::{error, info, warn}; -use parking_lot::Mutex; -use vulkano::{command_buffer::PrimaryCommandBufferAbstract, sync::GpuFuture}; - -const PIXEL_SIZE: usize = 4; -const FB_WIDTH: usize = 1920; -const FB_HEIGHT: usize = 1080; - -#[derive(Clone)] -struct FrameData { - image: ImageBuffer, Vec>, - framebuffer: Arc>>, - video_frame: Arc>>, -} - -pub struct Turbo { - camera: Camera, - engine: Engine, - scene: Scene, - target_fps: f32, -} - -impl Turbo { - pub fn new() -> Result { - let engine = Engine::new()?; - let scene = Scene::new(engine.get_vkdevice(), engine.gfx_queue_family_index())?; - let camera = Camera::new().update_aspect_ratio_from_scene(scene.width_height()); - - let target_fps = 30.0_f32; - - Ok(Self { - engine, - scene, - camera, - target_fps, - }) - } - - pub fn load_basic_scene(mut self) -> Result { - let donut_model = self.scene.load_gltf("oreo_donut/scene.gltf")?; // , adamHead/adamHead.gltf - let donut_node = self.scene.create_node(Some(donut_model)); - self.scene.add_node(donut_node); - Ok(self) - } - - pub fn get_scene_width_height(&self) -> [u32; 2] { - self.scene.width_height() - } - - pub fn get_fps(&self) -> f32 { - self.target_fps - } - - fn get_fps_i32(&self) -> u32 { - self.target_fps as u32 - } - - pub async fn render( - &mut self, - // render_input_receiver: Receiver, - livekit_rtc_src: NativeVideoSource, - ) -> Result<()> { - let (pipeline, framebuffer) = self.scene.new_gfx_pipeline_and_frame_buffer()?; - let now = Instant::now(); - let rotation_start = now; - let mut num_of_frames = 0; - let fps = self.get_fps_i32(); - - let mut interval = - tokio::time::interval(Duration::from_millis(1000 / self.target_fps as u64)); - - let [w, h] = self.scene.width_height(); - let default_image = - ImageBuffer::, Vec>::from_raw(w, h, vec![0u8; FB_WIDTH * FB_HEIGHT * 4]) - .unwrap(); - let mut frame_data = FrameData { - image: default_image, - framebuffer: Arc::new(Mutex::new(vec![0u8; FB_WIDTH * FB_HEIGHT * 4])), - video_frame: Arc::new(Mutex::new(VideoFrame { - rotation: VideoRotation::VideoRotation0, - buffer: I420Buffer::new(FB_WIDTH as u32, FB_HEIGHT as u32), - timestamp_us: 0, - })), - }; - - loop { - // if let Some(input) = get_user_input(&render_input_receiver)? { - // let input = input.as_str(); - // if is_exit_cmd(input) { - // return Ok(()); - // } - // self.scene.handle_input(input); - // } - - let elapsed_time = rotation_start.elapsed(); - - self.camera.update_rotation( - elapsed_time.as_secs_f64() as f32 - + elapsed_time.subsec_nanos() as f32 / 20_000_000_000.0, - ); - - self.scene - .update_camera_subbuffer_allocator(self.camera.format_to_subbuffer_data())?; - - let command_buffer = self.scene.build_primary_cmd_buffer( - &pipeline, - framebuffer.clone(), - self.camera.get_model_matrix(), - )?; - - let finished = command_buffer.execute(self.engine.get_gfx_queue())?; - - finished.then_signal_fence_and_flush()?.wait(None)?; - - let buffer_content = self.scene.img_buffer_content()?.to_vec(); - - let [w, h] = self.scene.width_height(); - - let image = match ImageBuffer::, Vec>::from_raw(w, h, buffer_content) { - Some(img) => img, - None => bail!("Failed to create image buffer"), - }; - - frame_data.image = image; - - if let Err(e) = tokio::task::spawn_blocking({ - let frame_data = frame_data.clone(); - let source = livekit_rtc_src.clone(); - move || { - let img_vec = frame_data.image.as_raw(); - let mut framebuffer = frame_data.framebuffer.lock(); - let mut video_frame = frame_data.video_frame.lock(); - let i420_buffer = &mut video_frame.buffer; - - let (stride_y, stride_u, stride_v) = i420_buffer.strides(); - let (data_y, data_u, data_v) = i420_buffer.data_mut(); - - framebuffer.clone_from_slice(img_vec); - - yuv_helper::abgr_to_i420( - &framebuffer, - (FB_WIDTH * PIXEL_SIZE) as u32, - data_y, - stride_y, - data_u, - stride_u, - data_v, - stride_v, - FB_WIDTH as i32, - FB_HEIGHT as i32, - ); - - source.capture_frame(&*video_frame); - } - }) - .await - { - error!("Error sending video frame to livekit {e}"); - }; - - interval.tick().await; - - // if num_of_frames % fps == 0 { - // info!( - // "[ Frame time after limiter : {:.3} ms ]", - // curr_frame_time.elapsed().as_secs_f64() * 1000.0 - // ); - // } - - num_of_frames = (num_of_frames + 1) % fps; - } - } -} - -fn get_user_input(input_revr: &Receiver) -> Result> { - match input_revr.try_recv() { - Ok(input) => Ok(Some(input.clone())), - Err(err) => match err { - std::sync::mpsc::TryRecvError::Empty => Ok(None), - std::sync::mpsc::TryRecvError::Disconnected => { - bail!("Render input channel disconnected") - } - }, - } -} - -fn is_exit_cmd(input: &str) -> bool { - input == "exit" - || input == "quit" - || input == "q" - || input == "bye" - || input == "goodbye" - || input == "ciao" - || input == "adios" -} - -impl Drop for Turbo { - fn drop(&mut self) { - warn!("DROPPED Turbo"); - } -} - -unsafe impl Send for Turbo {} -unsafe impl Sync for Turbo {} diff --git a/lkgpt/src/utils/create_bot_token.rs b/lkgpt/src/utils/create_bot_token.rs deleted file mode 100644 index 699b912..0000000 --- a/lkgpt/src/utils/create_bot_token.rs +++ /dev/null @@ -1,27 +0,0 @@ -use livekit_api::access_token::{AccessToken, VideoGrants}; - -pub fn create_bot_token(room_name: String, ai_name: &str) -> anyhow::Result { - let api_key = std::env::var("LIVEKIT_API_KEY")?; - let api_secret = std::env::var("LIVEKIT_API_SECRET")?; - - let ttl = std::time::Duration::from_secs(60 * 5); // 10 minutes (in sync with frontend) - Ok( - AccessToken::with_api_key(api_key.as_str(), api_secret.as_str()) - .with_ttl(ttl) - .with_identity(ai_name) - .with_name(ai_name) - .with_grants(VideoGrants { - room: room_name, - room_list: true, - room_join: true, - room_admin: true, - can_publish: true, - room_record: true, - can_subscribe: true, - can_publish_data: true, - can_update_own_metadata: true, - ..Default::default() - }) - .to_jwt()?, - ) -} diff --git a/lkgpt/src/utils/mod.rs b/lkgpt/src/utils/mod.rs deleted file mode 100644 index 5ee776c..0000000 --- a/lkgpt/src/utils/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -mod create_bot_token; - -pub use create_bot_token::*; diff --git a/lkgpt/src/webrtc.rs b/lkgpt/src/webrtc.rs deleted file mode 100644 index 254a1c9..0000000 --- a/lkgpt/src/webrtc.rs +++ /dev/null @@ -1,133 +0,0 @@ -use std::sync::Arc; - -use anyhow::Result; -use async_openai::{config::OpenAIConfig, Client as OPENAI_CLIENT}; -use livekit::{publication::LocalTrackPublication, Room}; - -use livekit as lsdk; -use log::{error, info, warn}; -use lsdk::RoomError; -use parking_lot::Mutex; -use tokio::{ - sync::mpsc::{Receiver, UnboundedReceiver}, - task::JoinHandle, -}; - -use crate::{ - gpt::gpt, - room_events::handle_room_events, - stt::STT, - track_pub::{publish_tracks, TracksPublicationData}, - tts::TTS, - turbo::Turbo, - utils, -}; - -pub struct TurboLivekitConnector { - room: Arc, - text_input_tx: tokio::sync::mpsc::UnboundedSender, - cmd_input_sender: std::sync::mpsc::Sender, - room_event_handle: JoinHandle>, - video_pub: LocalTrackPublication, - audio_pub: LocalTrackPublication, - gpt_thread_handle: JoinHandle<()>, - render_thread_handle: Option>, -} - -const BOT_NAME: &str = "talking_donut"; - -impl TurboLivekitConnector { - pub async fn new(participant_room_name: String) -> Result { - // ************** REQUIRED ENV VARS ************** - let open_ai_org_id = std::env::var("OPENAI_ORG_ID").expect("OPENAI_ORG_ID must be"); - let lvkt_url = std::env::var("LIVEKIT_WS_URL").expect("LIVEKIT_WS_URL is not set"); - - // ************** CONNECT TO ROOM ************** - let lvkt_token = utils::create_bot_token(participant_room_name, BOT_NAME)?; - let room_options = lsdk::RoomOptions { - ..Default::default() - }; - let (room, room_events) = lsdk::Room::connect(&lvkt_url, &lvkt_token, room_options).await?; - info!("Established connection with room. ID -> [{}]", room.name()); - let room = Arc::new(room); - - // ************** CREATE MESSAGING CHANNELS ************** - let (cmd_input_sender, cmd_input_receiver) = std::sync::mpsc::channel::(); - let (gpt_input_tx, gpt_input_rx) = tokio::sync::mpsc::unbounded_channel::(); - let (to_voice_tx, from_gpt_rx) = tokio::sync::mpsc::unbounded_channel::(); - - // ************** SETUP OPENAI, TTS, & STT ************** - let TracksPublicationData { - video_pub, - video_src, - audio_src, - audio_pub, - } = publish_tracks(room.clone()).await?; - - let openai_client = - OPENAI_CLIENT::with_config(OpenAIConfig::new().with_org_id(open_ai_org_id)); - let mut turbo = Turbo::new()?.load_basic_scene()?; - let stt_cleint = STT::new(gpt_input_tx.clone()).await?; - let mut tts_client = TTS::new()?; - tts_client.setup_ws_client(audio_src).await?; - - // ************** CREATE THREADS TO KICK THINGS OFF ************** - let room_event_handle = tokio::spawn(handle_room_events( - gpt_input_tx.clone(), - stt_cleint, - room_events, - )); - - // let tts_receiver_handle = tokio::spawn(tts_receiver(from_gpt_rx, tts_client_for_receiver)); - - // let tts_thread_handle = tokio::spawn(tts.transcribe(main_input_rx)); - - let gpt_thread_handle = tokio::spawn(async { - if let Err(e) = gpt(gpt_input_rx, openai_client, tts_client).await { - error!("GPT thread exited with error: {e}"); - } - }); - - let render_thread_handle = tokio::spawn(async move { - if let Err(e) = turbo.render(video_src).await { - error!("Turbo graphics render thread exited with error: {e}"); - } - }); - - Ok(Self { - room, - text_input_tx: gpt_input_tx, - audio_pub, - video_pub, - room_event_handle, - cmd_input_sender, - gpt_thread_handle, - render_thread_handle: Some(render_thread_handle), - }) - } - - pub fn get_thread_handle(&mut self) -> JoinHandle<()> { - self.render_thread_handle - .take() - .expect("render thread handle should not be None") - } - - pub fn get_txt_input_sender(&mut self) -> tokio::sync::mpsc::UnboundedSender { - self.text_input_tx.clone() - } - - async fn shutdown(&mut self) -> Result<(), RoomError> { - self.room.close().await - } -} - -impl Drop for TurboLivekitConnector { - fn drop(&mut self) { - if let Err(e) = futures::executor::block_on(self.shutdown()) { - warn!("Error shutting down turbo webrtc | {e}"); - }; - } -} - -unsafe impl Send for TurboLivekitConnector {} -unsafe impl Sync for TurboLivekitConnector {}